diff --git a/.github/workflows/test-shared.yml b/.github/workflows/test-shared.yml new file mode 100644 index 00000000000000..b9e93b5cd7f590 --- /dev/null +++ b/.github/workflows/test-shared.yml @@ -0,0 +1,85 @@ +name: Test Shared librairies + +on: + pull_request: + paths-ignore: + - .mailmap + - README.md + - .github/** + - '!.github/workflows/test-shared.yml' + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: + - main + - canary + - v[0-9]+.x-staging + - v[0-9]+.x + paths-ignore: + - .mailmap + - README.md + - .github/** + - '!.github/workflows/test-shared.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FLAKY_TESTS: keep_retrying + NIXPKGS_REPO: https://github.com/NixOS/nixpkgs + NIXPKGS_PIN: 97eb7ee0da337d385ab015a23e15022c865be75c + +permissions: + contents: read + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + system: x86_64-linux + - runner: ubuntu-24.04-arm + system: aarch64-linux + - runner: macos-13 + system: x86_64-darwin + - runner: macos-14 + system: aarch64-darwin + name: '${{ matrix.system }}: with shared libraries' + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31 + with: + extra_nix_config: sandbox = true + github_access_token: ${{ secrets.GITHUB_TOKEN }} + nix_path: nixpkgs=${{ env.NIXPKGS_REPO }}/archive/${{ env.NIXPKGS_PIN }}.tar.gz + + - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16 + with: + name: nodejs + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - name: Configure sccache + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + core.exportVariable('SCCACHE_GHA_VERSION', 'on'); + core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on'); + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + + - name: Build Node.js and run tests + run: | + nix-shell \ + --pure --keep FLAKY_TESTS \ + --keep SCCACHE_GHA_VERSION --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \ + --arg loadJSBuiltinsDynamically false \ + --arg ccache '(import {}).sccache' \ + --arg devTools '[]' \ + --arg benchmarkTools '[]' \ + --run ' + make run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS" + ' diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index e6bb4cef52c5bd..a2a0d605cce259 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -30,6 +30,7 @@ on: - llhttp - minimatch - nbytes + - nixpkgs-unstable - nghttp2 - nghttp3 - ngtcp2 @@ -198,6 +199,14 @@ jobs: cat temp-output tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true rm temp-output + - id: nixpkgs-unstable + subsystem: tools + label: tools + run: | + ./tools/dep_updaters/update-nixpkgs-pin.sh > temp-output + cat temp-output + tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true + rm temp-output - id: nghttp2 subsystem: deps label: dependencies diff --git a/BUILDING.md b/BUILDING.md index 41ec280b478a4d..d33fc69fd4f966 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -239,6 +239,7 @@ Consult previous versions of this document for older versions of Node.js: Installation via Linux package manager can be achieved with: +* Nix, NixOS: `nix-shell` * Ubuntu, Debian: `sudo apt-get install python3 g++-12 gcc-12 make python3-pip` * Fedora: `sudo dnf install python3 gcc-c++ make python3-pip` * CentOS and RHEL: `sudo yum install python3 gcc-c++ make python3-pip` @@ -259,6 +260,75 @@ installed, you can find them under the menu `Xcode -> Open Developer Tool -> More Developer Tools...`. This step will install `clang`, `clang++`, and `make`. +#### Nix integration + +If you are using Nix and direnv, you can use the following to get started: + +```bash +echo 'use_nix --arg sharedLibDeps {} --argstr icu small' > .envrc +direnv allow . +make build-ci -j12 +``` + +The use of `make build-ci` is to ensure you are using the `CONFIG_FLAGS` +environment variable. You can also specify it manually: + +```sh +./configure $CONFIG_FLAGS +make -j12 +``` + +Passing the `--arg sharedLibDeps {}` instructs direnv and Nix to generate an +environment that uses the vendored-in native dependencies. Using the vendored-in +dependencies result in a result closer to the official binaries, the tradeoff +being the build will take longer to complete as you'd have to build those +dependencies instead of using the cached ones from the Nix cache. You can omit +that flag to use all the shared dependencies, or specify only some dependencies: + +```bash +cat -> .envrc <<'EOF' +use nix --arg sharedLibDeps '{ + inherit (import {}) + openssl + zlib + ; +}' +EOF +``` + +Passing the `--argstr icu small` instructs direnv and Nix to pass `--with-intl=small` in +the `CONFIG_FLAGS` environment variable. If you omit this, the prebuilt ICU from Nix cache +will be used, which should speed up greatly compilation time. + +The use of `direnv` is completely optional, you can also use `nix-shell` directly, +e.g. here's a command you can use to build a binary for benchmarking purposes: + +```bash +# Passing `--arg loadJSBuiltinsDynamically false` to instruct the compiler to +# embed the JS core files so it is no longer affected by local changes +# (necessary for getting useful benchmark results). +# Passing `--arg devTools '[]' --arg benchmarkTools '[]'` since we don't need +# those to build node. +nix-shell \ + --arg loadJSBuiltinsDynamically false \ + --arg devTools '[]' --arg benchmarkTools '[]' \ + --run 'make build-ci -j12' + +mv out/Release/node ./node_old + +# ... +# Make your local changes, and re-build node + +nix-shell \ + --arg loadJSBuiltinsDynamically false \ + --arg devTools '[]' --arg benchmarkTools '[]' \ + --run 'make build-ci -j12' + +nix-shell --pure --run './node benchmark/compare.js --old ./node_old --new ./node http | Rscript benchmark/compare.R' +``` + +There are additional attributes you can pass, see `shell.nix` file for more details. + #### Building Node.js If the path to your build directory contains a space, the build will likely @@ -267,7 +337,6 @@ fail. To build Node.js: ```bash -export CXX=g++-12 ./configure make -j4 ``` diff --git a/deps/npm/node_modules/node-gyp/gyp/pylib/gyp/xcode_emulation.py b/deps/npm/node_modules/node-gyp/gyp/pylib/gyp/xcode_emulation.py index 85a63dfd7ae0e2..48aa22bf899b8e 100644 --- a/deps/npm/node_modules/node-gyp/gyp/pylib/gyp/xcode_emulation.py +++ b/deps/npm/node_modules/node-gyp/gyp/pylib/gyp/xcode_emulation.py @@ -521,7 +521,7 @@ def _GetSdkVersionInfoItem(self, sdk, infoitem): # most sensible route and should still do the right thing. try: return GetStdoutQuiet(["xcrun", "--sdk", sdk, infoitem]) - except GypError: + except (GypError, OSError): pass def _SdkRoot(self, configname): @@ -1355,7 +1355,7 @@ def _DefaultSdkRoot(self): return default_sdk_root try: all_sdks = GetStdout(["xcodebuild", "-showsdks"]) - except GypError: + except (GypError, OSError): # If xcodebuild fails, there will be no valid SDKs return "" for line in all_sdks.splitlines(): @@ -1509,7 +1509,7 @@ def XcodeVersion(): raise GypError("xcodebuild returned unexpected results") version = version_list[0].split()[-1] # Last word on first line build = version_list[-1].split()[-1] # Last word on last line - except GypError: # Xcode not installed so look for XCode Command Line Tools + except (GypError, OSError): # Xcode not installed so look for XCode Command Line Tools version = CLTVersion() # macOS Catalina returns 11.0.0.0.1.1567737322 if not version: raise GypError("No Xcode or CLT version detected!") @@ -1542,15 +1542,15 @@ def CLTVersion(): try: output = GetStdout(["/usr/sbin/pkgutil", "--pkg-info", key]) return re.search(regex, output).groupdict()["version"] - except GypError: + except (GypError, OSError): continue regex = re.compile(r"Command Line Tools for Xcode\s+(?P\S+)") try: output = GetStdout(["/usr/sbin/softwareupdate", "--history"]) return re.search(regex, output).groupdict()["version"] - except GypError: - return None + except (GypError, OSError): + return "11.0.0.0.1.1567737322" def GetStdoutQuiet(cmdlist): diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md index a60a06b695f242..1dcb2a62a952bb 100644 --- a/doc/contributing/writing-and-running-benchmarks.md +++ b/doc/contributing/writing-and-running-benchmarks.md @@ -26,6 +26,10 @@ Basic Unix tools are required for some benchmarks. [Git for Windows][git-for-windows] includes Git Bash and the necessary tools, which need to be included in the global Windows `PATH`. +If you are using Nix, all the required tools are already listed in the +`benchmarkTools` argument of the `shell.nix` file, so you can skip those +prerequesites. + ### HTTP benchmark requirements Most of the HTTP benchmarks require a benchmarker to be installed. This can be diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000000000..3e537192cf13c1 --- /dev/null +++ b/shell.nix @@ -0,0 +1,124 @@ +{ + pkgs ? import {}, + loadJSBuiltinsDynamically ? true, # Load `lib/**.js` from disk instead of embedding + ncu-path ? null, # Provide this if you want to use a local version of NCU + icu ? pkgs.icu, + sharedLibDeps ? { + inherit (pkgs) + ada + brotli + c-ares + libuv + nghttp2 + nghttp3 + ngtcp2 + openssl + simdjson + sqlite + zlib + zstd + ; + http-parser = pkgs.llhttp; + simdutf = pkgs.simdutf.overrideAttrs { + version = "6.5.0"; + + src = pkgs.fetchFromGitHub { + owner = "simdutf"; + repo = "simdutf"; + tag = "v6.5.0"; + hash = "sha256-bZ4r62GMz2Dkd3fKTJhelitaA8jUBaDjG6jOysEg8Nk="; + }; + }; + uvwasi = pkgs.uvwasi.overrideAttrs { + version = "0.0.22"; + + src = pkgs.fetchFromGitHub { + owner = "nodejs"; + repo = "uvwasi"; + rev = "refs/heads/v0.0.22-proposal"; + hash = "sha256-vrtL9HDzbbbx3QmRsifRzNTYilC4zgo9JmJoA9haUFQ="; + }; + postPatch = '' + substituteInPlace CMakeLists.txt \ + --replace-fail 'list(APPEND uvwasi_cflags -fvisibility=hidden --std=gnu89)' 'list(APPEND uvwasi_cflags --std=gnu89)' + ''; + cmakeFlags = [ + "-DUVWASI_BUILD_TESTS=OFF" + ]; + }; + }, + ccache ? pkgs.ccache, + ninja ? pkgs.ninja, + devTools ? [ + pkgs.curl + pkgs.gh + pkgs.git + pkgs.jq + pkgs.shellcheck + ] ++ (if (ncu-path == null) then [ + pkgs.node-core-utils + ] else [ + (pkgs.writeShellScriptBin "git-node" "exec \"${ncu-path}/bin/git-node.js\" \"$@\"") + (pkgs.writeShellScriptBin "ncu-ci" "exec \"${ncu-path}/bin/ncu-ci.js\" \"$@\"") + (pkgs.writeShellScriptBin "ncu-config" "exec \"${ncu-path}/bin/ncu-config.js\" \"$@\"") + ]), + benchmarkTools ? [ + pkgs.R + pkgs.rPackages.ggplot2 + pkgs.rPackages.plyr + pkgs.wrk + ], + extraConfigFlags ? [ + "--without-npm" + "--debug-node" + ], +}: + +let + useSharedICU = if builtins.isString icu then icu == "system" else icu != null; + useSharedAda = builtins.hasAttr "ada" sharedLibDeps; + useSharedOpenSSL = builtins.hasAttr "openssl" sharedLibDeps; +in +pkgs.mkShell { + inherit (pkgs.nodejs_latest) nativeBuildInputs; + + buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optionals useSharedICU [ icu ]; + + packages = [ + ccache + ] ++ devTools ++ benchmarkTools; + + shellHook = if (ccache != null) then '' + export CC="${pkgs.lib.getExe ccache} $CC" + export CXX="${pkgs.lib.getExe ccache} $CXX" + ls "${pkgs.lib.getLib sharedLibDeps.uvwasi}" + '' else ""; + + BUILD_WITH = if (ninja != null) then "ninja" else "make"; + NINJA = if (ninja != null) then "${pkgs.lib.getExe ninja}" else ""; + CI_SKIP_TESTS = pkgs.lib.concatStringsSep "," ([ + ] ++ pkgs.lib.optionals useSharedAda [ + # Different versions of Ada affect the WPT tests + "test-url" + ] ++ pkgs.lib.optionals useSharedOpenSSL [ + # Path to the openssl.cnf is different from the expected one + "test-strace-openat-openssl" + ]); + CONFIG_FLAGS = builtins.toString ([ + (if icu == null + then "--without-intl" + else "--with-intl=${if useSharedICU then "system" else icu}-icu") + ] ++ extraConfigFlags ++ pkgs.lib.optionals (ninja != null) [ + "--ninja" + ] ++ pkgs.lib.optionals loadJSBuiltinsDynamically [ + "--node-builtin-modules-path=${builtins.toString ./.}" + ] ++ pkgs.lib.concatMap (name: [ + "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}" + "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-libpath=${ + pkgs.lib.getLib sharedLibDeps.${name} + }/lib" + "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-include=${ + pkgs.lib.getInclude sharedLibDeps.${name} + }/include" + ]) (builtins.attrNames sharedLibDeps)); +} diff --git a/test/parallel/test-setproctitle.js b/test/parallel/test-setproctitle.js index b08302e0a35ac0..368bc85800a9a9 100644 --- a/test/parallel/test-setproctitle.js +++ b/test/parallel/test-setproctitle.js @@ -4,7 +4,7 @@ const common = require('../common'); const { isMainThread } = require('worker_threads'); // FIXME add sunos support -if (common.isSunOS || common.isIBMi) { +if (common.isSunOS || common.isIBMi || common.isWindows) { common.skip(`Unsupported platform [${process.platform}]`); } @@ -25,15 +25,10 @@ assert.notStrictEqual(process.title, title); process.title = title; assert.strictEqual(process.title, title); -// Test setting the title but do not try to run `ps` on Windows. -if (common.isWindows) { - common.skip('Windows does not have "ps" utility'); -} - try { execSync('command -v ps'); } catch (err) { - if (err.status === 1) { + if (err.status === 1 || err.status === 127) { common.skip('The "ps" utility is not available'); } throw err; @@ -53,5 +48,5 @@ exec(cmd, common.mustSucceed((stdout, stderr) => { title += ` (${path.basename(process.execPath)})`; // Omitting trailing whitespace and \n - assert.strictEqual(stdout.replace(/\s+$/, '').endsWith(title), true); + assert.ok(stdout.trimEnd().endsWith(title)); })); diff --git a/tools/dep_updaters/update-nixpkgs-pin.sh b/tools/dep_updaters/update-nixpkgs-pin.sh new file mode 100755 index 00000000000000..f6305d020a914c --- /dev/null +++ b/tools/dep_updaters/update-nixpkgs-pin.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -ex +# Shell script to update Nixpkgs pin in the source tree to the most recent +# version on the unstable channel. + +BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd) +WORKFLOW_FILE="$BASE_DIR/.github/workflows/test-shared.yml" + +NIXPKGS_REPO=$(grep NIXPKGS_REPO: "$WORKFLOW_FILE" | awk -F': ' '{ print $2 }') +CURRENT_VERSION_SHA1=$(grep NIXPKGS_PIN: "$WORKFLOW_FILE" | awk -F': ' '{ print $2 }') +CURRENT_VERSION=$(echo "$CURRENT_UPSTREAM_SHA1" | head -c 7) + +NEW_UPSTREAM_SHA1=$(git ls-remote "$NIXPKGS_REPO.git" nixpkgs-unstable | awk '{print $1}') +NEW_VERSION=$(echo "$NEW_UPSTREAM_SHA1" | head -c 7) + + +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + +compare_dependency_version "nixpkgs-unstable" "$NEW_VERSION" "$CURRENT_VERSION" + +TMP_FILE=$(mktemp) +sed "s/$CURRENT_VERSION_SHA1/$NEW_UPSTREAM_SHA1/" "$WORKFLOW_FILE" > "$TMP_FILE" +mv "$TMP_FILE" "$WORKFLOW_FILE" + +echo "All done!" +echo "" +echo "Please git add and commit the new version:" +echo "" +echo "$ git add $WORKFLOW_FILE" +echo "$ git commit -m 'tools: bump nixpkgs-unstable pin to $NEW_VERSION'" +echo "" + +# The last line of the script should always print the new version, +# as we need to add it to $GITHUB_ENV variable. +echo "NEW_VERSION=$NEW_VERSION" diff --git a/tools/gyp/pylib/gyp/xcode_emulation.py b/tools/gyp/pylib/gyp/xcode_emulation.py index 0746865dc84b72..48aa22bf899b8e 100644 --- a/tools/gyp/pylib/gyp/xcode_emulation.py +++ b/tools/gyp/pylib/gyp/xcode_emulation.py @@ -521,7 +521,7 @@ def _GetSdkVersionInfoItem(self, sdk, infoitem): # most sensible route and should still do the right thing. try: return GetStdoutQuiet(["xcrun", "--sdk", sdk, infoitem]) - except GypError: + except (GypError, OSError): pass def _SdkRoot(self, configname): @@ -1350,11 +1350,12 @@ def _DefaultSdkRoot(self): if xcode_version < "0500": return "" default_sdk_path = self._XcodeSdkPath("") - if default_sdk_root := XcodeSettings._sdk_root_cache.get(default_sdk_path): + default_sdk_root = XcodeSettings._sdk_root_cache.get(default_sdk_path) + if default_sdk_root: return default_sdk_root try: all_sdks = GetStdout(["xcodebuild", "-showsdks"]) - except GypError: + except (GypError, OSError): # If xcodebuild fails, there will be no valid SDKs return "" for line in all_sdks.splitlines(): @@ -1508,7 +1509,7 @@ def XcodeVersion(): raise GypError("xcodebuild returned unexpected results") version = version_list[0].split()[-1] # Last word on first line build = version_list[-1].split()[-1] # Last word on last line - except GypError: # Xcode not installed so look for XCode Command Line Tools + except (GypError, OSError): # Xcode not installed so look for XCode Command Line Tools version = CLTVersion() # macOS Catalina returns 11.0.0.0.1.1567737322 if not version: raise GypError("No Xcode or CLT version detected!") @@ -1541,15 +1542,15 @@ def CLTVersion(): try: output = GetStdout(["/usr/sbin/pkgutil", "--pkg-info", key]) return re.search(regex, output).groupdict()["version"] - except GypError: + except (GypError, OSError): continue regex = re.compile(r"Command Line Tools for Xcode\s+(?P\S+)") try: output = GetStdout(["/usr/sbin/softwareupdate", "--history"]) return re.search(regex, output).groupdict()["version"] - except GypError: - return None + except (GypError, OSError): + return "11.0.0.0.1.1567737322" def GetStdoutQuiet(cmdlist): @@ -1786,9 +1787,11 @@ def _GetXcodeEnv( env["INFOPLIST_PATH"] = xcode_settings.GetBundlePlistPath() env["WRAPPER_NAME"] = xcode_settings.GetWrapperName() - if install_name := xcode_settings.GetInstallName(): + install_name = xcode_settings.GetInstallName() + if install_name: env["LD_DYLIB_INSTALL_NAME"] = install_name - if install_name_base := xcode_settings.GetInstallNameBase(): + install_name_base = xcode_settings.GetInstallNameBase() + if install_name_base: env["DYLIB_INSTALL_NAME_BASE"] = install_name_base xcode_version, _ = XcodeVersion() if xcode_version >= "0500" and not env.get("SDKROOT"):