Merge pull request #2998 from input-output-hk/chore/line-up-nodejs-to-electrons-14.16.0

[DDW-1109] Allow specifying Node.js version; patch `electron-rebuild` forcing proper Electron-Node.js headers; fix M1 CI

Author: danielmain

default.nix

@@ -72,7 +72,20 @@ let
     cardanoLib = localLib.iohkNix.cardanoLib;
     daedalus-bridge = self.bridgeTable.${nodeImplementation};
 
-    nodejs = pkgs.nodejs-16_x;
+    nodejs = let
+      njPath = pkgs.path + "/pkgs/development/web/nodejs";
+      buildNodeJs = pkgs.callPackage (import (njPath + "/nodejs.nix")) {
+        python = pkgs.python3;
+        icu = pkgs.icu68; # can’t build against ICU 69: <https://chromium-review.googlesource.com/c/v8/v8/+/2477751>
+      };
+    in
+      buildNodeJs {
+        enableNpm = true;
+        version = "14.17.0";
+        sha256 = "1vf989canwcx0wdpngvkbz2x232yccp7fzs1vcbr60rijgzmpq2n";
+        patches = pkgs.lib.optional pkgs.stdenv.isDarwin (njPath + "/bypass-xcodebuild.diff");
+      };
+
     nodePackages = pkgs.nodePackages.override { nodejs = self.nodejs; };
     yarnInfo = {
       version = "1.22.4";

installers/nix/electron.nix

@@ -1,8 +1,8 @@
 { stdenv, lib, makeWrapper, fetchurl, unzip, atomEnv, libuuid, at-spi2-atk, at_spi2_core, libxshmfence,
-  libxkbcommon }:
+  libxkbcommon, runCommand, binutils-unwrapped }:
 
 let
-  version = "13.6.3";
+  version = (builtins.fromJSON (builtins.readFile ../../package.json)).dependencies.electron;
   name = "electron-${version}";
 
   throwSystem = throw "Unsupported system: ${stdenv.hostPlatform.system}";

nix/yarn-nix-shell.sh

@@ -29,5 +29,9 @@ if [ -z "$command" ] ; then
 fi
 
 export NETWORK
+
+# Prevent segfaults on Darwin in `GC_*` code:
+export GC_DONT_GC=1
+
 # `return` will make the user stay in `nix-shell` after the initial command finishes:
 exec nix-shell --argstr nodeImplementation cardano --argstr cluster "$cluster" --command "$command ; return"

package.json

@@ -13,7 +13,7 @@
     "build:main": "yarn webpack -c source/main/webpack.config.js --progress",
     "build:renderer": "yarn webpack -c source/renderer/webpack.config.js --progress",
     "build:cleanup": "rimraf ./dist",
-    "build:electron": "electron-rebuild -w usb --useCache -s --debug",
+    "build:electron": "electron-rebuild --useCache && electron-rebuild -w usb --useCache -s --debug",
     "check:all": "yarn prettier:check && yarn lint && yarn compile && yarn stylelint && yarn lockfile:check && yarn i18n:manage && yarn storybook:build",
     "start": "yarn electron ./",
     "start:dev": "nodemon --watch 'dist/main' --exec 'NODE_ENV=development yarn start'",

release.nix

@@ -69,8 +69,11 @@ in {
   in {
   daedalus-installer = allArchesNoWindows;
   yaml2json = allArchesNoWindows;
+  nodejs = allArchesNoWindows;
   bridgeTable = {
     cardano = allArches;
   };
   cardano-node = allArches;
-}))
+})) // {
+  recurseForDerivations = {};
+}

scripts/postinstall.sh

@@ -2,4 +2,9 @@
 
 if [[ "$CI" != "true" ]]; then
   yarn lockfile:fix
+
+  # Let’s patch electron-rebuild to force correct Node.js headers to
+  # build native modules against even in `nix-shell`, otherwise, it
+  # doesn’t work reliably.
+  eval "$(nix-build -A rawapp.patchElectronRebuild)"
 fi

shell.nix

@@ -111,16 +111,6 @@ let
       ln -svf $(type -P cardano-node)
       ln -svf $(type -P cardano-wallet)
       ln -svf $(type -P cardano-cli)
-      mkdir -p ${BUILDTYPE}/
-      ${let
-        # (TODO: investigate why – @michalrus)
-        sourceBUILDTYPE = "Release";
-      in ''
-        ln -svf $PWD/node_modules/usb/build/${sourceBUILDTYPE}/usb_bindings.node ${BUILDTYPE}/
-        ln -svf $PWD/node_modules/node-hid/build/${sourceBUILDTYPE}/HID.node ${BUILDTYPE}/
-        ln -svf $PWD/node_modules/node-hid/build/${sourceBUILDTYPE}/HID_hidraw.node ${BUILDTYPE}/
-        ln -svf $PWD/node_modules/usb-detection/build/${sourceBUILDTYPE}/detection.node ${BUILDTYPE}/
-      ''}
 
       ${pkgs.lib.optionalString (nodeImplementation == "cardano") ''
         source <(cardano-node --bash-completion-script `type -p cardano-node`)
@@ -135,14 +125,51 @@ let
         ''
       }
       yarn install --frozen-lockfile
+
+      # Rebuild native modules for <https://www.electronjs.org/docs/latest/tutorial/using-native-node-modules>:
+      find Debug/ Release/ -name '*.node' | xargs rm -v || true
       yarn build:electron
+
+      ${let
+        # Several native modules have to be linked in ${BUILDTYPE}/ in
+        # root directory, for `yarn dev` to work correctly. If a Debug
+        # version of such extension exists, we use it, otherwise, we
+        # use Release:
+        tryLink = dependency: fileName: ''
+          symlinkTarget=$(ls 2>/dev/null -d \
+            "$PWD/node_modules/${dependency}/build/Debug/${fileName}" \
+            "$PWD/node_modules/${dependency}/build/Release/${fileName}" \
+            | head -1
+          )
+
+          if [ -z "$symlinkTarget" ] ; then
+            echo >&2 "error: symlink target not found: ‘${fileName}’ in ‘${dependency}’"
+            # ~exit 1~ — do not exit, let the person fix from inside `nix-shell`
+          fi
+
+          ${localLib.optionalString pkgs.stdenv.isLinux ''
+            ${pkgs.patchelf}/bin/patchelf --set-rpath ${pkgs.lib.makeLibraryPath [
+              pkgs.stdenv.cc.cc pkgs.udev
+            ]} "$symlinkTarget"
+          ''}
+
+          mkdir -p ${BUILDTYPE}/
+          ln -svf "$symlinkTarget" ${BUILDTYPE}/
+          unset symlinkTarget
+        '';
+      in ''
+        ${tryLink "usb"           "usb_bindings.node"}
+        ${tryLink "usb-detection" "detection.node"}
+        ${tryLink "node-hid"      "HID.node"}
+        ${localLib.optionalString pkgs.stdenv.isLinux ''
+          ${tryLink "node-hid"      "HID_hidraw.node"}
+        ''}
+      ''}
+
       ${localLib.optionalString pkgs.stdenv.isLinux ''
-        ${pkgs.patchelf}/bin/patchelf --set-rpath ${pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc pkgs.udev ]} ${BUILDTYPE}/usb_bindings.node
-        ${pkgs.patchelf}/bin/patchelf --set-rpath ${pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc pkgs.udev ]} ${BUILDTYPE}/HID.node
-        # TODO: is this needed for `detection.node`?
-        ${pkgs.patchelf}/bin/patchelf --set-rpath ${pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc pkgs.udev ]} ${BUILDTYPE}/detection.node
         ln -svf ${daedalusPkgs.electron}/bin/electron ./node_modules/electron/dist/electron
       ''}
+
       echo 'jq < $LAUNCHER_CONFIG'
       echo debug the node by running debug-node
     '';

yarn2nix.nix

@@ -10,7 +10,7 @@
 , lz4
 , pkgconfig
 , systemd
-, writeShellScriptBin
+, writeShellScript
 , xz
 , nodePackages
 , zlib
@@ -35,24 +35,29 @@ let
     main = "main/index.js";
   };
   newPackagePath = builtins.toFile "package.json" (builtins.toJSON newPackage);
-  windowsElectronVersion = "13.6.3";
-  electronPath = "https://github.com/electron/electron/releases/download/v${windowsElectronVersion}";
+  electronVersion = origPackage.dependencies.electron;
+  electronPath = "https://github.com/electron/electron/releases/download/v${electronVersion}";
   windowsElectron = fetchurl {
-    url = "${electronPath}/electron-v${windowsElectronVersion}-win32-x64.zip";
+    url = "${electronPath}/electron-v${electronVersion}-win32-x64.zip";
     sha256 = "18085a2509447fef8896daeee96a12f48f8e60a4d5ec4cfab44d8d59b9d89a72";
   };
   electronPathHash = builtins.hashString "sha256" electronPath;
   electron-cache = runCommand "electron-cache" {} ''
     # newer style
     mkdir -p $out/${electronPathHash}/
-    ln -sv ${windowsElectron} $out/${electronPathHash}/electron-v${windowsElectronVersion}-win32-x64.zip
-    mkdir $out/httpsgithub.comelectronelectronreleasesdownloadv${windowsElectronVersion}electron-v${windowsElectronVersion}-win32-x64.zip
-    ln -s ${windowsElectron} $out/httpsgithub.comelectronelectronreleasesdownloadv${windowsElectronVersion}electron-v${windowsElectronVersion}-win32-x64.zip/electron-v${windowsElectronVersion}-win32-x64.zip
+    ln -sv ${windowsElectron} $out/${electronPathHash}/electron-v${electronVersion}-win32-x64.zip
+    mkdir $out/httpsgithub.comelectronelectronreleasesdownloadv${electronVersion}electron-v${electronVersion}-win32-x64.zip
+    ln -s ${windowsElectron} $out/httpsgithub.comelectronelectronreleasesdownloadv${electronVersion}electron-v${electronVersion}-win32-x64.zip/electron-v${electronVersion}-win32-x64.zip
   '';
-  electron-gyp = fetchurl {
-    url = "https://www.electronjs.org/headers/v${windowsElectronVersion}/node-v${windowsElectronVersion}-headers.tar.gz";
+  electron-node-headers = fetchurl {
+    url = "https://www.electronjs.org/headers/v${electronVersion}/node-v${electronVersion}-headers.tar.gz";
     sha256 = "f8567511857ab62659505ba5158b6ad69afceb512105a3251d180fe47f44366c";
   };
+  electron-node-headers-unpacked = runCommand "electron-node-headers-${electronVersion}-unpacked" {} ''
+    tar -xf ${electron-node-headers}
+    mkdir -p $out
+    mv node_headers/* $out/
+  '';
   filter = name: type: let
     baseName = baseNameOf (toString name);
     sansPrefix = lib.removePrefix (toString ./.) name;
@@ -74,6 +79,38 @@ let
     pkgconfig
     libusb
   ];
+
+  # We patch `node_modules/electron-rebuild` to force specific Node.js
+  # headers to be used when building native extensions for
+  # Electron. Electron’s Node.js ABI differs from the same version of
+  # Node.js, because different libraries are used in Electon,
+  # e.g. BoringSSL instead of OpenSSL,
+  # cf. <https://www.electronjs.org/docs/latest/tutorial/using-native-node-modules>
+  #
+  # We also use this same code in `shell.nix`, since for some reason
+  # `electron-rebuild` there determines incorrect headers to use
+  # automatically, and we keep getting ABI errors. TODO: investigate
+  # why…
+  #
+  # TODO: That `sed` is rather awful… Can it be done better? – @michalrus
+  patchElectronRebuild = writeShellScript "patch-electron-rebuild" ''
+    echo 'Patching electron-rebuild to force our Node.js headers…'
+
+    nodeGypJs=lib/src/module-type/node-gyp.js
+    if [ ! -e $nodeGypJs ] ; then
+      # makes it work both here, and in shell.nix:
+      nodeGypJs="node_modules/electron-rebuild/$nodeGypJs"
+    fi
+    if [ ! -e $nodeGypJs ] ; then
+      echo >&2 'fatal: shouldn’t happen unless electron-rebuild changes'
+      exit 1
+    fi
+
+    # Patch idempotently (matters in repetitive shell.nix):
+    if ! grep -qF ${electron-node-headers} $nodeGypJs ; then
+      sed -r 's|const extraNodeGypArgs.*|\0 extraNodeGypArgs.push("--tarball", "${electron-node-headers}", "--nodedir", "${electron-node-headers-unpacked}");|' -i $nodeGypJs
+    fi
+  '';
 in
 yarn2nix.mkYarnPackage {
   name = "daedalus-js";
@@ -141,12 +178,6 @@ yarn2nix.mkYarnPackage {
     export HOME=$(realpath home)
     yarn --offline run build
 
-    mkdir -pv $HOME/.electron-gyp/
-    tar -xvf ${electron-gyp} -C $HOME/.electron-gyp
-    mv -vi $HOME/.electron-gyp/node_headers $HOME/.electron-gyp/${windowsElectronVersion}/
-
-    ln -sv $HOME/.electron-gyp $HOME/.node-gyp
-
     #export DEBUG=electron-rebuild
 
     ls -ltrha $NIX_BUILD_TOP/daedalus/node_modules/
@@ -206,25 +237,24 @@ yarn2nix.mkYarnPackage {
   #  libunistring
   #  libusb1
   #] ++ stdenv.cc.libc.buildInputs;
+
+  # `yarnPreBuild` is only used in `yarn2nix.mkYarnModules`, not `yarn2nix.mkYarnPackage`:
   yarnPreBuild = ''
     mkdir -p $HOME/.node-gyp/${nodejs.version}
     echo 9 > $HOME/.node-gyp/${nodejs.version}/installVersion
     ln -sfv ${nodejs}/include $HOME/.node-gyp/${nodejs.version}
   '';
+
+  inherit patchElectronRebuild; # for use in shell.nix
+
   pkgConfig = {
     electron-rebuild = {
-      # TODO: That is rather awful… Can it be done better? – @michalrus
       postInstall = ''
-        nodeGypJs=lib/src/module-type/node-gyp.js
-        if [ ! -e $nodeGypJs ] ; then
-          echo >&2 'shouldn’t happen unless electron-rebuild changes'
-          exit 1
-        fi
-
-        sed -r 's|const extraNodeGypArgs.*|\0 extraNodeGypArgs.push("--tarball", "${electron-gyp}", "--nodedir", process.env["HOME"] + "/.node-gyp/${windowsElectronVersion}");|' -i $nodeGypJs
+        ${patchElectronRebuild}
       '';
     };
   };
+
   # work around some purity problems in nix
   yarnLock = ./yarn.lock;
   packageJSON = ./package.json;