[DDW-596] Try to fix the Linux build

Author: michalrus

.buildkite/pipeline.yml

@@ -1,21 +1,21 @@
 env:
   ARTIFACT_BUCKET: s3://ci-output-sink
 steps:
-  - label: 'daedalus-x86_64-darwin'
-    command: 'scripts/build-installer-unix.sh --build-id $BUILDKITE_BUILD_NUMBER'
-    env:
-      NIX_SSL_CERT_FILE: /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
-    agents:
-      queue: daedalus
-      system: x86_64-darwin
+  # - label: 'daedalus-x86_64-darwin'
+  #   command: 'scripts/build-installer-unix.sh --build-id $BUILDKITE_BUILD_NUMBER'
+  #   env:
+  #     NIX_SSL_CERT_FILE: /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
+  #   agents:
+  #     queue: daedalus
+  #     system: x86_64-darwin
   - label: 'daedalus-x86_64-linux-nix'
     command: 'scripts/build-installer-nix.sh $BUILDKITE_BUILD_NUMBER'
     agents:
       system: x86_64-linux
-  - label: 'daedalus-x86_64-windows-nix'
-    command: 'scripts/build-cross-windows.sh $BUILDKITE_BUILD_NUMBER'
-    agents:
-      system: x86_64-linux
+  # - label: 'daedalus-x86_64-windows-nix'
+  #   command: 'scripts/build-cross-windows.sh $BUILDKITE_BUILD_NUMBER'
+  #   agents:
+  #     system: x86_64-linux
 
   # TODO: Re-enable once script is fixed
   # - label: 'release.nix'

.gitignore

@@ -31,6 +31,7 @@ build/Release
 # Dependency directory
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
+/node_modules.*
 
 # OSX
 .DS_Store

package.json

@@ -302,7 +302,6 @@
   },
   "resolutions": {
     "@types/react": "16.9.56",
-    "**/**/minimatch": "3.0.4",
     "**/**/handlebars": "4.7.7",
     "**/**/elliptic": "6.5.4",
     "**/**/bl": "4.1.0",

scripts/minimize-runtime-dependencies--linux.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+#
+# This script *can* help in minimizing runtime dependencies of
+# Daedalus… but only the ones which prevent Electron from starting.
+#
+# That’s a good starting point before we get this automatically from
+# webpack – <https://input-output.atlassian.net/browse/DDW-1103>.
+#
+
+set -o errexit
+set -o pipefail
+
+if [ -z "$IN_NIX_SHELL" ] ; then
+  echo >&2 'fatal: please, enter nix-shell first'
+  exit 1
+fi
+
+dir=$(git rev-parse --show-toplevel)
+srcDir="$dir"/node_modules.pre-minimize-"$(date -Ins)"
+dstDir="$dir"/node_modules
+
+yarn build
+mv "$dstDir" "$srcDir"
+mkdir "$dstDir"
+
+while true ; do
+
+  the_log=$(mktemp)
+  export the_log
+
+  bash -c '
+    trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+    electron ./ 2>&1 | tee "$the_log"
+  ' &
+  child_pid=$!
+
+  sleep 2
+  kill TERM $child_pid
+
+  module=$(head "$the_log" -n 2 | tail -n 1 | grep -F 'Error: Cannot find module')
+  rm "$the_log"
+
+  if [ -z "$module" ]; then
+    echo 'done?'
+    exit 77
+  else
+    module=$(echo "$module" | sed -r "s/^.*'(.*)'/\1/")
+
+    echo
+    echo
+    echo "Missing module is: ‘$module’."
+
+    module=$(echo "$module" | cut -d/ -f1)
+
+    echo "Press <RET> to copy ‘$srcDir/$module’ → ‘$dstDir/’, and continue…"
+    read -r
+
+    cp -r "$srcDir/$module" "$dstDir/"
+  fi
+
+done

yarn.lock

@@ -2986,7 +2986,11 @@
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
 
-"@types/minimatch@*", "@types/minimatch@^3.0.3":
+"@types/minimatch@*":
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40"
+
+"@types/minimatch@^3.0.3":
   version "3.0.5"
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40"
 
@@ -11690,9 +11694,27 @@ minimalistic-crypto-utils@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a"
 
-[email protected], minimatch@^3.0.2, minimatch@^3.0.4, minimatch@^3.1.2:
-  version "3.0.4"
-  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
+[email protected]:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
+  dependencies:
+    brace-expansion "^1.1.7"
+
+minimatch@^3.0.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
+  dependencies:
+    brace-expansion "^1.1.7"
+
+minimatch@^3.0.4:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
+  dependencies:
+    brace-expansion "^1.1.7"
+
+minimatch@^3.1.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
   dependencies:
     brace-expansion "^1.1.7"
 

yarn2nix.nix

@@ -18,8 +18,10 @@
 let
   cluster' = launcherConfig.networkName;
   yarn2nix = import (fetchzip {
-    url = "https://github.com/moretea/yarn2nix/archive/v1.0.0.tar.gz";
-    sha256 = "02bzr9j83i1064r1r34cn74z7ccb84qb5iaivwdplaykyyydl1k8";
+    # v1.0.0 with a PR to handle duplicate file names between @types/* and original/* – <https://github.com/nix-community/yarn2nix/pull/75>
+    # TODO: use the version from recent Nixpkgs
+    url = "https://github.com/nix-community/yarn2nix/archive/276994748d556e0812bb1bc5f92ac095b5da71d2.tar.gz";
+    sha256 = "1fxiq43w8mfs0aiyj4kazwjl6b829a5r0jbx6bcs3kmil9asq3fg";
   }) {
     inherit pkgs nodejs yarn;
   };
@@ -140,38 +142,26 @@ yarn2nix.mkYarnPackage {
     #export DEBUG=electron-rebuild
 
     ls -ltrha $NIX_BUILD_TOP/daedalus/node_modules/
-    function dup() {
-      cp -vr node_modules/''${1}/ node_modules/''${1}-temp
-      rm -v node_modules/''${1}
-      mv -v node_modules/''${1}-temp node_modules/''${1}
-      chmod -R +w node_modules/''${1}
-    }
 
-    dup keccak
-    dup node-hid
-    dup usb
-    dup @ledgerhq
-    dup electron-chromedriver
-    dup blake-hash
-    dup blake2
-    dup tiny-secp256k1
-    dup usb-detection
+    chmod -R +w node_modules/
 
     # We ship debug version because the release one has issues with ledger nano s
     node_modules/.bin/electron-rebuild -w usb --useCache -s --debug
 
     mkdir -p $out/bin $out/share/daedalus
-    cp -R dist/* $out/share/daedalus
+    cp -R deps/daedalus/dist/* $out/share/daedalus
     cp ${newPackagePath} $out/share/daedalus/package.json
     pushd $out/share/daedalus
     ${nukeAllRefs}
     popd
     mkdir -p $out/share/fonts
     ln -sv $out/share/daedalus/renderer/assets $out/share/fonts/daedalus
     mkdir -pv $out/share/daedalus/node_modules
-    cp -rv $node_modules/{\@babel,\@protobufjs,regenerator-runtime,node-fetch,\@trezor,runtypes,parse-uri,randombytes,safe-buffer,bip66,pushdata-bitcoin,bitcoin-ops,typeforce,varuint-bitcoin,create-hash,blake2b,nanoassert,blake2b-wasm,bs58check,bs58,base-x,create-hmac,wif,ms,keccak,semver-compare,long,define-properties,object-keys,has,function-bind,es-abstract,has-symbols,json-stable-stringify,tiny-worker,cashaddrjs,big-integer,inherits,bchaddrjs,cross-fetch,trezor-connect,js-chain-libs-node,bignumber.js,call-bind,get-intrinsic,base64-js,ieee754,cbor-web,util-deprecate,bech32,blake-hash,blake2,tiny-secp256k1,bn.js,elliptic,minimalistic-assert,minimalistic-crypto-utils,brorand,hash.js,hmac-drbg,int64-buffer,object.values,bytebuffer,protobufjs,usb-detection} $out/share/daedalus/node_modules/
+    cp -rv $node_modules/{\@babel,\@protobufjs,regenerator-runtime,node-fetch,\@trezor,runtypes,parse-uri,randombytes,safe-buffer,bip66,pushdata-bitcoin,bitcoin-ops,typeforce,varuint-bitcoin,create-hash,blake2b,nanoassert,blake2b-wasm,bs58check,bs58,base-x,create-hmac,wif,ms,keccak,semver-compare,long,define-properties,object-keys,has,function-bind,es-abstract,has-symbols,json-stable-stringify,tiny-worker,cashaddrjs,big-integer,inherits,bchaddrjs,cross-fetch,trezor-connect,js-chain-libs-node,bignumber.js,call-bind,get-intrinsic,base64-js,ieee754,cbor-web,util-deprecate,bech32,blake-hash,blake2,tiny-secp256k1,bn.js,elliptic,minimalistic-assert,minimalistic-crypto-utils,brorand,hash.js,hmac-drbg,int64-buffer,object.values,bytebuffer,protobufjs,usb-detection,babel-runtime,bindings,brotli,buffer,clone,deep-equal,dfa,eventemitter2,file-uri-to-path,fontkit,functions-have-names,has-property-descriptors,has-tostringtag,is-arguments,is-date-object,is-regex,linebreak,node-hid,object-is,pdfkit,png-js,regexp.prototype.flags,restructure,tiny-inflate,unicode-properties,unicode-trie} $out/share/daedalus/node_modules/
     find $out $NIX_BUILD_TOP -name '*.node'
 
+    chmod -R +w $out
+
     mkdir -pv $out/share/daedalus/build
     cp node_modules/usb/build/Debug/usb_bindings.node $out/share/daedalus/build/usb_bindings.node
     cp node_modules/node-hid/build/Debug/HID_hidraw.node $out/share/daedalus/build/HID_hidraw.node
@@ -181,12 +171,16 @@ yarn2nix.mkYarnPackage {
     done
 
     node_modules/.bin/electron-rebuild -w usb-detection --useCache -s
-    cp node_modules/usb-detection/build/Release/detection.node $out/share/daedalus/build/detection.node
-    for file in $out/share/daedalus/build/detection.node; do
+    mkdir -p $out/share/daedalus/node_modules/usb-detection/build/
+    cp node_modules/usb-detection/build/Release/detection.node $out/share/daedalus/node_modules/usb-detection/build/detection.node
+    for file in $out/share/daedalus/node_modules/usb-detection/build/detection.node; do
       $STRIP $file
       patchelf --shrink-rpath $file
     done
   '';
+  distPhase = ''
+    # unused
+  '';
   #allowedReferences = [ "out" ];
   #allowedRequisites = [
   #  systemd.lib