Merge remote-tracking branch 'origin/main' into 1.31-releases

Author: addaleax

.evergreen.yml

@@ -440,7 +440,7 @@ tasks:
           compass_distribution: compass
       - func: package
         vars:
-          debug: 'hadron*,mongo*,compass*'
+          debug: 'hadron*,mongo*,compass*,electron*'
           compass_distribution: compass
       - func: save-windows-artifacts
         vars:
@@ -472,7 +472,7 @@ tasks:
           compass_distribution: compass-isolated
       - func: package
         vars:
-          debug: 'hadron*,mongo*,compass*'
+          debug: 'hadron*,mongo*,compass*,electron*'
           compass_distribution: compass-isolated
       - func: save-windows-artifacts
         vars:
@@ -504,7 +504,7 @@ tasks:
           compass_distribution: compass-readonly
       - func: package
         vars:
-          debug: 'hadron*,mongo*,compass*'
+          debug: 'hadron*,mongo*,compass*,electron*'
           compass_distribution: compass-readonly
       - func: save-windows-artifacts
         vars:

.evergreen/node-gyp-bug-workaround.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# This is a workaround for a node-gyp bug that has not fully been investigated
+# due to problems reproducing it outside of CI environments (even though it
+# occurs both in evergreen and github actions).
+# Something seems to go wrong when node-gyp extracts the Node.js header tarball,
+# on Windows specifically (this is most likely because node-tar treats
+# the overwriting of existing files differently on Windows than on other OS --
+# for good reasons, but still).
+# The most likely cause of this issue is that node-gyp somehow extracts the
+# same headers tarball twice, in parallel, in the same location, with race
+# conditions in the tar extraction code leading to issues.
+# The extraction result ends up in %LOCALAPPDATA%\node-gyp\Cache.
+# Manually extracting the tarballs will solve this issue, so we're doing that
+# here.
+# For actually resolving the bug, we would probably need somebody with a local
+# reproduction. However, it seems likely that other people will also encounter
+# this issue, so there's also a good chance that this workaround will just
+# not be needed with a future node-gyp version.
+
+if [ x"$NODE_JS_VERSION" = x"" ]; then
+  if node -v; then
+    export NODE_JS_VERSION=$(node -p 'process.version.slice(1)')
+  else
+    echo "Need NODE_JS_VERSION to be set or Node.js to be installed for node-gyp bug workaround script"
+    exit 1
+  fi
+fi
+
+if [ x"$LOCALAPPDATA" = x"" ]; then
+  echo "No LOCALAPPDATA set, ignoring node-gyp bug workaround script"
+  exit
+fi
+
+set -x
+CACHEDIR="$LOCALAPPDATA/node-gyp/Cache"
+rm -rvf "$CACHEDIR"
+mkdir -p "$CACHEDIR/$NODE_JS_VERSION"
+cd "$CACHEDIR/$NODE_JS_VERSION"
+curl -sSfLO "https://nodejs.org/download/release/v$NODE_JS_VERSION/node-v$NODE_JS_VERSION-headers.tar.gz"
+tar --strip-components=1 -xvzf "node-v$NODE_JS_VERSION-headers.tar.gz"
+for arch in x64 x86 arm64; do
+  mkdir $arch
+  pushd $arch
+  curl -sSfLO "https://nodejs.org/download/release/v$NODE_JS_VERSION/win-$arch/node.lib" || echo "no $arch v$NODE_JS_VERSION .lib file"
+  popd
+done
+
+# Finally, store the right installVersion value for current node-gyp versions
+echo 9 > installVersion

.evergreen/preinstall.sh

@@ -1,21 +1,5 @@
 #!/usr/bin/env bash
 
-if [[ $OSTYPE == "cygwin" ]]; then
-    export PLATFORM='win32'
-    export IS_WINDOWS=true
-elif [[ `uname` == Darwin ]]; then
-    export PLATFORM='darwin'
-    export IS_OSX=true
-else
-    export PLATFORM='linux'
-    export IS_LINUX=true
-    if [[ `cat /etc/*release | grep ^NAME | grep Red` ]]; then
-        export IS_RHEL=true
-    elif [[ `cat /etc/*release | grep ^NAME | grep Ubuntu` ]]; then
-        export IS_UBUNTU=true
-    fi
-fi
-
 echo "========================="
 echo "Important Environment Variables"
 echo "========================="
@@ -51,6 +35,9 @@ if [ -n "$IS_WINDOWS" ]; then
     ./node.exe node_modules/npm2/bin/npm-cli.js i -g npm@$NPM_VERSION
     rm -rf node_modules/npm2/
     chmod +x npm.cmd npm
+
+    cd ..
+    .evergreen/node-gyp-bug-workaround.sh
 else
     echo "Installing nodejs v${NODE_JS_VERSION} for ${PLATFORM}..."
     curl -fs \
@@ -69,4 +56,4 @@ else
 
     ./bin/node lib/node_modules/npm2/bin/npm-cli.js i -g npm@$NPM_VERSION
     rm -rf lib/node_modules/npm2/
-fi
+fi

.evergreen/print-compass-env.js

@@ -19,10 +19,6 @@ function printVar(name, value) {
 }
 
 function printCompassEnv() {
-  const {
-    EVERGREEN_BUILD_VARIANT
-  } = process.env;
-
   let {
     // This is an env var set in bash that we exported in print-compass-env.sh
     OSTYPE
@@ -48,34 +44,41 @@ function printCompassEnv() {
   # - https://github.com/mongodb-js/compass/pull/2403
   # - https://github.com/mongodb-js/compass/pull/2410
   */
-  if (originalPWD.startsWith('\/cygdrive\/c')) {
+  if (originalPWD.startsWith('/cygdrive/c')) {
     // Change cygdrive from c to z without chanding rest of the path
-    newPWD = originalPWD.replace('\/cygdrive\/c', '\/cygdrive\/z');
+    newPWD = originalPWD.replace('/cygdrive/c', '/cygdrive/z');
     // we have to change the directory in the shell script we're outputting, not in this node process
     console.log(`cd "${newPWD}";`);
     console.log('echo "Changed cwd on cygwin. Current working dir: $PWD";');
   }
 
-  const pathsToPrepend = []
+  const pathsToPrepend = [];
 
   if (OSTYPE === 'cygwin') {
     // NOTE lucas: for git-core addition, See
     // https://jira.mongodb.org/browse/COMPASS-4122
     pathsToPrepend.unshift('/cygdrive/c/wixtools/bin');
     pathsToPrepend.unshift(`${newPWD}/.deps`);
-    pathsToPrepend.unshift('/cygdrive/c/Program Files/Git/mingw32/libexec/git-core');
-    printVar('APPDATA', 'Z:\\\;');
+    pathsToPrepend.unshift(
+      '/cygdrive/c/Program Files/Git/mingw32/libexec/git-core'
+    );
+    printVar('APPDATA', 'Z:\\;');
   } else {
     pathsToPrepend.unshift(`${newPWD}/.deps/bin`);
   }
 
-  if (EVERGREEN_BUILD_VARIANT === 'rhel') {
+  if (process.env.IS_RHEL === 'true') {
     // To build node modules on RHEL post electron 13 we need
     // a newer c++ compiler version, this adds it.
     // https://jira.mongodb.org/browse/COMPASS-5150
     pathsToPrepend.unshift('/opt/mongodbtoolchain/v3/bin');
   }
 
+  if (process.env.PLATFORM === 'linux') {
+    // Make sure that linux is using python 3.6 (node-gyp requirement)
+    pathsToPrepend.unshift('/opt/python/3.6/bin');
+  }
+
   PATH = maybePrependPaths(PATH, pathsToPrepend);
   printVar('PATH', PATH);
 
@@ -91,6 +94,13 @@ function printCompassEnv() {
   printVar('npm_config_cache', npmCacheDir);
   // npm tmp is deprecated, but let's keep it around just in case
   printVar('npm_config_tmp', npmTmpDir);
+
+  printVar('PLATFORM', process.env.PLATFORM);
+  printVar('IS_WINDOWS', process.env.IS_WINDOWS);
+  printVar('IS_OSX', process.env.IS_OSX);
+  printVar('IS_LINUX', process.env.IS_LINUX);
+  printVar('IS_RHEL', process.env.IS_RHEL);
+  printVar('IS_UBUNTU', process.env.IS_UBUNTU);
 }
 
 printCompassEnv();

.evergreen/print-compass-env.sh

@@ -1,6 +1,22 @@
 #! /usr/bin/env bash
 
+if [[ $OSTYPE == "cygwin" ]]; then
+    export PLATFORM='win32'
+    export IS_WINDOWS=true
+elif [[ $(uname) == Darwin ]]; then
+    export PLATFORM='darwin'
+    export IS_OSX=true
+else
+    export PLATFORM='linux'
+    export IS_LINUX=true
+    if [[ $(cat /etc/*release | grep ^NAME | grep Red) ]]; then
+        export IS_RHEL=true
+    elif [[ $(cat /etc/*release | grep ^NAME | grep Ubuntu) ]]; then
+        export IS_UBUNTU=true
+    fi
+fi
+
 export BASHPATH="$PATH"
 export OSTYPE="$OSTYPE"
 
-.evergreen/print-compass-env.js
+.evergreen/print-compass-env.js

.github/workflows/authors-and-third-party-notices.yaml

@@ -25,7 +25,10 @@ jobs:
       - name: Install [email protected]
         run: |
           npm install -g [email protected]
-          npx --yes rimraf $LOCALAPPDATA/node-gyp/Cache || true
+
+      - name: Run node-gyp bug workaround script
+        run: |
+          bash .evergreen/node-gyp-bug-workaround.sh
 
       - name: Install Dependencies
         run: |

.github/workflows/build.yaml

@@ -52,7 +52,10 @@ jobs:
       - name: Install [email protected]
         run: |
           npm install -g [email protected]
-          npx --yes rimraf $LOCALAPPDATA/node-gyp/Cache || true
+
+      - name: Run node-gyp bug workaround script
+        run: |
+          bash .evergreen/node-gyp-bug-workaround.sh
 
       - name: Install Dependencies
         run: |

.github/workflows/check-test.yaml

@@ -49,7 +49,10 @@ jobs:
       - name: Install [email protected]
         run: |
           npm install -g [email protected]
-          npx --yes rimraf $LOCALAPPDATA/node-gyp/Cache || true
+
+      - name: Run node-gyp bug workaround script
+        run: |
+          bash .evergreen/node-gyp-bug-workaround.sh
 
       - name: Install Dependencies
         run: |

.github/workflows/connectivity-tests.yaml

@@ -79,7 +79,10 @@ jobs:
       - name: Install [email protected]
         run: |
           npm install -g [email protected]
-          npx --yes rimraf $LOCALAPPDATA/node-gyp/Cache || true
+
+      - name: Run node-gyp bug workaround script
+        run: |
+          bash .evergreen/node-gyp-bug-workaround.sh
 
       - name: Install Dependencies
         run: |