Merge branch 'main' into ni/node22

Author: nirinchev

.evergreen.yml

@@ -26,6 +26,7 @@ if [ `uname` = Darwin ]; then
   # match what Node.js 20 does on their own builder machines
   export CFLAGS='-mmacosx-version-min=10.15'
   export CXXFLAGS='-mmacosx-version-min=10.15'
+  export MACOSX_DEPLOYMENT_TARGET=10.15
 fi
 
 # The CI machines we have for Windows and x64 macOS are not

.evergreen/compile-artifact.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -euo pipefail
+
+DOCKER_HUB_URL="https://index.docker.io/v1/"
+
+STDIN=$(cat)
+
+ACTION="$1"
+
+case "$ACTION" in
+  get)
+    SERVER_URL="$STDIN"
+
+    if [[ "$SERVER_URL" == "$DOCKER_HUB_URL" ]]; then
+      if [[ -z "${DOCKERHUB_USERNAME:-}" || -z "${DOCKERHUB_PASSWORD:-}" ]]; then
+        echo "Error: DOCKERHUB_USERNAME or DOCKERHUB_PASSWORD environment variables are not set." >&2
+        exit 1
+      fi
+
+      echo "{\"Username\": \"$DOCKERHUB_USERNAME\", \"Secret\": \"$DOCKERHUB_PASSWORD\"}"
+    else
+      echo "Error: No credentials available for $SERVER_URL" >&2
+      exit 1
+    fi
+    ;;
+
+  *)
+    echo "Unsupported action: $ACTION" >&2
+    exit 1
+    ;;
+esac

.evergreen/docker-config/bin/docker-credential-from-env

@@ -0,0 +1,6 @@
+{
+	"auths": {
+		"https://index.docker.io/v1/": {}
+	},
+	"credsStore": "from-env"
+} 

.evergreen/docker-config/config.json

@@ -32,39 +32,62 @@ const MONGOSH_PACKAGES = fs.readdirSync(pathToPackages, { withFileTypes: true })
     ...(JSON.parse(fs.readFileSync(path.join(pathToPackages, d.name, 'package.json'))).mongosh || {})
   }));
 
+const ALL_UNIT_TEST_BUILD_VARIANTS = [];
 const ALL_UNIT_TESTS = [];
-for (const packageInfo of MONGOSH_PACKAGES) {
-  for (const { shortName: nShort, versionSpec: nVersion, skipNodeVersionCheck } of NODE_VERSIONS) {
-    const defaultVariants = ['darwin', 'linux', 'win32'];
-    if (packageInfo.unitTestsOnly) {
-      const id = `n${nShort}_${packageInfo.name.replace(/-/g, '_')}`;
-      ALL_UNIT_TESTS.push({
-        id, nShort, nVersion, skipNodeVersionCheck,
-        packageName: packageInfo.name,
-        variants: packageInfo.variants ?? defaultVariants
-      });
-    } else {
-      for (const { shortName: mShort, versionSpec: mVersion } of MONGODB_VERSIONS) {
-        let variants = packageInfo.variants ?? defaultVariants;
-        variants = (mShort === '42xe') ?
+const platformToDetails = {
+  'darwin': { displayName: 'MacOS 14 arm64', runOn: 'macos-14-arm64', executableOsId: 'darwin-arm64' },
+  'linux': { displayName: 'Ubuntu 20.04 x64', runOn: 'ubuntu2004-small', executableOsId: 'darwin-arm64', tags: ["nightly-driver"]},
+  'win32': { displayName: 'Windows', runOn: 'windows-vsCurrent-small', executableOsId: 'win32' },
+}
+const defaultVariants = ['darwin', 'linux', 'win32'];
+
+for (const { shortName: nShort, versionSpec: nVersion, skipNodeVersionCheck } of NODE_VERSIONS) {
+  for (const platform of defaultVariants) {
+const platformDetails = platformToDetails[platform];
+  ALL_UNIT_TEST_BUILD_VARIANTS.push({
+          ...platformDetails,
+          name: `${platform}-n${nShort}`,
+          displayName: `${platformDetails.displayName} n${nShort} (Unit tests)`,
+          id: `${platform}-n${nShort}`,
+          runWithUnitTestsOnly: true,
+          tags: platformDetails.tags ?? [],
+          platform, nShort, nVersion, skipNodeVersionCheck,
+        });
+  for (const { shortName: mShort, versionSpec: mVersion } of MONGODB_VERSIONS) {
+        if (mShort === '42xe' && (platform === 'linux' || platform === 'darwin')) {
           // The MongoDB 4.2 enterprise server does not work on Ubuntu 20.04 or arm64 macOS
-          variants.filter(v => v !== 'linux' && v !== 'darwin') :
+          continue;
+        }
+        if (['42xc', '42xe', '44xc', '44xe', '50xc', '50xe'].includes(mShort) && (platform === 'darwin')) {
           // Unit tests on macOS use arm64 and therefore require 6.0+
-          (['42xc', '42xe', '44xc', '44xe', '50xc', '50xe'].includes(mShort)) ?
-          variants.filter(v => v !== 'darwin') :
-          variants;
-        const id = `m${mShort}_n${nShort}_${packageInfo.name.replace(/-/g, '_')}`;
-        ALL_UNIT_TESTS.push({
-          id, nShort, nVersion, mShort, mVersion, skipNodeVersionCheck,
-          packageName: packageInfo.name,
-          variants
+          continue;
+        }
+        ALL_UNIT_TEST_BUILD_VARIANTS.push({
+          ...platformDetails,
+          name: `tests_${platform}-m${mShort}_n${nShort}`,
+          id: `${platform}-m${mShort}_n${nShort}`,
+          runWithUnitTestsOnly: false,
+          tags: [...(platformDetails.tags ?? []), ...(mShort === 'latest' ? ["mlatest"] : [])],
+          displayName: `${platformDetails.displayName}${mShort === undefined ? '' : ` m${mShort}`} n${nShort} (Unit tests)`,
+          platform,
+          nShort, nVersion, mShort, mVersion, skipNodeVersionCheck,
         });
       }
-    }
   }
 }
 
-const ALL_UNIT_TEST_BUILD_VARIANTS = ['darwin_unit', 'linux_unit', 'win32_unit'];
+for (const packageInfo of MONGOSH_PACKAGES) {
+  const id = `${packageInfo.name.replace(/-/g, '_')}`;
+
+  const variants = packageInfo.variants ?? defaultVariants
+  ALL_UNIT_TESTS.push({
+    id,
+    packageName: packageInfo.name,
+    unitTestsOnly: packageInfo.unitTestsOnly,
+    variants,
+  });
+}
+
 
 const { RELEASE_PACKAGE_MATRIX } = require('../config/release-package-matrix');
 
@@ -280,22 +303,24 @@ functions:
         content_type: application/x-gzip
 
   check_coverage:
-    <% for (let buildVariant of ALL_UNIT_TEST_BUILD_VARIANTS) {
-        for (let unitTest of ALL_UNIT_TESTS.filter(t => t.variants.includes(buildVariant.replace(/_unit$/, '')))) { %>
+    <% for (let variant of ALL_UNIT_TEST_BUILD_VARIANTS) {
+        for (let unitTest of ALL_UNIT_TESTS.filter(t => t.variants.includes(variant.platform) && (!t.unitTestsOnly || (t.unitTestsOnly && variant.runWithUnitTestsOnly)))) {
+        let buildVariant = variant.id;
+         %>
     - command: s3.get
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
-        local_file: src/nyc-output-<% out(buildVariant) %>-<% out(unitTest.id) %>.tgz
-        remote_file: mongosh/binaries/${revision}/${revision_order_id}/nyc-output-<% out(buildVariant) %>-<% out(unitTest.id) %>.tgz
+        local_file: src/nyc-output-<% out(buildVariant) %>_<% out(unitTest.id) %>.tgz
+        remote_file: mongosh/binaries/${revision}/${revision_order_id}/nyc-output-<% out(buildVariant) %>_<% out(unitTest.id) %>.tgz
         bucket: mciuploads
     - command: shell.exec
       params:
         working_dir: src
         shell: bash
         script: |
           set -e
-          tar xvzf nyc-output-<% out(buildVariant) %>-<% out(unitTest.id) %>.tgz
+          tar xvzf nyc-output-<% out(buildVariant) %>_<% out(unitTest.id) %>.tgz
     <% } } %>
     - command: shell.exec
       params:
@@ -358,6 +383,8 @@ functions:
         shell: bash
         env:
           NODE_JS_VERSION: ${node_js_version}
+          DOCKERHUB_USERNAME: ${dockerhub_username}
+          DOCKERHUB_PASSWORD: ${dockerhub_password}
         script: |
           set -e
           {
@@ -381,6 +408,8 @@ functions:
           NODE_JS_VERSION: ${node_js_version}
           TEST_MONGOSH_EXECUTABLE: ${test_mongosh_executable|}
           KERBEROS_JUMPHOST_DOCKERFILE: ${kerberos_jumphost_dockerfile|}
+          DOCKERHUB_USERNAME: ${dockerhub_username}
+          DOCKERHUB_PASSWORD: ${dockerhub_password}
         script: |
           set -e
           {
@@ -610,6 +639,8 @@ functions:
           DISTRO_ID_OVERRIDE: ${distro_id}
           PACKAGE_VARIANT: ${package_variant}
           ARTIFACT_URL_EXTRA_TAG: unsigned
+          DOCKERHUB_USERNAME: ${dockerhub_username}
+          DOCKERHUB_PASSWORD: ${dockerhub_password}
   sign_artifact:
     - command: expansions.write
       type: setup
@@ -716,6 +747,8 @@ functions:
         shell: bash
         env:
           NODE_JS_VERSION: ${node_js_version}
+          DOCKERHUB_USERNAME: ${dockerhub_username}
+          DOCKERHUB_PASSWORD: ${dockerhub_password}
         script: |
           set -e
           set -x
@@ -730,6 +763,8 @@ functions:
         shell: bash
         env:
           NODE_JS_VERSION: ${node_js_version}
+          DOCKERHUB_USERNAME: ${dockerhub_username}
+          DOCKERHUB_PASSWORD: ${dockerhub_password}
         script: |
           set -e
           set -x
@@ -1083,7 +1118,7 @@ tasks:
   - name: check
     depends_on:
       - name: compile_ts
-        variant: linux_unit
+        variant: linux_compile
     commands:
       - func: checkout
       - func: install
@@ -1110,22 +1145,22 @@ tasks:
   # UNIT TESTS
   # E.g. test_m60xc_n20 stands for mongod 6.0.x, community edition, Node.js 20
   ###
-  <% for (const { id, nShort, nVersion, mShort, mVersion, skipNodeVersionCheck, packageName } of ALL_UNIT_TESTS) { %>
+  <% for (const { id, packageName } of ALL_UNIT_TESTS) { %>
   - name: test_<% out(id) %>
-    tags: <% out(["unit-test", ...(mShort === 'latest' ? ["mlatest"] : [])]) %>
+    tags: <% out(["assigned_to_jira_team_mongosh_mongosh", "unit-test"]) %>
     depends_on:
       - name: compile_ts
-        variant: linux_unit
+        variant: linux_compile
     commands:
       - func: checkout
       - func: install
         vars:
-          node_js_version: "<% out(nVersion) %>"
+          node_js_version: ${node_js_version}
       - func: test
         vars:
-          mongosh_server_test_version: "<% out(mVersion || '') %>"
-          node_js_version: "<% out(nVersion) %>"
-          mongosh_skip_node_version_check: "<% out(skipNodeVersionCheck) %>"
+          mongosh_server_test_version: ${mongosh_server_test_version}
+          node_js_version: ${node_js_version}
+          mongosh_skip_node_version_check: ${mongosh_skip_node_version_check}
           mongosh_test_id: "<% out(id) %>"
           mongosh_run_only_in_package: "<% out(packageName) %>"
           task_name: ${task_name}
@@ -1137,7 +1172,7 @@ tasks:
   - name: test_vscode
     depends_on:
       - name: compile_ts
-        variant: linux_unit
+        variant: linux_compile
     commands:
       - func: checkout
       - func: install
@@ -1148,10 +1183,10 @@ tasks:
           node_js_version: "<% out(NODE_JS_VERSION_20) %>"
           task_name: ${task_name}
   - name: test_connectivity
-    tags: ["extra-integration-test"]
+    tags: ["extra-integration-test", "assigned_to_jira_team_mongosh_mongosh"]
     depends_on:
       - name: compile_ts
-        variant: linux_unit
+        variant: linux_compile
     commands:
       - func: checkout
       - func: install
@@ -1162,10 +1197,10 @@ tasks:
           node_js_version: "<% out(NODE_JS_VERSION_20) %>"
           task_name: ${task_name}
   - name: test_apistrict
-    tags: ["extra-integration-test"]
+    tags: ["extra-integration-test", "assigned_to_jira_team_mongosh_mongosh"]
     depends_on:
       - name: compile_ts
-        variant: linux_unit
+        variant: linux_compile
     commands:
       - func: checkout
       - func: install
@@ -1181,7 +1216,7 @@ tasks:
     tags: ["compile-artifact"]
     depends_on:
       - name: compile_ts
-        variant: linux_unit
+        variant: linux_compile
     commands:
       - func: checkout
       - func: install
@@ -1201,7 +1236,7 @@ tasks:
     tags: ["extra-integration-test"]
     depends_on:
       - name: compile_ts
-        variant: linux_unit
+        variant: linux_compile
     commands:
       - func: checkout
       - func: install
@@ -1493,7 +1528,7 @@ tasks:
     exec_timeout_secs: 86400
     depends_on:
       - name: compile_ts
-        variant: linux_unit
+        variant: linux_compile
     commands:
       - func: checkout
       - func: install
@@ -1511,16 +1546,21 @@ tasks:
 
 # Need to run builds for every possible build variant.
 buildvariants:
-  - name: darwin_unit
-    display_name: "MacOS 14 arm64 (Unit tests)"
-    run_on: macos-14-arm64
+  <% for (const variant of ALL_UNIT_TEST_BUILD_VARIANTS) { %>
+  - name: <% out(variant.name) %>
+    display_name: "<% out(variant.displayName) %>"
+    run_on: <% out(variant.runOn) %>
+    tags: <% out(variant.tags) %>
     expansions:
-      executable_os_id: darwin-arm64
+      executable_os_id: <% out(variant.executableOsId) %>
+      mongosh_server_test_version: "<% out(variant.mVersion || '') %>"
+      node_js_version: "<% out(variant.nVersion) %>"
+      mongosh_skip_node_version_check: "<% out(variant.skipNodeVersionCheck) %>"
     tasks:
-      - name: check
-      <% for (const test of ALL_UNIT_TESTS.filter(t => t.variants.includes('darwin'))) { %>
+     <% for (const test of ALL_UNIT_TESTS.filter(test => test.variants.includes(variant.platform))) { %>
       - name: test_<% out(test.id) %>
       <% } %>
+  <% } %>
   - name: darwin
     display_name: "MacOS Big Sur"
     run_on: macos-11
@@ -1542,21 +1582,23 @@ buildvariants:
       - name: compile_artifact
       - name: e2e_tests_darwin_arm64
 
-  - name: linux_unit
-    display_name: "Ubuntu 20.04 x64 (Unit tests)"
+  - name: linux_compile
+    display_name: "Ubuntu 20.04 x64 (Compile and Check)"
     run_on: ubuntu2004-small
     tags: ["nightly-driver"]
     tasks:
       - name: compile_ts
       - name: check
-      <% for (const test of ALL_UNIT_TESTS.filter(t => t.variants.includes('linux'))) { %>
-      - name: test_<% out(test.id) %>
-      <% } %>
+  - name: linux_other
+    display_name: "Ubuntu 20.04 x64 (Other Tests)"
+    run_on: ubuntu2004-small
+    tags: ["nightly-driver"]
+    tasks:
       - name: test_vscode
       - name: test_connectivity
       - name: test_apistrict
   - name: linux_coverage
-    display_name: "Coverage and Static Analysis Check"
+    display_name: "Ubuntu 20.04 x64 (Coverage and Static Analysis Check)"
     run_on: ubuntu2004-small
     tasks:
       - name: check_coverage
@@ -1892,17 +1934,6 @@ buildvariants:
       executable_os_id: darwin-arm64
     tasks:
       - name: e2e_tests_darwin_arm64
-
-  - name: win32_unit
-    display_name: "Windows (Unit tests)"
-    run_on: windows-vsCurrent-small
-    expansions:
-      executable_os_id: win32
-    tasks:
-      - name: check
-      <% for (const test of ALL_UNIT_TESTS.filter(t => t.variants.includes('win32'))) { %>
-      - name: test_<% out(test.id) %>
-      <% } %>
   - name: win32
     display_name: "Windows VS 2022"
     run_on: windows-vsCurrent-small