Merge remote-tracking branch 'origin/beta-releases' into ga-releases

Author: github-actions[bot]

.depalignrc.json

@@ -26,6 +26,9 @@
     ],
     "js-yaml": [
       "^3.13.1"
+    ],
+    "yargs": [
+      "^4.8.1"
     ]
   }
 }

.evergreen/buildvariants-and-tasks.in.yml

@@ -28,10 +28,10 @@ const PACKAGE_BUILD_VARIANTS = [
     name: 'package-ubuntu',
     display_name: 'Package Ubuntu',
 
-    // # NOTE: We are packaging on Ubuntu 16.04 in order to use glibc 2.23
+    // # NOTE: We are packaging on Ubuntu 18.04 in order to use glibc 2.27
     // # when compiling/re-building addons. This ensures compatibility with other
-    // # debian platforms that have glibc 2.23 or newer.
-    run_on: 'ubuntu1604-large',
+    // # debian platforms that have glibc 2.27 or newer.
+    run_on: 'ubuntu1804-large',
     silk_asset_group: 'compass-ubuntu',
   },
   {
@@ -275,6 +275,12 @@ buildvariants:
     <% } %>
     <% } %>
 
+  - name: test-web-sandbox-atlas-cloud
+    display_name: Test Web Sandbox (w/ Atlas Cloud login)
+    run_on: ubuntu2004-large
+    tasks:
+      - name: test-web-sandbox-atlas-cloud
+
   - name: generate-vulnerability-report
     display_name: Vulnerability Report
     run_on: ubuntu2004-large
@@ -504,6 +510,21 @@ tasks:
   <% } %>
 <% } %>
 
+  - name: test-web-sandbox-atlas-cloud
+    tags:
+      - required-for-publish
+      - run-on-pr
+      - assigned_to_jira_team_compass_compass
+      - foliage_check_task_only
+    commands:
+      - func: prepare
+      - func: install
+      - func: bootstrap
+      - func: test-web-sandbox-atlas-cloud
+        vars:
+          compass_distribution: compass
+          debug: 'compass-e2e-tests*,electron*,hadron*,mongo*'
+
   - name: create_static_analysis_report
     tags: ['required-for-publish', 'run-on-pr']
     depends_on:

.evergreen/buildvariants-and-tasks.yml

@@ -35,7 +35,7 @@ buildvariants:
     expansions:
       silk_asset_group: compass-ubuntu
     display_name: Package Ubuntu
-    run_on: ubuntu1604-large
+    run_on: ubuntu1804-large
     tasks:
       - name: package-compass
       - name: package-compass-isolated
@@ -257,6 +257,11 @@ buildvariants:
       - name: test-web-sandbox-firefox-1
       - name: test-web-sandbox-firefox-2
       - name: test-web-sandbox-firefox-3
+  - name: test-web-sandbox-atlas-cloud
+    display_name: Test Web Sandbox (w/ Atlas Cloud login)
+    run_on: ubuntu2004-large
+    tasks:
+      - name: test-web-sandbox-atlas-cloud
   - name: generate-vulnerability-report
     display_name: Vulnerability Report
     run_on: ubuntu2004-large
@@ -1715,6 +1720,20 @@ tasks:
           e2e_test_groups: 3
           e2e_test_group: 3
           debug: compass-e2e-tests*,electron*,hadron*,mongo*
+  - name: test-web-sandbox-atlas-cloud
+    tags:
+      - required-for-publish
+      - run-on-pr
+      - assigned_to_jira_team_compass_compass
+      - foliage_check_task_only
+    commands:
+      - func: prepare
+      - func: install
+      - func: bootstrap
+      - func: test-web-sandbox-atlas-cloud
+        vars:
+          compass_distribution: compass
+          debug: compass-e2e-tests*,electron*,hadron*,mongo*
   - name: create_static_analysis_report
     tags:
       - required-for-publish

.evergreen/functions.yml

@@ -86,7 +86,7 @@ post:
       target: src/all-e2e-logs.tgz
       source_dir: src/packages/compass-e2e-tests/.log
       include:
-        - "**"
+        - '**'
   - command: s3.put
     params:
       <<: *save-artifact-params-private
@@ -203,6 +203,15 @@ functions:
           # Install dependencies
           bash ".evergreen/retry-with-backoff.sh" .evergreen/npm_ci.sh
 
+          # Will fail if versions of direct dependencies listed in package-lock
+          # are not matching versions defined in package.json file of any of the
+          # workspace packages
+          # This command is very noisy when running from root with --all, store
+          # the output in a file that will be uploaded with rest of the logs
+          LS_ALL_STDOUT_FILE="$(npm config get cache)/_logs/$(date -u +"%Y-%m-%dT%H_%M_%SZ")-npm-ls-all.log"
+          echo "Validating dependencies with \`npm ls --all\`..."
+          (npm ls --all > $LS_ALL_STDOUT_FILE && echo "No mismatched dependency versions") || echo "\nThe \`npm ls\` command failed with mismatched dependencies error. This usually means that the dependency versions listed in package.json are not matching dependencies resolved and recorded in package-lock.json. If you updated package.json files in your PR, inspect the error output and try to re-install offending dependncies to fix the package-lock file."
+
   bootstrap:
     - command: shell.exec
       type: setup
@@ -294,7 +303,7 @@ functions:
           echo "Running tests in Electron runtime"
 
           # Preparing native addons for electron runtime
-          npm run --unsafe-perm electron-rebuild
+          DEBUG=electron-rebuild* npm run --unsafe-perm electron-rebuild
 
           npm run --unsafe-perm test-ci-electron -- --stream
 
@@ -588,10 +597,9 @@ functions:
           DEBUG: ${debug|}
           MONGODB_VERSION: ${mongodb_version|}
           MONGODB_RUNNER_VERSION: ${mongodb_version|}
-          E2E_TEST_GROUPS: ${e2e_test_groups}
-          E2E_TEST_GROUP: ${e2e_test_group}
+          COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups}
+          COMPASS_E2E_TEST_GROUP: ${e2e_test_group}
           ATLAS_LOCAL_VERSION: latest
-          HADRON_DISTRIBUTION: compass
         script: |
           set -e
           # Load environment variables
@@ -620,8 +628,8 @@ functions:
           DEBUG: ${debug|}
           MONGODB_VERSION: ${mongodb_version|}
           MONGODB_RUNNER_VERSION: ${mongodb_version|}
-          E2E_TEST_GROUPS: ${e2e_test_groups}
-          E2E_TEST_GROUP: ${e2e_test_group}
+          COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups}
+          COMPASS_E2E_TEST_GROUP: ${e2e_test_group}
         script: |
           set -e
           # Load environment variables
@@ -655,14 +663,50 @@ functions:
           DEBUG: ${debug|}
           MONGODB_VERSION: ${mongodb_version|}
           MONGODB_RUNNER_VERSION: ${mongodb_version|}
-          BROWSER_NAME: ${browser_name}
-          E2E_TEST_GROUPS: ${e2e_test_groups}
-          E2E_TEST_GROUP: ${e2e_test_group}
+          COMPASS_E2E_BROWSER_NAME: ${browser_name}
+          COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups}
+          COMPASS_E2E_TEST_GROUP: ${e2e_test_group}
+        script: |
+          set -e
+          # Load environment variables
+          eval $(.evergreen/print-compass-env.sh)
+          npm run --unsafe-perm --workspace compass-e2e-tests test-ci web
+
+    
+  test-web-sandbox-atlas-cloud:
+    - command: shell.exec
+      # It can take a very long time for Atlas cluster to get deployed
+      timeout_secs: 2400
+      params:
+        working_dir: src
+        shell: bash
+        env:
+          <<: *compass-env
+          DEBUG: ${debug|}
+          COMPASS_E2E_ATLAS_CLOUD_SANDBOX_CLOUD_CONFIG: 'qa'
+          COMPASS_E2E_ATLAS_CLOUD_SANDBOX_USERNAME: ${e2e_tests_compass_web_atlas_username}
+          COMPASS_E2E_ATLAS_CLOUD_SANDBOX_PASSWORD: ${e2e_tests_compass_web_atlas_password}
+          COMPASS_E2E_ATLAS_CLOUD_SANDBOX_DBUSER_USERNAME: ${e2e_tests_compass_web_atlas_db_username}
+          COMPASS_E2E_ATLAS_CLOUD_SANDBOX_DBUSER_PASSWORD: ${e2e_tests_compass_web_atlas_password}
+          MCLI_PUBLIC_API_KEY: ${e2e_tests_mcli_public_api_key}
+          MCLI_PRIVATE_API_KEY: ${e2e_tests_mcli_private_api_key}
+          MCLI_ORG_ID: ${e2e_tests_mcli_org_id}
+          MCLI_PROJECT_ID: ${e2e_tests_mcli_project_id}
+          MCLI_OPS_MANAGER_URL: ${e2e_tests_mcli_ops_manager_url}
         script: |
           set -e
           # Load environment variables
           eval $(.evergreen/print-compass-env.sh)
-          npm run --unsafe-perm --workspace compass-e2e-tests test-web
+          # Create Atlas cluster for test project
+          source .evergreen/start-atlas-cloud-cluster.sh
+          # Run the tests
+          echo "Starting e2e tests..."
+          # We're only running a special subset of tests as provisioning atlas
+          # clusters in CI is both pricey and flakey, so we want to limit the
+          # coverage to reduce those factors (at least for now)
+          npm run --unsafe-perm --workspace compass-e2e-tests test-ci -- -- web \
+            --test-atlas-cloud-sandbox \
+            --test-filter="atlas-cloud/**/*"
 
   test-connectivity:
     - command: shell.exec

.evergreen/node-gyp-bug-workaround.sh

@@ -39,9 +39,6 @@ if [ -n "$IS_WINDOWS" ]; then
     ./node.exe node_modules/npm2/bin/npm-cli.js i -g npm@$NPM_VERSION
     rm -rf node_modules/npm2/
     chmod +x npm.cmd npm
-
-    cd ..
-    .evergreen/node-gyp-bug-workaround.sh
 else
     if command -v ldd &> /dev/null && `ldd $(which bash) | grep 'libc.so' | awk '{print $3}'` | grep -Eq 'release version 2.(1|2[0-7])'; then
         echo "Installing unofficial nodejs compiled for glibc 2.17 v${NODE_JS_VERSION} for ${PLATFORM} on ${ARCH}..."

.evergreen/preinstall.sh

@@ -68,13 +68,10 @@ function printCompassEnv() {
   }
 
   if (process.env.PLATFORM === 'linux') {
-    // To build node modules on linux post electron 13 we need
-    // a newer c++ compiler version, this adds it.
+    // To build node modules on linux post electron 13 we need a newer c++
+    // compiler version and at least python v3.9, this adds it.
     // https://jira.mongodb.org/browse/COMPASS-5150
-    pathsToPrepend.unshift('/opt/mongodbtoolchain/v3/bin');
-
-    // Make sure that linux is using python 3.6 (node-gyp requirement)
-    pathsToPrepend.unshift('/opt/python/3.6/bin');
+    pathsToPrepend.unshift('/opt/mongodbtoolchain/v4/bin');
   }
 
   PATH = maybePrependPaths(PATH, pathsToPrepend);
@@ -103,17 +100,19 @@ function printCompassEnv() {
   printVar('IS_RHEL', process.env.IS_RHEL);
   printVar('IS_UBUNTU', process.env.IS_UBUNTU);
   printVar('DEBUG', process.env.DEBUG);
-  printVar('MONGODB_VERSION', process.env.MONGODB_VERSION || process.env.MONGODB_DEFAULT_VERSION);
+  printVar(
+    'MONGODB_VERSION',
+    process.env.MONGODB_VERSION || process.env.MONGODB_DEFAULT_VERSION
+  );
   printVar('DEV_VERSION_IDENTIFIER', process.env.DEV_VERSION_IDENTIFIER);
   printVar('EVERGREEN_REVISION', process.env.EVERGREEN_REVISION);
-  printVar('EVERGREEN_REVISION_ORDER_ID', process.env.EVERGREEN_REVISION_ORDER_ID);
+  printVar(
+    'EVERGREEN_REVISION_ORDER_ID',
+    process.env.EVERGREEN_REVISION_ORDER_ID
+  );
 
-  if (process.platform === 'darwin') {
-    // Without this, kerberos 2.1.1 is broken on macOS, but this flag is only
-    // really relevant for Linux.
-    // https://jira.mongodb.org/browse/NODE-6320
-    printVar('GYP_DEFINES', 'kerberos_use_rtld=false');
-  }
+  // https://jira.mongodb.org/browse/NODE-6320
+  printVar('GYP_DEFINES', `kerberos_use_rtld=${process.platform === 'linux'}`);
 }
 
 printCompassEnv();

.evergreen/print-compass-env.js

@@ -2,7 +2,7 @@
 
 set -e
 
-export MONGODB_DEFAULT_VERSION=6.0.x
+export MONGODB_DEFAULT_VERSION=7.0.x
 
 if [[ $OSTYPE == "cygwin" ]]; then
     export PLATFORM='win32'

.evergreen/print-compass-env.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Atlas limits the naming to something like /^[\w\d-]{,23}$/ (and will auto
+# truncate if it's too long) so we're very limited in terms of how unique this
+# name can be. Hopefully the epoch + part of git hash is enough for these to not
+# overlap when tests are running
+ATLAS_CLOUD_TEST_CLUSTER_NAME="e2e-$(date +"%s")-$(git rev-parse HEAD)"
+
+function atlascli() {
+  docker run \
+    -e MCLI_PUBLIC_API_KEY \
+    -e MCLI_PRIVATE_API_KEY \
+    -e MCLI_ORG_ID \
+    -e MCLI_PROJECT_ID \
+    -e MCLI_OPS_MANAGER_URL \
+    mongodb/atlas atlas $@
+}
+
+cleanup() {
+  echo "Scheduling Atlas deployment \`$ATLAS_CLOUD_TEST_CLUSTER_NAME\` for deletion..."
+  atlascli clusters delete $ATLAS_CLOUD_TEST_CLUSTER_NAME --force
+}
+
+trap cleanup EXIT
+
+echo "Creating Atlas deployment \`$ATLAS_CLOUD_TEST_CLUSTER_NAME\` to test against..."
+atlascli clusters create $ATLAS_CLOUD_TEST_CLUSTER_NAME \
+  --provider AWS \
+  --region US_EAST_1 \
+  --tier M10
+
+echo "Waiting for the deployment to be provisioned..."
+atlascli clusters watch "$ATLAS_CLOUD_TEST_CLUSTER_NAME"
+
+echo "Getting connection string for provisioned cluster..."
+ATLAS_CLOUD_TEST_CLUSTER_CONNECTION_STRING_JSON="$(atlascli clusters connectionStrings describe $ATLAS_CLOUD_TEST_CLUSTER_NAME -o json)"
+
+export COMPASS_E2E_ATLAS_CLOUD_SANDBOX_DEFAULT_CONNECTIONS="{\"$ATLAS_CLOUD_TEST_CLUSTER_NAME\": $ATLAS_CLOUD_TEST_CLUSTER_CONNECTION_STRING_JSON}"
+echo "Cluster connections: $COMPASS_E2E_ATLAS_CLOUD_SANDBOX_DEFAULT_CONNECTIONS"

.evergreen/start-atlas-cloud-cluster.sh

@@ -55,5 +55,6 @@ jobs:
           commit-message: 'chore(release): bump package versions'
           branch: ci/bump-packages
           title: 'chore(release): bump package versions'
+          labels: no-title-validation
           body: |
             - Bump package versions