Merge remote-tracking branch 'origin/beta-releases' into ga-releases

Author: github-actions[bot]

.evergreen.yml

@@ -1,7 +1,11 @@
 stepback: false
 exec_timeout_secs: 5400
 ignore:
+  - docs/**/*.md
   - AUTHORS
+  - CODE_OF_CONDUCT.md
+  - CONTRIBUTING.md
+  - README.md
   - THIRD-PARTY-NOTICES.md
 include:
   - filename: .evergreen/functions.yml

.evergreen/buildvariants-and-tasks.in.yml

@@ -61,19 +61,18 @@ const PACKAGE_BUILD_VARIANTS = [
 ];
 
 const SMOKETEST_BUILD_VARIANTS = [
-  {
-    name: 'smoketest-ubuntu',
-    display_name: 'Smoketest Ubuntu',
-    run_on: 'ubuntu2004-large',
-    depends_on: 'package-ubuntu',
-  },
-  
 //  {
-//    name: 'smoketest-windows',
-//    display_name: 'Smoketest Windows',
-//    run_on: 'windows-vsCurrent-large',
-//    depends_on: 'package-windows',
+//    name: 'smoketest-ubuntu',
+//    display_name: 'Smoketest Ubuntu',
+//    run_on: 'ubuntu2004-large',
+//    depends_on: 'package-ubuntu',
 //  },
+  {
+    name: 'smoketest-windows',
+    display_name: 'Smoketest Windows',
+    run_on: 'windows-vsCurrent-large',
+    depends_on: 'package-windows',
+  },
 //  {
 //    name: 'smoketest-rhel',
 //    display_name: 'Smoketest RHEL',
@@ -288,7 +287,6 @@ buildvariants:
     run_on: ubuntu2004-large
     tasks:
       - name: publish
-      - name: publish-packages-next
       - name: publish-dev-release-info
       - name: create_static_analysis_report
 
@@ -430,17 +428,6 @@ tasks:
       - func: get-all-artifacts
       - func: publish
 
-  - name: publish-packages-next
-    tags: []
-    depends_on:
-      - name: '.required-for-publish'
-        variant: '*'
-    commands:
-      - func: prepare
-      - func: install
-      - func: bootstrap
-      - func: publish-packages-next
-
   - name: publish-dev-release-info
     tags: []
     depends_on:

.evergreen/buildvariants-and-tasks.yml

@@ -76,12 +76,12 @@ buildvariants:
       - name: package-compass
       - name: package-compass-isolated
       - name: package-compass-readonly
-  - name: smoketest-ubuntu-compass
-    display_name: Smoketest Ubuntu (compass)
-    run_on: ubuntu2004-large
+  - name: smoketest-windows-compass
+    display_name: Smoketest Windows (compass)
+    run_on: windows-vsCurrent-large
     depends_on:
       - name: package-compass
-        variant: package-ubuntu
+        variant: package-windows
     tasks:
       - name: smoketest-compass
   - name: smoketest-macos-x64-compass
@@ -251,7 +251,6 @@ buildvariants:
     run_on: ubuntu2004-large
     tasks:
       - name: publish
-      - name: publish-packages-next
       - name: publish-dev-release-info
       - name: create_static_analysis_report
   - name: connectivity-tests
@@ -409,16 +408,6 @@ tasks:
           scope: mongodb-compass
       - func: get-all-artifacts
       - func: publish
-  - name: publish-packages-next
-    tags: []
-    depends_on:
-      - name: .required-for-publish
-        variant: '*'
-    commands:
-      - func: prepare
-      - func: install
-      - func: bootstrap
-      - func: publish-packages-next
   - name: publish-dev-release-info
     tags: []
     depends_on:

.evergreen/connectivity-tests/run.sh

@@ -22,7 +22,7 @@ docker run \
   -e E2E_TESTS_FREE_TIER_HOST="${E2E_TESTS_FREE_TIER_HOST}" \
   -e E2E_TESTS_ATLAS_USERNAME="${E2E_TESTS_ATLAS_USERNAME}" \
   -e E2E_TESTS_ATLAS_PASSWORD="${E2E_TESTS_ATLAS_PASSWORD}" \
-  -e E2E_TESTS_ATLAS_X509_PEM="${E2E_TESTS_ATLAS_X509_PEM}" \
+  -e E2E_TESTS_ATLAS_X509_PEM_BASE64="${E2E_TESTS_ATLAS_X509_PEM_BASE64}" \
   -e MONGODB_VERSION="${MONGODB_VERSION}" \
   --add-host mongodb-kerberos-1.example.com:0.0.0.0 \
   --add-host mongodb-kerberos-2.example.com:0.0.0.0 \

.evergreen/create-sbom.sh

@@ -18,8 +18,8 @@ trap_handler() {
 }
 trap trap_handler ERR EXIT
 
-scp -i "$SIGNING_SERVER_PRIVATE_KEY_CYGPATH" -P "$SIGNING_SERVER_PORT" .sbom/dependencies.json /tmp/silkbomb.env /tmp/artifactory_password "$SIGNING_SERVER_USERNAME"@"$SIGNING_SERVER_HOSTNAME":/tmp/
-ssh -i "$SIGNING_SERVER_PRIVATE_KEY_CYGPATH" -p "$SIGNING_SERVER_PORT" "$SIGNING_SERVER_USERNAME"@"$SIGNING_SERVER_HOSTNAME" \
+scp -v -i "$SIGNING_SERVER_PRIVATE_KEY_CYGPATH" -P "$SIGNING_SERVER_PORT" .sbom/dependencies.json /tmp/silkbomb.env /tmp/artifactory_password "$SIGNING_SERVER_USERNAME"@"$SIGNING_SERVER_HOSTNAME":/tmp/
+ssh -v -i "$SIGNING_SERVER_PRIVATE_KEY_CYGPATH" -p "$SIGNING_SERVER_PORT" "$SIGNING_SERVER_USERNAME"@"$SIGNING_SERVER_HOSTNAME" \
   "(cat /tmp/dependencies.json | jq -r '.[] | "'"pkg:npm/" + .name + "@" + .version'"' > /tmp/purls.txt) && \
   echo "pkg:generic/mongo_crypt_shared@${CRYPT_SHARED_VERSION}" >> /tmp/purls.txt && \
   (cat /tmp/artifactory_password | docker login artifactory.corp.mongodb.com --username '${ARTIFACTORY_USERNAME}' --password-stdin ; rm -f /tmp/artifactor_password ) && \
@@ -30,4 +30,4 @@ ssh -i "$SIGNING_SERVER_PRIVATE_KEY_CYGPATH" -p "$SIGNING_SERVER_PORT" "$SIGNING
     --silk-asset-group "${SILK_ASSET_GROUP}" --sbom-in /tmp/sbom-lite.json && \
   docker run --env-file /tmp/silkbomb.env --rm -v /tmp:/tmp artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 download \
     --silk-asset-group "${SILK_ASSET_GROUP}" --sbom-out /tmp/sbom.json"
-scp -i "$SIGNING_SERVER_PRIVATE_KEY_CYGPATH" -P "$SIGNING_SERVER_PORT" "$SIGNING_SERVER_USERNAME"@"$SIGNING_SERVER_HOSTNAME":/tmp/{sbom-lite.json,sbom.json,purls.txt} .sbom/
+scp -v -i "$SIGNING_SERVER_PRIVATE_KEY_CYGPATH" -P "$SIGNING_SERVER_PORT" "$SIGNING_SERVER_USERNAME"@"$SIGNING_SERVER_HOSTNAME":/tmp/{sbom-lite.json,sbom.json,purls.txt} .sbom/

.evergreen/functions.yml

@@ -45,6 +45,15 @@ variables:
     # secrets
     HADRON_METRICS_INTERCOM_APP_ID: ${metrics_intercom_app_id}
     HADRON_METRICS_SEGMENT_API_KEY: ${metrics_segment_api_key}
+    GITHUB_TOKEN: ${devtoolsbot_github_token}
+    DOWNLOAD_CENTER_AWS_ACCESS_KEY_ID: ${aws_key_evergreen_integrations}
+    DOWNLOAD_CENTER_AWS_SECRET_ACCESS_KEY: ${aws_secret_evergreen_integrations}
+    EVERGREEN_BUCKET_NAME: mciuploads
+    EVERGREEN_BUCKET_KEY_PREFIX: ${project}/${revision}_${revision_order_id}
+    MONGODB_RUNNER_LOG_DIR: ${workdir}/src/.testserver/
+    DOCKERHUB_USERNAME: ${dockerhub_username}
+    DOCKERHUB_PASSWORD: ${dockerhub_password}
+  - &compass-e2e-secrets
     E2E_TESTS_METRICS_URI: ${e2e_tests_metrics_string}
     E2E_TESTS_ATLAS_HOST: ${e2e_tests_atlas_host}
     E2E_TESTS_DATA_LAKE_HOST: ${e2e_tests_data_lake_host}
@@ -53,7 +62,7 @@ variables:
     E2E_TESTS_FREE_TIER_HOST: ${e2e_tests_free_tier_host}
     E2E_TESTS_ATLAS_USERNAME: ${e2e_tests_atlas_username}
     E2E_TESTS_ATLAS_PASSWORD: ${e2e_tests_atlas_password}
-    E2E_TESTS_ATLAS_X509_PEM: ${e2e_tests_atlas_x509_pem}
+    E2E_TESTS_ATLAS_X509_PEM_BASE64: ${e2e_tests_atlas_x509_pem_base64}
     E2E_TESTS_ATLAS_IAM_ACCESS_KEY_ID: ${e2e_tests_atlas_iam_aws_access_key_id}
     E2E_TESTS_ATLAS_IAM_SECRET_ACCESS_KEY: ${e2e_tests_atlas_iam_aws_secret_access_key}
     E2E_TESTS_ATLAS_IAM_TEMP_ROLE_ARN: ${e2e_tests_atlas_iam_temp_role_arn}
@@ -62,24 +71,8 @@ variables:
     E2E_TESTS_ATLAS_READANYDATABASE_STRING: ${e2e_tests_atlas_readanydatabase_string}
     E2E_TESTS_ATLAS_CUSTOMROLE_STRING: ${e2e_tests_atlas_customrole_string}
     E2E_TESTS_ATLAS_SPECIFICPERMISSION_STRING: ${e2e_tests_atlas_specificpermission_string}
-    MACOS_NOTARY_KEY: ${macos_notary_key}
-    MACOS_NOTARY_SECRET: ${macos_notary_secret}
-    MACOS_NOTARY_CLIENT_URL: 'https://macos-notary-1628249594.s3.amazonaws.com/releases/client/latest/darwin_amd64.zip'
-    MACOS_NOTARY_API_URL: 'https://dev.macos-notary.build.10gen.cc/api'
-    GITHUB_TOKEN: ${devtoolsbot_github_token}
-    DOWNLOAD_CENTER_AWS_ACCESS_KEY_ID: ${aws_key_evergreen_integrations}
-    DOWNLOAD_CENTER_AWS_SECRET_ACCESS_KEY: ${aws_secret_evergreen_integrations}
-    EVERGREEN_BUCKET_NAME: mciuploads
-    EVERGREEN_BUCKET_KEY_PREFIX: ${project}/${revision}_${revision_order_id}
-    MONGODB_RUNNER_LOG_DIR: ${workdir}/src/.testserver/
     E2E_TESTS_ATLAS_CS_WITHOUT_SEARCH: ${e2e_tests_atlas_cs_without_search}
     E2E_TESTS_ATLAS_CS_WITH_SEARCH: ${e2e_tests_atlas_cs_with_search}
-    GARASIGN_USERNAME: ${garasign_username}
-    GARASIGN_PASSWORD: ${garasign_password}
-    ARTIFACTORY_USERNAME: ${artifactory_username}
-    ARTIFACTORY_PASSWORD: ${artifactory_password}
-    DOCKERHUB_USERNAME: ${dockerhub_username}
-    DOCKERHUB_PASSWORD: ${dockerhub_password}
 
 # This is here with the variables because anchors aren't supported across includes
 post:
@@ -159,8 +152,6 @@ functions:
 
           # Make all the dirs
           mkdir -p $ARTIFACTS_PATH
-          mkdir -p $NPM_CACHE_DIR
-          mkdir -p $NPM_TMP_DIR
 
     - command: shell.exec
       type: setup
@@ -202,18 +193,9 @@ functions:
           npm config ls -l
           echo "(if npm fails, debug.log will be uploaded to S3)"
 
-          # Install dependencies
+          # Install and check dependencies
           bash ".evergreen/retry-with-backoff.sh" .evergreen/npm_ci.sh
 
-          # Will fail if versions of direct dependencies listed in package-lock
-          # are not matching versions defined in package.json file of any of the
-          # workspace packages
-          # This command is very noisy when running from root with --all, store
-          # the output in a file that will be uploaded with rest of the logs
-          LS_ALL_STDOUT_FILE="$(npm config get cache)/_logs/$(date -u +"%Y-%m-%dT%H_%M_%SZ")-npm-ls-all.log"
-          echo "Validating dependencies with \`npm ls --all\`..."
-          (npm ls --all > $LS_ALL_STDOUT_FILE && echo "No mismatched dependency versions") || echo "\nThe \`npm ls\` command failed with mismatched dependencies error. This usually means that the dependency versions listed in package.json are not matching dependencies resolved and recorded in package-lock.json. If you updated package.json files in your PR, inspect the error output and try to re-install offending dependncies to fix the package-lock file."
-
   bootstrap:
     - command: shell.exec
       type: setup
@@ -450,7 +432,7 @@ functions:
         script: |
           set -e
 
-          .evergreen/create-sbom.sh
+          bash ".evergreen/retry-with-backoff.sh" .evergreen/create-sbom.sh
     - command: shell.exec
       params:
         working_dir: src
@@ -459,12 +441,25 @@ functions:
           <<: *compass-env
           DEBUG: ${debug}
           npm_config_loglevel: ${npm_loglevel}
-          HADRON_DISTRIBUTION: ${compass_distribution}
+
+          # macOS signing secrets
+          MACOS_NOTARY_KEY: ${macos_notary_key}
+          MACOS_NOTARY_SECRET: ${macos_notary_secret}
+          MACOS_NOTARY_CLIENT_URL: 'https://macos-notary-1628249594.s3.amazonaws.com/releases/client/latest/darwin_amd64.zip'
+          MACOS_NOTARY_API_URL: 'https://dev.macos-notary.build.10gen.cc/api'
+
+          # linux / windows signing secrets
+          GARASIGN_USERNAME: ${garasign_username}
+          GARASIGN_PASSWORD: ${garasign_password}
+          ARTIFACTORY_USERNAME: ${artifactory_username}
+          ARTIFACTORY_PASSWORD: ${artifactory_password}
           SIGNING_SERVER_HOSTNAME: ${SIGNING_SERVER_HOSTNAME}
           SIGNING_SERVER_PRIVATE_KEY: ${SIGNING_SERVER_PRIVATE_KEY}
           SIGNING_SERVER_PRIVATE_KEY_CYGPATH: ${SIGNING_SERVER_PRIVATE_KEY_CYGPATH}
           SIGNING_SERVER_USERNAME: ${SIGNING_SERVER_USERNAME}
           SIGNING_SERVER_PORT: ${SIGNING_SERVER_PORT}
+
+          HADRON_DISTRIBUTION: ${compass_distribution}
           GITHUB_PR_NUMBER: ${github_pr_number}
           PAPERTRAIL_KEY_ID: ${papertrail_key_id}
           PAPERTRAIL_SECRET_KEY: ${papertrail_secret_key}
@@ -596,6 +591,7 @@ functions:
         shell: bash
         env:
           <<: *compass-env
+          <<: *compass-e2e-secrets
           DEBUG: ${debug|}
           MONGODB_VERSION: ${mongodb_version|}
           MONGODB_RUNNER_VERSION: ${mongodb_version|}
@@ -625,6 +621,7 @@ functions:
         shell: bash
         env:
           <<: *compass-env
+          <<: *compass-e2e-secrets
           COMPASS_APP_PATH_ORIGINAL: ${appPath}
           COMPASS_APP_NAME: ${packagerOptions.name}
           DEBUG: ${debug|}
@@ -660,6 +657,7 @@ functions:
         shell: bash
         env:
           <<: *compass-env
+          <<: *compass-e2e-secrets
           DEBUG: ${debug|}
           MONGODB_VERSION: ${mongodb_version|}
           MONGODB_RUNNER_VERSION: ${mongodb_version|}
@@ -669,12 +667,28 @@ functions:
           # Load environment variables
           eval $(.evergreen/print-compass-env.sh)
 
+          if [[ "$IS_WINDOWS" == "true" ]]; then
+            # TODO: windows_setup
+            # TODO: windows_msi
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip
+          fi
+
           if [[ "$IS_OSX" == "true" ]]; then
             echo "Disabling clipboard usage in e2e tests (TODO: https://jira.mongodb.org/browse/BUILD-14780)"
             export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true"
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip
           fi
 
-          npm run --unsafe-perm --workspace compass-e2e-tests smoketest
+          #if [[ "$IS_UBUNTU" == "true" ]]; then
+            # TODO: linux_deb
+            # TODO: linux_tar
+          #fi
+
+          #if [[ "$IS_RHEL" == "true" ]]; then
+            # TODO: linux_rpm
+            # TODO: rhel_tar
+          #fi
 
   test-web-sandbox:
     - command: shell.exec
@@ -685,6 +699,7 @@ functions:
         shell: bash
         env:
           <<: *compass-env
+          <<: *compass-e2e-secrets
           COMPASS_APP_PATH_ORIGINAL: ${appPath}
           COMPASS_APP_NAME: ${packagerOptions.name}
           DEBUG: ${debug|}
@@ -699,7 +714,6 @@ functions:
           eval $(.evergreen/print-compass-env.sh)
           npm run --unsafe-perm --workspace compass-e2e-tests test-ci web
 
-    
   test-web-sandbox-atlas-cloud:
     - command: shell.exec
       # It can take a very long time for Atlas cluster to get deployed
@@ -709,6 +723,7 @@ functions:
         shell: bash
         env:
           <<: *compass-env
+          <<: *compass-e2e-secrets
           DEBUG: ${debug|}
           COMPASS_E2E_ATLAS_CLOUD_SANDBOX_USERNAME: ${e2e_tests_compass_web_atlas_username}
           COMPASS_E2E_ATLAS_CLOUD_SANDBOX_PASSWORD: ${e2e_tests_compass_web_atlas_password}
@@ -741,6 +756,7 @@ functions:
         shell: bash
         env:
           <<: *compass-env
+          <<: *compass-e2e-secrets
           COMPASS_SKIP_KERBEROS_TESTS: 'true'
           COMPASS_RUN_DOCKER_TESTS: 'true'
           DEBUG: ${debug}
@@ -955,37 +971,6 @@ functions:
           echo "Downloading release assets from evergreen bucket..."
           npm run --workspace mongodb-compass download
 
-  generate-tasks:
-    - command: generate.tasks
-      params:
-        files:
-          - tasks.json
-
-  publish-packages-next:
-    - command: shell.exec
-      params:
-        working_dir: src
-        shell: bash
-        env:
-          <<: *compass-env
-          NPM_TOKEN: ${devtoolsbot_npm_token}
-        script: |
-          # Only package publish for commits on the main evergreen project.
-          if [[ "${requester}" == "commit" ]] && [[ "${project}" == "10gen-compass-main" ]]; then
-            set -e
-            # Load environment variables
-            eval $(.evergreen/print-compass-env.sh)
-            echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> .npmrc
-            npm config list
-            echo "Publishing packages as $(npm whoami)"
-            npm run version-packages-next
-            # unstage after lerna staged version
-            git reset
-            # mark files as unchanged so that lerna can publish
-            git update-index --assume-unchanged $(git diff --name-only HEAD)
-            bash ".evergreen/retry-with-backoff.sh" npm run publish-packages-next
-          fi
-
   generate-vulnerability-report:
     - command: shell.exec
       params:

.evergreen/npm_ci.sh

@@ -2,9 +2,20 @@
 
 set -e
 
-npm cache clean -f
+# Remove the cache and any potential install leftovers before installing again.
+# We are running this script with a retry to deal with network issues, in some
+# rare cases npm leaves stuff behind messing up a new attempt
+rm -rf "$NPM_CACHE_DIR"
 rm -rf node_modules
 find configs -name 'node_modules' -type d -prune -exec rm -rf '{}' +
 find packages -name 'node_modules' -type d -prune -exec rm -rf '{}' +
 find scripts -name 'node_modules' -type d -prune -exec rm -rf '{}' +
-npm ci --unsafe-perm
+npm ci --unsafe-perm
+
+# Will fail if versions of direct dependencies listed in package-lock are not
+# matching versions defined in package.json file of any of the workspace
+# packages. This command is very noisy when running from root with --all, store
+# the output in a file that will be uploaded with rest of the logs
+LS_ALL_STDOUT_FILE="$(npm config get cache)/_logs/$(date -u +"%Y-%m-%dT%H_%M_%SZ")-npm-ls-all.log"
+echo "Validating dependencies with \`npm ls --all\`..."
+(npm ls --all >$LS_ALL_STDOUT_FILE && echo "No mismatched dependency versions") || (echo ""; echo "The \`npm ls\` command failed with mismatched dependencies error. This usually means that the dependency versions listed in package.json are not matching dependencies resolved and recorded in package-lock.json. If you updated package.json files in your PR, inspect the error output and try to re-install offending dependncies to fix the package-lock file." && exit 1)