feat: update libmongocrypt and fix tests (#2076)

- Migrate mongosh to new mongodb-client-encryption repository
- Update mongodb-client-encryption to 6.1.0
- Enable QE Range as GA and disable the RangePreview version of the algorithm
- Use the 8.0 server crypt_shared library everywhere and consequentially
  break out-of-the-box automatic encryption on platforms with lower glibc
  requirements

Co-authored-by: Neal Beeken <[email protected]>
Co-authored-by: Anna Henningsen <[email protected]>

Author: 3 people

.evergreen.yml

@@ -15,8 +15,8 @@ const MONGODB_VERSIONS = [
   { shortName: '60xe', versionSpec: '6.0.x-enterprise' },
   { shortName: '70xc', versionSpec: '7.0.x' },
   { shortName: '70xe', versionSpec: '7.0.x-enterprise' },
-  { shortName: '80xc', versionSpec: '>= 8.0.0-rc15' },
-  { shortName: '80xe', versionSpec: '>= 8.0.0-rc15-enterprise' },
+  { shortName: '80xc', versionSpec: '8.0.0-rc17' },
+  { shortName: '80xe', versionSpec: '8.0.0-rc17-enterprise' },
   { shortName: 'latest', versionSpec: 'latest-alpha-enterprise' }
 ];
 const NODE_VERSIONS = [
@@ -743,6 +743,8 @@ functions:
       params:
         working_dir: src
         shell: bash
+        env:
+          MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|}
         script: |
           set -e
           set -x
@@ -769,6 +771,8 @@ functions:
       params:
         working_dir: src
         shell: bash
+        env:
+          MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|}
         script: |
           set -e
           set -x
@@ -794,6 +798,8 @@ functions:
       params:
         working_dir: src
         shell: bash
+        env:
+          MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT: ${no_automatic_encryption_support|}
         script: |
           set -e
           set -x
@@ -1192,7 +1198,7 @@ tasks:
   # E2E TESTS
   ###
   <% for (const { executableOsId, compileBuildVariant } of RELEASE_PACKAGE_MATRIX) {
-    for (const mVersion of ['stable', '8.0.0-rc15', '6.0.x']) {
+    for (const mVersion of ['stable', '8.0.0-rc17', '6.0.x']) {
       for (const fipsVariant of ['fips', 'nofips']) {
     %>
   - name: e2e_tests_<% out(executableOsId.replace(/-/g, '_')) %><% out(mVersion === 'stable' ? '' : '_' + mVersion.replace(/[^a-zA-Z0-9]/g, '')) %><% out(fipsVariant === 'fips' ? '_fips' : '') %>
@@ -1732,8 +1738,8 @@ buildvariants:
     run_on: ubuntu2404-small
     tags: ["nightly-driver"]
     tasks:
-      - name: e2e_tests_linux_x64_800rc15
-      - name: e2e_tests_linux_x64_openssl3_800rc15
+      - name: e2e_tests_linux_x64_800rc17
+      - name: e2e_tests_linux_x64_openssl3_800rc17
   - name: e2e_debian10_x64
     display_name: "Debian 10 x64 (E2E Tests)"
     run_on: debian10-small
@@ -1795,8 +1801,8 @@ buildvariants:
     display_name: "Ubuntu 24.04 arm64 (E2E Tests)"
     run_on: ubuntu2404-arm64-small
     tasks:
-      - name: e2e_tests_linux_arm64_800rc15
-      - name: e2e_tests_linux_arm64_openssl3_800rc15
+      - name: e2e_tests_linux_arm64_800rc17
+      - name: e2e_tests_linux_arm64_openssl3_800rc17
   - name: e2e_amazon2_arm64
     display_name: "Amazon Linux 2 arm64 (E2E Tests)"
     run_on: amazon2-arm64-large
@@ -1965,6 +1971,8 @@ buildvariants:
   - name: pkg_smoke_tests_rhel72_s390x
     display_name: "package smoke tests (RHEL 7.2 s390x)"
     run_on: rhel72-zseries-small
+    expansions:
+      no_automatic_encryption_support: 1
     tasks:
       - name: pkg_test_rpmextract_rpm_s390x
   - name: pkg_smoke_tests_rhel83_s390x

.evergreen/evergreen.yml.in

@@ -47,7 +47,7 @@
     "generate-error-overview": "npm run generate-error-overview --workspace @mongosh/errors",
     "update-authors": "ts-node -P configs/tsconfig-mongosh/tsconfig.common.json scripts/generate-authors.ts",
     "create-dependency-sbom-lists": "npm run webpack-build -w packages/cli-repl && npm run write-node-js-dep && npm run create-purls-file && npm run create-first-party-dependency-lists",
-    "create-first-party-dependency-lists": "mongodb-sbom-tools fetch-codeql-results --first-party-deps-list-dest=.sbom/first-party-deps.json --dependencies=.sbom/dependencies.json --exclude-repos=mongodb-js/kerberos",
+    "create-first-party-dependency-lists": "mongodb-sbom-tools fetch-codeql-results --first-party-deps-list-dest=.sbom/first-party-deps.json --dependencies=.sbom/dependencies.json --exclude-repos=mongodb-js/kerberos,mongodb-client-encryption",
     "create-purls-file": "node scripts/create-purls.js .sbom/dependencies.json .sbom/node-js-dep.json > .sbom/purls.txt",
     "preupdate-third-party-notices": "npm run create-dependency-sbom-lists",
     "update-third-party-notices": "mongodb-sbom-tools generate-3rd-party-notices --product='mongosh' --dependencies=.sbom/dependencies.json > THIRD_PARTY_NOTICES.md",

package-lock.json

@@ -32,7 +32,7 @@ async function preCompileHook(nodeSourceTree: string) {
       env: {
         ...process.env,
         FLE_NODE_SOURCE_PATH: nodeSourceTree,
-        LIBMONGOCRYPT_VERSION: `node-v${fleAddonVersion}`,
+        MONGODB_CLIENT_ENCRYPTION_VERSION: `v${fleAddonVersion}`,
       },
       stdio: 'inherit',
     }

package.json

@@ -34,13 +34,12 @@ export async function downloadCryptLibrary(
   );
   // Download mongodb for latest server version, including rapid releases
   // (for the platforms that they exist for, i.e. for ppc64le/s390x only pick stable releases).
-  let versionSpec = 'continuous';
-  if (/ppc64/.test(opts.arch || process.arch)) {
-    versionSpec = 'stable';
-  }
-  if (/s390x/.test(opts.arch || process.arch)) {
-    versionSpec = '6.0.x'; // The 7.x+ server releases don't have RHEL7-compatible crypt_shared libraries
-  }
+  // TODO(MONGOSH-1833): The current 'continuous' release is not compatible with 8.x rc server releases. So we are using
+  // 8.0.0-rc17 (current latest) for now and once 8.0 is released we should switch back to continuous.
+  const versionSpec = '8.0.0-rc17';
+  //if (/ppc64|s390x/.test(opts.arch || process.arch)) {
+  //  versionSpec = 'stable';
+  //}
   const { downloadedBinDir: libdir, version } =
     await downloadMongoDbWithVersionInfo(cryptTmpTargetDir, versionSpec, opts);
   const cryptLibrary = path.join(
@@ -73,11 +72,10 @@ function lookupReleaseDistro(packageVariant: PackageVariant): {
     case 'ppc64le':
       return { platform: 'linux', distro: 'rhel81' };
     case 's390x':
-      return { platform: 'linux', distro: 'rhel72' };
+      return { platform: 'linux', distro: 'rhel83' };
     case 'arm64':
-      return { platform: 'linux', distro: 'amazon2' };
     case 'x64':
-      return { platform: 'linux', distro: 'rhel70' };
+      return { platform: 'linux', distro: 'rhel8' };
     default:
       break;
   }

packages/build/src/compile/signable-compiler.ts

@@ -13,7 +13,8 @@ const assert = function(value, message) {
   }
 };
 if (db.version().startsWith('4.0.') ||
-    !db.runCommand({buildInfo:1}).modules.includes('enterprise')) {
+    !db.runCommand({buildInfo:1}).modules.includes('enterprise') ||
+    !!process.env.MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT) {
   // No FLE on mongod < 4.2 or community
   print('Test skipped')
   process.exit(0)

packages/build/src/packaging/download-crypt-library.ts

@@ -73,8 +73,13 @@ export async function runSmokeTests({
   const expectFipsSupport =
     !!process.env.MONGOSH_SMOKE_TEST_OS_HAS_FIPS_SUPPORT &&
     (await buildInfo()).sharedOpenssl;
+  const expectAutomaticEncryptionSupport =
+    !process.env.MONGOSH_NO_AUTOMATIC_ENCRYPTION_SUPPORT;
   if (!wantPerformanceTesting) {
-    console.log('FIPS support required to pass?', { expectFipsSupport });
+    console.log('FIPS/FLE support required to pass?', {
+      expectFipsSupport,
+      expectAutomaticEncryptionSupport,
+    });
   }
   const perfResults: PerfTestResult[] = [];