chore(ci): expand test matrix MONGOSH-1140 MONGOSH-1142 MONGOSH-1146 (#1541)

Add docker-based smoke tests for Debian 12 and e2e tests for RHEL9
and Ubuntu 22.04.

Author: addaleax

.evergreen.yml

@@ -861,10 +861,11 @@ tasks:
   # E2E TESTS
   ###
   <% for (const { executableOsId, compileBuildVariant } of RELEASE_PACKAGE_MATRIX) {
-    for (const mVersion of ['stable', '5.0.x']) {
+    // TODO(MONGOSH-1548): Account for 7.0.0 GA release when it happens
+    for (const mVersion of ['stable', '5.0.x', '7.0.0-rc10']) {
       for (const fipsVariant of ['fips', 'nofips']) {
     %>
-  - name: e2e_tests_<% out(executableOsId.replace(/-/g, '_')) %><% out(mVersion === 'stable' ? '' : '_50x') %><% out(fipsVariant === 'fips' ? '_fips' : '') %>
+  - name: e2e_tests_<% out(executableOsId.replace(/-/g, '_')) %><% out(mVersion === 'stable' ? '' : '_' + mVersion.replace(/[^a-zA-Z0-9]/g, '')) %><% out(fipsVariant === 'fips' ? '_fips' : '') %>
     tags: ["e2e-test"]
     depends_on:
       - name: compile_artifact
@@ -1144,13 +1145,25 @@ buildvariants:
     run_on: rhel80-small
     tasks:
       - name: e2e_tests_linux_x64
+  - name: e2e_rhel90_x64
+    display_name: "RHEL 9.0 x64 (E2E Tests)"
+    run_on: rhel90-small
+    tasks:
+      - name: e2e_tests_linux_x64
   - name: e2e_rhel83_x64
     display_name: "RHEL 8.3 x64 (E2E Tests, FIPS-available OS)"
     run_on: rhel83-fips
     tasks:
       - name: e2e_tests_linux_x64
       - name: e2e_tests_linux_x64_openssl11
       - name: e2e_tests_linux_x64_openssl11_fips
+  - name: e2e_rhel92_x64
+    display_name: "RHEL 9.2 x64 (E2E Tests, FIPS-available OS)"
+    run_on: rhel92-fips
+    tasks:
+      - name: e2e_tests_linux_x64
+      - name: e2e_tests_linux_x64_openssl3
+      - name: e2e_tests_linux_x64_openssl3_fips
   - name: e2e_ubuntu1804_x64
     display_name: "Ubuntu 18.04 x64 (E2E Tests)"
     run_on: ubuntu1804-small
@@ -1162,6 +1175,12 @@ buildvariants:
     tasks:
       - name: e2e_tests_linux_x64
       - name: e2e_tests_linux_x64_openssl11
+  - name: e2e_ubuntu2204_x64
+    display_name: "Ubuntu 22.04 x64 (E2E Tests)"
+    run_on: ubuntu2204-small
+    tasks:
+      - name: e2e_tests_linux_x64
+      - name: e2e_tests_linux_x64_openssl3
   - name: e2e_debian9_x64
     display_name: "Debian 9 x64 (E2E Tests)"
     run_on: debian92-small
@@ -1189,6 +1208,11 @@ buildvariants:
     run_on: amazon2-small
     tasks:
       - name: e2e_tests_linux_x64
+  - name: e2e_amazon2023_x64
+    display_name: "Amazon Linux 2023 x64 (E2E Tests)"
+    run_on: amazon2023.0-small
+    tasks:
+      - name: e2e_tests_linux_x64_700rc10
   - name: e2e_suse12_x64
     display_name: "SLES 12 x64 (E2E Tests)"
     run_on: suse12-sp5-small
@@ -1210,16 +1234,31 @@ buildvariants:
     tasks:
       - name: e2e_tests_linux_arm64
       - name: e2e_tests_linux_arm64_openssl11
+  - name: e2e_ubuntu2204_arm64
+    display_name: "Ubuntu 22.04 arm64 (E2E Tests)"
+    run_on: ubuntu2204-arm64-small
+    tasks:
+      - name: e2e_tests_linux_arm64
   - name: e2e_amazon2_arm64
     display_name: "Amazon Linux 2 arm64 (E2E Tests)"
     run_on: amazon2-arm64-small
     tasks:
       - name: e2e_tests_linux_arm64
+  - name: e2e_amazon2023_arm64
+    display_name: "Amazon Linux 2023 arm64 (E2E Tests)"
+    run_on: amazon2023.0-arm64-small
+    tasks:
+      - name: e2e_tests_linux_arm64_700rc10
   - name: e2e_rhel82_arm64
     display_name: "RHEL 8.2 arm64 (E2E Tests)"
     run_on: rhel82-arm64-small
     tasks:
       - name: e2e_tests_linux_arm64
+  - name: e2e_rhel90_arm64
+    display_name: "RHEL 9.0 arm64 (E2E Tests)"
+    run_on: rhel90-arm64-small
+    tasks:
+      - name: e2e_tests_linux_arm64
   - name: e2e_rhel81_ppc64le
     display_name: "RHEL 8.1 PPC (E2E Tests)"
     run_on: rhel81-power8-small

.evergreen/evergreen.yml.in

@@ -20,7 +20,7 @@ exports.RELEASE_PACKAGE_MATRIX = [
     compileBuildVariant: 'linux_x64_build',
     packages: [
       { name: 'linux-x64', description: 'Linux Tarball 64-bit', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu20.04-tgz'] },
-      { name: 'deb-x64', description: 'Debian / Ubuntu 64-bit', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu18.04-deb', 'ubuntu20.04-deb', 'ubuntu22.04-deb', 'ubuntu22.04-nohome-deb', 'debian9-deb', 'debian10-deb', 'debian11-deb'] },
+      { name: 'deb-x64', description: 'Debian / Ubuntu 64-bit', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu18.04-deb', 'ubuntu20.04-deb', 'ubuntu22.04-deb', 'ubuntu22.04-nohome-deb', 'debian9-deb', 'debian10-deb', 'debian11-deb', 'debian12-deb'] },
       { name: 'rpm-x64', description: 'RHEL / CentOS / Fedora / Suse 64-bit', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['centos7-rpm', 'amazonlinux2-rpm', 'amazonlinux2023-rpm', 'rocky8-rpm', 'rocky9-rpm', 'fedora34-rpm', 'suse12-rpm', 'suse15-rpm', 'amazonlinux1-rpm'] }
     ]
   },
@@ -38,7 +38,7 @@ exports.RELEASE_PACKAGE_MATRIX = [
     compileBuildVariant: 'linux_x64_build_openssl3',
     packages: [
       { name: 'linux-x64-openssl3', description: 'Linux Tarball 64-bit (shared OpenSSL 3)', packageOn: 'linux_package', smokeTestKind: 'none' },
-      { name: 'deb-x64-openssl3', description: 'Debian / Ubuntu 64-bit (shared OpenSSL 3)', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu22.04-deb', 'ubuntu22.04-fips-deb'] },
+      { name: 'deb-x64-openssl3', description: 'Debian / Ubuntu 64-bit (shared OpenSSL 3)', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu22.04-deb', 'ubuntu22.04-fips-deb', 'debian12-deb'] },
       { name: 'rpm-x64-openssl3', description: 'RHEL / CentOS 64-bit (shared OpenSSL 3)', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['rocky8-epel-rpm', 'rocky9-rpm', 'rocky9-fips-rpm', 'amazonlinux2023-rpm'] }
     ]
   },
@@ -47,7 +47,7 @@ exports.RELEASE_PACKAGE_MATRIX = [
     compileBuildVariant: 'linux_arm64_build',
     packages: [
       { name: 'linux-arm64', description: 'Linux Tarball arm64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu20.04-tgz'] },
-      { name: 'deb-arm64', description: 'Debian / Ubuntu arm64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu18.04-deb', 'ubuntu20.04-deb', 'ubuntu22.04-deb', 'debian10-deb', 'debian11-deb'] },
+      { name: 'deb-arm64', description: 'Debian / Ubuntu arm64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu18.04-deb', 'ubuntu20.04-deb', 'ubuntu22.04-deb', 'debian10-deb', 'debian11-deb', 'debian12-deb'] },
       { name: 'rpm-arm64', description: 'RHEL / CentOS arm64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['rocky8-rpm', 'rocky9-rpm', 'fedora34-rpm', 'amazonlinux2-rpm', 'amazonlinux2023-rpm'] }
     ]
   },

config/release-package-matrix.js

@@ -149,7 +149,10 @@ describe('Auth e2e', function () {
           await assertUserAuth();
         });
         it('digestPassword', async function () {
-          if (process.env.MONGOSH_TEST_E2E_FORCE_FIPS) {
+          if (
+            process.env.MONGOSH_TEST_E2E_FORCE_FIPS ||
+            process.env.DISTRO_ID === 'rhel92-fips'
+          ) {
             return this.skip(); // No SCRAM-SHA-1 in FIPS mode
           }
           await shell.executeLine(`use ${dbName}`);
@@ -212,7 +215,10 @@ describe('Auth e2e', function () {
           shell.assertNoErrors();
         });
         it('digestPassword', async function () {
-          if (process.env.MONGOSH_TEST_E2E_FORCE_FIPS) {
+          if (
+            process.env.MONGOSH_TEST_E2E_FORCE_FIPS ||
+            process.env.DISTRO_ID === 'rhel92-fips'
+          ) {
             return this.skip(); // No SCRAM-SHA-1 in FIPS mode
           }
           await shell.executeLine(`use ${dbName}`);
@@ -930,7 +936,10 @@ describe('Auth e2e', function () {
     });
     context('with specific auth mechanisms', function () {
       it('can auth with SCRAM-SHA-1', async function () {
-        if (process.env.MONGOSH_TEST_E2E_FORCE_FIPS) {
+        if (
+          process.env.MONGOSH_TEST_E2E_FORCE_FIPS ||
+          process.env.DISTRO_ID === 'rhel92-fips'
+        ) {
           return this.skip(); // No SCRAM-SHA-1 in FIPS mode
         }
         const connectionString = await testServer.connectionString();
@@ -954,6 +963,29 @@ describe('Auth e2e', function () {
         shell.assertNoErrors();
       });
       it('provides a helpful error message for SCRAM-SHA-1 in FIPS mode', async function () {
+        {
+          // This test is not particularly meaningful if we're using the system OpenSSL installation
+          // and it is not properly configured for FIPS to begin with. This is the case on e.g.
+          // Ubuntu 22.04 in evergreen CI.
+          const preTestShell = TestShell.start({
+            args: [
+              '--quiet',
+              '--nodb',
+              '--tlsFIPSMode',
+              '--eval',
+              'tls.createSecureContext()',
+            ],
+          });
+          if (
+            (await preTestShell.waitForExit()) === 1 &&
+            preTestShell.output.includes(
+              'digital envelope routines::unsupported'
+            )
+          ) {
+            return this.skip();
+          }
+        }
+
         const connectionString = await testServer.connectionString();
         shell = TestShell.start({
           args: [

package-lock.json

@@ -235,9 +235,10 @@ describe('OIDC auth e2e', function () {
     await verifyUser(shell, 'testuser', 'testServer-group');
     await shell.executeLine('db.getMongo().setReadPref("primaryPreferred");');
     await verifyUser(shell, 'testuser', 'testServer-group');
-    await shell.executeLine(
-      `db = connect(${JSON.stringify(cs + '/?authMechanism=MONGODB-OIDC')})`
-    );
+    const cs2 = await testServer.connectionString({
+      authMechanism: 'MONGODB-OIDC',
+    });
+    await shell.executeLine(`db = connect(${JSON.stringify(cs2)})`);
     await verifyUser(shell, 'testuser', 'testServer-group');
     shell.assertNoErrors();
     expect(tokenFetches).to.equal(1);
@@ -255,8 +256,9 @@ describe('OIDC auth e2e', function () {
     await shell.waitForPrompt();
 
     await verifyUser(shell, 'testuser', 'testServer-group');
-    const cs2 =
-      (await testServer2.connectionString()) + '/?authMechanism=MONGODB-OIDC';
+    const cs2 = await testServer2.connectionString({
+      authMechanism: 'MONGODB-OIDC',
+    });
     await shell.executeLine(`db = connect(${JSON.stringify(cs2)})`);
     await verifyUser(shell, 'testuser', 'testServer2-group');
     shell.assertNoErrors();

packages/cli-repl/test/e2e-auth.spec.ts

@@ -12,6 +12,12 @@ describe('snippet integration tests', function () {
   let shell: TestShell;
   let makeTestShell: () => TestShell;
   beforeEach(async function () {
+    if (process.env.DISTRO_ID === 'rhel92-fips') {
+      // TODO: The HTTPS requests we are making for snippet support do not work
+      // with the FIPS configuration on the RHEL 9.2 FIPS-enabled machines.
+      return this.skip();
+    }
+
     makeTestShell = () =>
       TestShell.start({
         args: ['--nodb'],

packages/cli-repl/test/e2e-oidc.spec.ts

@@ -0,0 +1,12 @@
+FROM debian:12
+
+ARG artifact_url=""
+ADD ${artifact_url} /tmp
+ADD node_modules /usr/share/mongodb-crypt-library-version/node_modules
+RUN apt-get update
+RUN apt-get install -y man-db
+RUN apt-get install -y /tmp/*mongosh*.deb
+RUN /usr/bin/mongosh --build-info
+RUN env MONGOSH_RUN_NODE_SCRIPT=1 mongosh /usr/share/mongodb-crypt-library-version/node_modules/.bin/mongodb-crypt-library-version /usr/lib/mongosh_crypt_v1.so | grep -Eq '^mongo_(crypt|csfle)_v1-'
+RUN man mongosh | grep -q tlsAllowInvalidCertificates
+ENTRYPOINT [ "mongosh" ]