control which tests run with a command line parameter

Author: lerouxb

.evergreen/functions.yml

@@ -700,14 +700,17 @@ functions:
           if [[ "$IS_WINDOWS" == "true" ]]; then
             # TODO: windows_setup
             # TODO: windows_msi
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests=auto-update-from
           fi
 
           if [[ "$IS_OSX" == "true" ]]; then
             echo "Disabling clipboard usage in e2e tests (TODO: https://jira.mongodb.org/browse/BUILD-14780)"
             export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true"
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg
+            # NOTE: We're also skipping auto-update of the macOS app in CI
+            # because it doesn't work. Running a different test to make sure it
+            # can install and run successfully at least.
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip --tests=time-to-first-query
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg --tests=time-to-first-query
           fi
 
           #if [[ "$IS_UBUNTU" == "true" ]]; then

packages/compass-e2e-tests/tests/auto-update.test.ts

@@ -14,7 +14,7 @@ function wait(ms: number) {
 
 describe('Auto-update', function () {
   it('auto-update from', async function () {
-    if (process.env.TEST_NAME !== 'AUTO_UPDATE_FROM') {
+    if (process.env.TEST_NAME !== 'auto-update-from') {
       // we don't want this test to execute along with all the others under
       // normal circumstances because it is destructive - it overwrites Compass
       // itself

packages/compass-smoke-tests/src/cli.ts

@@ -26,6 +26,8 @@ import { installWindowsZIP } from './installers/windows-zip';
 const SUPPORTED_PLATFORMS = ['win32', 'darwin', 'linux'] as const;
 const SUPPORTED_ARCHS = ['x64', 'arm64'] as const;
 
+const SUPPORTED_TESTS = ['time-to-first-query', 'auto-update-from'] as const;
+
 function isSupportedPlatform(
   value: unknown
 ): value is typeof SUPPORTED_PLATFORMS[number] {
@@ -100,7 +102,14 @@ const argv = yargs(hideBin(process.argv))
   .option('localPackage', {
     type: 'boolean',
     description: 'Use the local package instead of downloading',
-  });
+  })
+  .option('tests', {
+    type: 'string',
+    choices: SUPPORTED_TESTS,
+    description: 'Which tests to run',
+  })
+  .array('tests')
+  .default('tests', []);
 
 type TestSubject = PackageDetails & {
   filepath: string;
@@ -178,6 +187,7 @@ async function run() {
       'platform',
       'arch',
       'package',
+      'tests',
     ])
   );
 
@@ -187,30 +197,37 @@ async function run() {
   const install = getInstaller(kind);
 
   try {
-    const { appPath, uninstall } = install({
-      appName,
-      filepath,
-      destinationPath: context.sandboxPath,
-    });
+    if (context.tests.length === 0) {
+      console.log('Warning: not performing any tests. Pass --tests.');
+    }
+
+    for (const testName of context.tests) {
+      const { appPath, uninstall } = install({
+        appName,
+        filepath,
+        destinationPath: context.sandboxPath,
+      });
 
-    try {
-      if (context.platform === 'darwin' && process.env.CI) {
-        // Auto-update does not work on mac in CI at the moment. So in that case
-        // we just run the E2E tests to make sure the app at least starts up.
-        runTimeToFirstQuery({
-          appName,
-          appPath,
-        });
-      } else {
-        runUpdateTest({
-          appName,
-          appPath,
-          autoUpdatable,
-          testName: 'AUTO_UPDATE_FROM',
-        });
+      try {
+        if (testName === 'time-to-first-query') {
+          // Auto-update does not work on mac in CI at the moment. So in that case
+          // we just run the E2E tests to make sure the app at least starts up.
+          runTimeToFirstQuery({
+            appName,
+            appPath,
+          });
+        }
+        if (testName === 'auto-update-from') {
+          runUpdateTest({
+            appName,
+            appPath,
+            autoUpdatable,
+            testName,
+          });
+        }
+      } finally {
+        await uninstall();
       }
-    } finally {
-      await uninstall();
     }
   } finally {
     console.log('Cleaning up sandbox');

packages/compass-smoke-tests/src/context.ts

@@ -9,4 +9,5 @@ export type SmokeTestsContext = {
   forceDownload?: boolean;
   localPackage?: boolean;
   sandboxPath: string;
+  tests: string[];
 };