feat(smoke-tests): test auto-updating from the version of Compass that was just packaged COMPASS-8536

.evergreen/buildvariants-and-tasks.in.yml

@@ -469,6 +469,7 @@ tasks:
       - func: bootstrap
         vars:
           scope: 'compass-e2e-tests'
+      - func: get-compass-mongodb-com
       - func: smoketest-packaged-app
         vars:
           mongodb_version: latest-enterprise

.evergreen/buildvariants-and-tasks.yml

@@ -495,6 +495,7 @@ tasks:
       - func: bootstrap
         vars:
           scope: compass-e2e-tests
+      - func: get-compass-mongodb-com
       - func: smoketest-packaged-app
         vars:
           mongodb_version: latest-enterprise

.evergreen/functions.yml

@@ -648,6 +648,36 @@ functions:
 
           npm run --unsafe-perm --workspace compass-e2e-tests test-packaged-ci
 
+  get-compass-mongodb-com:
+    - command: github.generate_token
+      params:
+        owner: 10gen
+        repo: compass-mongodb-com
+        expansion_name: generated_token
+        permissions:  # optional
+          contents: read
+    - command: shell.exec
+      type: setup
+      params:
+        working_dir: src
+        shell: bash
+        env:
+          <<: *compass-env
+        script: |
+          set -e
+          eval $(.evergreen/print-compass-env.sh)
+          git clone https://x-access-token:${generated_token}@github.com/10gen/compass-mongodb-com.git ../compass-mongodb-com
+          cd ../compass-mongodb-com
+          echo "Using node version:";
+          node -v;
+          echo "Using npm version:";
+          npm -v;
+          echo "Using gcc version:"
+          gcc --version;
+          echo "Using g++ version:"
+          g++ --version;
+          npm ci --engine-strict=false
+
   smoketest-packaged-app:
     - command: shell.exec
       # Fail the task if it's idle for 10 mins
@@ -676,8 +706,8 @@ functions:
           if [[ "$IS_OSX" == "true" ]]; then
             echo "Disabling clipboard usage in e2e tests (TODO: https://jira.mongodb.org/browse/BUILD-14780)"
             export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true"
-            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg
             npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip
+            npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg
           fi
 
           #if [[ "$IS_UBUNTU" == "true" ]]; then

.evergreen/preinstall.sh

@@ -13,9 +13,9 @@ echo "APPDATA: $APPDATA"
 echo "PATH: $PATH"
 
 # these are super useful if you want to run the smoke tests locally
-echo "DEV_VERSION_IDENTIFIER: $DEV_VERSION_IDENTIFIER"
-echo "EVERGREEN_BUCKET_NAME: $EVERGREEN_BUCKET_NAME"
-echo "EVERGREEN_BUCKET_KEY_PREFIX: $EVERGREEN_BUCKET_KEY_PREFIX"
+echo "export DEV_VERSION_IDENTIFIER=$DEV_VERSION_IDENTIFIER"
+echo "export EVERGREEN_BUCKET_KEY_PREFIX=$EVERGREEN_BUCKET_KEY_PREFIX"
+echo "export EVERGREEN_BUCKET_NAME=$EVERGREEN_BUCKET_NAME"
 
 echo "IS_OSX: $IS_OSX"
 echo "IS_LINUX: $IS_LINUX"

packages/compass-e2e-tests/helpers/selectors.ts

@@ -1380,3 +1380,12 @@ export const GlobalWrites = {
   SampleFindingDocuments: '[data-testid="sample-finding-documents"]',
   SampleInsertingDocuments: '[data-testid="sample-inserting-documents"]',
 };
+
+// Auto-update toasts
+export const AutoUpdateToast = '[data-testid="toast-compass-update"]';
+export const AutoUpdateRestartButton =
+  '[data-testid="auto-update-restart-button"]';
+export const AutoUpdateDownloadLink =
+  '[data-testid="auto-update-download-link"]';
+export const AutoUpdateReleaseNotesLink =
+  '[data-testid="auto-update-release-notes-link"]';

packages/compass-e2e-tests/tests/auto-update.test.ts

@@ -0,0 +1,98 @@
+import fs from 'fs';
+import path from 'path';
+import { expect } from 'chai';
+import {
+  init,
+  cleanup,
+  Selectors,
+  screenshotPathName,
+} from '../helpers/compass';
+import { LOG_PATH } from '../helpers/test-runner-paths';
+
+function wait(ms: number) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+describe('Auto-update', function () {
+  it('auto-update from', async function () {
+    if (process.env.TEST_NAME !== 'AUTO_UPDATE_FROM') {
+      // we don't want this test to execute along with all the others under
+      // normal circumstances because it is destructive - it overwrites Compass
+      // itself
+      this.skip();
+    }
+
+    // run the app and wait for it to auto-update
+    console.log('starting compass the first time');
+    const compass = await init('auto-update from', { firstRun: true });
+    const { browser } = compass;
+    try {
+      await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
+
+      if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
+        const restartButton = browser.$(Selectors.AutoUpdateRestartButton);
+        await restartButton.waitForDisplayed();
+
+        // We could click the restart button to apply the update and restart the
+        // app, but restarting the app confuses webdriverio or at least our test
+        // helpers. So we're going to just restart the app manually.
+        await browser.pause(1000);
+      } else {
+        // When auto-update is not supported the toast contains a link to
+        // download
+        const linkElement = browser.$(Selectors.AutoUpdateDownloadLink);
+        await linkElement.waitForDisplayed();
+        expect(await linkElement.getAttribute('href')).to.equal(
+          'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product'
+        );
+      }
+    } finally {
+      await browser.screenshot(screenshotPathName('auto-update-from'));
+      await cleanup(compass);
+
+      if (process.platform === 'darwin' && process.env.HOME) {
+        console.log('copying ShipIt dir if it exits');
+        const shipitDir = path.resolve(
+          process.env.HOME,
+          'Library',
+          'Caches',
+          'com.mongodb.compass.dev.ShipIt'
+        );
+
+        if (fs.existsSync(shipitDir)) {
+          console.log(`copying ${shipitDir}`);
+          fs.cpSync(shipitDir, `${LOG_PATH}/ShipIt`, { recursive: true });
+        } else {
+          console.log(`${shipitDir} does not exist`);
+        }
+      }
+    }
+
+    if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
+      console.log(
+        'pause to make sure the app properly exited before starting again'
+      );
+      await wait(60_000);
+
+      console.log('starting compass a second time');
+      // run the app again and check that the version changed
+      const compass = await init('auto-update from restart', {
+        firstRun: false,
+      });
+      const { browser } = compass;
+      try {
+        await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
+        await browser
+          .$(Selectors.AutoUpdateReleaseNotesLink)
+          .waitForDisplayed();
+      } finally {
+        await browser.screenshot(
+          screenshotPathName('auto-update-from-restart')
+        );
+        await cleanup(compass);
+      }
+    }
+  });
+});

packages/compass-smoke-tests/package.json

@@ -34,11 +34,13 @@
     "@mongodb-js/eslint-config-compass": "^1.2.5",
     "@mongodb-js/prettier-config-compass": "^1.1.5",
     "@mongodb-js/tsconfig-compass": "^1.1.5",
+    "cross-spawn": "^7.0.5",
     "depcheck": "^1.4.1",
     "eslint": "^7.25.0",
     "hadron-build": "^25.6.5",
     "lodash": "^4.17.21",
     "prettier": "^2.7.1",
+    "tree-kill": "^1.2.2",
     "typescript": "^5.0.4",
     "yargs": "^17.7.2"
   }

packages/compass-smoke-tests/src/cli.ts

@@ -3,6 +3,8 @@ import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import path from 'node:path';
 
+import crossSpawn from 'cross-spawn';
+import kill from 'tree-kill';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 import { pick } from 'lodash';
@@ -179,7 +181,9 @@ async function run() {
     ])
   );
 
-  const { kind, filepath, appName } = await getTestSubject(context);
+  const { kind, appName, filepath, autoUpdatable } = await getTestSubject(
+    context
+  );
   const install = getInstaller(kind);
 
   try {
@@ -190,7 +194,21 @@ async function run() {
     });
 
     try {
-      runTest({ appName, appPath });
+      if (context.platform === 'darwin' && process.env.CI) {
+        // Auto-update does not work on mac in CI at the moment. So in that case
+        // we just run the E2E tests to make sure the app at least starts up.
+        runE2ETest({
+          appName,
+          appPath,
+        });
+      } else {
+        runUpdateTest({
+          appName,
+          appPath,
+          autoUpdatable,
+          testName: 'AUTO_UPDATE_FROM',
+        });
+      }
     } finally {
       await uninstall();
     }
@@ -200,12 +218,53 @@ async function run() {
   }
 }
 
-type RunTestOptions = {
+type AutoUpdateServerOptions = {
+  port: number;
+  allowDowngrades?: boolean;
+};
+
+function startAutoUpdateServer({
+  port,
+  allowDowngrades,
+}: AutoUpdateServerOptions) {
+  const env: Record<string, string> = {
+    ...process.env,
+    PORT: port.toString(),
+  };
+  if (allowDowngrades) {
+    env.UPDATE_CHECKER_ALLOW_DOWNGRADES = 'true';
+  }
+
+  // a git repo that is not published to npm that evergreen clones for us in CI
+  // next to the Compass code
+  const cwd = path.join(
+    __dirname,
+    '..',
+    '..',
+    '..',
+    '..',
+    'compass-mongodb-com'
+  );
+
+  if (!fs.existsSync(cwd)) {
+    throw new Error(`compass-mongodb-com does not exist: ${cwd}`);
+  }
+
+  console.log('Starting auto-update server', cwd);
+  return crossSpawn.spawn('npm', ['run', 'start'], {
+    env,
+    cwd,
+    stdio: 'inherit',
+    shell: true,
+  });
+}
+
+type RunE2ETestOptions = {
   appName: string;
   appPath: string;
 };
 
-function runTest({ appName, appPath }: RunTestOptions) {
+function runE2ETest({ appName, appPath }: RunE2ETestOptions) {
   execute(
     'npm',
     [
@@ -229,6 +288,58 @@ function runTest({ appName, appPath }: RunTestOptions) {
   );
 }
 
+type RunUpdateTestOptions = {
+  appName: string;
+  appPath: string;
+  autoUpdatable?: boolean;
+  testName: string;
+};
+
+function runUpdateTest({
+  appName,
+  appPath,
+  autoUpdatable,
+  testName,
+}: RunUpdateTestOptions) {
+  const server = startAutoUpdateServer({
+    allowDowngrades: true,
+    port: 8080,
+  });
+  try {
+    execute(
+      'npm',
+      [
+        'run',
+        '--unsafe-perm',
+        'test-packaged',
+        '--workspace',
+        'compass-e2e-tests',
+        '--',
+        '--test-filter=auto-update',
+      ],
+      {
+        // We need to use a shell to get environment variables setup correctly
+        shell: true,
+        env: {
+          ...process.env,
+          HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE: 'http://localhost:8080',
+          AUTO_UPDATE_UPDATABLE: (!!autoUpdatable).toString(),
+          TEST_NAME: testName,
+          COMPASS_APP_NAME: appName,
+          COMPASS_APP_PATH: appPath,
+        },
+      }
+    );
+  } finally {
+    if (server.pid) {
+      console.log('Stopping auto-update server');
+      kill(server.pid, 'SIGINT');
+    } else {
+      console.log('cannnot stop auto-update server because no pid');
+    }
+  }
+}
+
 run()
   .then(function () {
     console.log('done');

packages/compass-smoke-tests/src/execute.ts

@@ -6,6 +6,10 @@ export function execute(
   args: string[],
   options?: SpawnOptions
 ) {
+  // print the command so that when it outputs to stdout we can see where it
+  // comes from
+  console.log(command, args);
+
   const { status, signal } = spawnSync(command, args, {
     stdio: 'inherit',
     ...options,