diff --git a/.github/workflows/test-installers.yml b/.github/workflows/test-installers.yml
index 234b81351f8..8229aaafb1a 100644
--- a/.github/workflows/test-installers.yml
+++ b/.github/workflows/test-installers.yml
@@ -21,8 +21,111 @@ on:
 
 jobs:
   test:
-    name: Dummy action
-    runs-on: ubuntu-latest
+    name: ${{ matrix.package }} test ${{ matrix.test }} (${{ matrix.hadron-distribution }})
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - time-to-first-query
+          - auto-update-from
+          - auto-update-to
+        package:
+          - osx_dmg
+          - osx_zip
+          - windows_zip
+          - windows_msi
+          - windows_setup
+          - linux_deb
+          # TODO: Re-enable (see https://github.com/mongodb-js/compass/actions/runs/13281152689/job/37079619474)
+          # - linux_tar
+          # TODO: Enable (needs a docker container)
+          # - linux_rpm
+        hadron-distribution:
+          - compass
+          - compass-readonly
+          - compass-isolated
+        include:
+          # MacOS
+          - package: osx_dmg
+            runs-on: macos-13
+            arch: x64
+            hadron-platform: darwin
+          - package: osx_zip
+            runs-on: macos-latest
+            arch: arm64
+            hadron-platform: darwin
+
+          # Windows
+          - package: windows_zip
+            runs-on: windows-latest
+            arch: x64
+            hadron-platform: windows
+          - package: windows_msi
+            runs-on: windows-latest
+            arch: x64
+            hadron-platform: windows
+          - package: windows_setup
+            runs-on: windows-latest
+            arch: x64
+            hadron-platform: windows
+
+          # Linux
+          - package: linux_deb
+            runs-on: ubuntu-latest
+            arch: x64
+            hadron-platform: linux
+          # - package: linux_tar
+          #   runs-on: ubuntu-latest
+          #   arch: x64
+          #   hadron-platform: linux
+          # - package: linux_rpm
+          #   runs-on: ubuntu-latest
+          #   arch: x64
+          #   hadron-platform: linux
+
+          # Install the update server for auto-update tests
+          - test: auto-update-from
+            install-update-server: true
+          - test: auto-update-to
+            install-update-server: true
+
+        exclude:
+          # TODO: See https://github.com/mongodb-js/compass/actions/runs/13281152689/job/37079620322
+          - package: osx_dmg
+            test: auto-update-from
+          # TODO: See https://github.com/mongodb-js/compass/actions/runs/13281152689/job/37079621700
+          - package: windows_setup
+            test: auto-update-from
+          # Skip auto-update tests for isolated and readonly distributions for now
+          # This is not supported by the update server we're using in the test harness
+          - hadron-distribution: compass-isolated
+            test: auto-update-from
+          - hadron-distribution: compass-isolated
+            test: auto-update-to
+          - hadron-distribution: compass-readonly
+            test: auto-update-from
+          - hadron-distribution: compass-readonly
+            test: auto-update-to
+          # Temporary skip failing auto-update-to tests
+          - test: auto-update-to
+            package: osx_dmg
+          - test: auto-update-to
+            package: windows_zip
+          - test: auto-update-to
+            package: windows_msi
+          - test: auto-update-to
+            package: windows_setup
+          - test: auto-update-to
+            package: linux_deb
+
+          # Skip time-to-first-query tests for readonly because it doesn't have editable documents
+          # See https://github.com/mongodb-js/compass/actions/runs/13286945911/job/37097791601
+          - hadron-distribution: compass-readonly
+            test: time-to-first-query
+
+    runs-on: ${{ matrix.runs-on }}
+    env:
+      DEBUG: compass:smoketests:*
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -31,6 +134,47 @@ jobs:
         with:
           node-version: 20
           cache: "npm"
-      - name: Install dependencies
+      - name: Cache downloads
+        uses: actions/cache@v4
+        with:
+          key: smoke-tests-downloads-${{ inputs.version }}-${{ runner.os }}-${{ runner.arch }}-${{ matrix.package }}
+          path: packages/compass-smoke-tests/.downloads
+      - name: Install dependencies and build packages
         run: npm ci
-        
\ No newline at end of file
+      - name: Bootstrap packages
+        run: npx lerna run bootstrap --scope @mongodb-js/compass-smoke-tests --include-dependencies
+
+      - name: Create GitHub App Token
+        if: matrix.install-update-server
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.DEVTOOLS_BOT_APP_ID }}
+          private-key: ${{ secrets.DEVTOOLS_BOT_PRIVATE_KEY }}
+          owner: 10gen
+          repositories: compass-mongodb-com
+      - name: Checkout Compass Update server
+        if: matrix.install-update-server
+        uses: actions/checkout@v4
+        with:
+          repository: 10gen/compass-mongodb-com
+          token: ${{ steps.app-token.outputs.token }}
+          # Make sure the value of GITHUB_TOKEN will not be persisted in repo's config
+          persist-credentials: false
+          path: 'compass-mongodb-com'
+      - name: Install Compass Update server
+        if: matrix.install-update-server
+        run: npm install --no-save --workspace packages/compass-smoke-tests ${{ github.workspace }}/compass-mongodb-com
+
+      - name: Run tests
+        env:
+          EVERGREEN_BUCKET_NAME: ${{ inputs.bucket_name }}
+          EVERGREEN_BUCKET_KEY_PREFIX: ${{ inputs.bucket_key_prefix }}
+          DEV_VERSION_IDENTIFIER: ${{ inputs.version }}
+          HADRON_DISTRIBUTION: ${{ matrix.hadron-distribution }}
+          PLATFORM: ${{ matrix.hadron-platform }}
+          ARCH: ${{ matrix.arch }}
+          # Exposing token to prevent update server from being rate limited
+          GITHUB_TOKEN: ${{ github.token }}
+        working-directory: packages/compass-smoke-tests
+        run: npm start -- --package ${{ matrix.package }} --tests ${{ matrix.test }}
diff --git a/package-lock.json b/package-lock.json
index 83076ea103c..a65b31ab518 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -46995,6 +46995,7 @@
         "@mongodb-js/prettier-config-compass": "^1.2.1",
         "@mongodb-js/tsconfig-compass": "^1.2.1",
         "@types/node": "^20",
+        "compass-e2e-tests": "^1.29.0",
         "debug": "^4.3.4",
         "depcheck": "^1.4.1",
         "eslint": "^7.25.0",
@@ -58706,6 +58707,7 @@
         "@mongodb-js/prettier-config-compass": "^1.2.1",
         "@mongodb-js/tsconfig-compass": "^1.2.1",
         "@types/node": "^20",
+        "compass-e2e-tests": "^1.29.0",
         "debug": "^4.3.4",
         "depcheck": "^1.4.1",
         "eslint": "^7.25.0",
diff --git a/packages/compass-smoke-tests/.depcheckrc b/packages/compass-smoke-tests/.depcheckrc
index 8630a3aa6b7..0fd93a3bd90 100644
--- a/packages/compass-smoke-tests/.depcheckrc
+++ b/packages/compass-smoke-tests/.depcheckrc
@@ -2,3 +2,5 @@ ignores:
  - '@mongodb-js/prettier-config-compass'
  - '@mongodb-js/tsconfig-compass'
  - compass-mongodb-com
+ # Calling into the e2e tests via child processes
+ - compass-e2e-tests
diff --git a/packages/compass-smoke-tests/package.json b/packages/compass-smoke-tests/package.json
index d681bee6cdc..f2b49a4074a 100644
--- a/packages/compass-smoke-tests/package.json
+++ b/packages/compass-smoke-tests/package.json
@@ -34,6 +34,7 @@
     "@mongodb-js/eslint-config-compass": "^1.3.1",
     "@mongodb-js/prettier-config-compass": "^1.2.1",
     "@mongodb-js/tsconfig-compass": "^1.2.1",
+    "compass-e2e-tests": "^1.29.0",
     "depcheck": "^1.4.1",
     "debug": "^4.3.4",
     "eslint": "^7.25.0",
diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts
index da495021da4..02a80808981 100755
--- a/packages/compass-smoke-tests/src/cli.ts
+++ b/packages/compass-smoke-tests/src/cli.ts
@@ -9,6 +9,7 @@ import { SUPPORTED_PACKAGES } from './packages';
 import { testTimeToFirstQuery } from './tests/time-to-first-query';
 import { testAutoUpdateFrom } from './tests/auto-update-from';
 import { testAutoUpdateTo } from './tests/auto-update-to';
+import { deleteSandboxesDirectory } from './directories';
 
 const debug = createDebug('compass:smoketests');
 
@@ -92,7 +93,7 @@ const argv = yargs(hideBin(process.argv))
   })
   .option('skipCleanup', {
     type: 'boolean',
-    description: 'Do not delete the sandbox after a run',
+    description: 'Do not delete the sandboxes after a run',
     default: false,
   })
   .option('tests', {
@@ -106,6 +107,27 @@ const argv = yargs(hideBin(process.argv))
 async function run() {
   const context: SmokeTestsContext = argv.parseSync();
 
+  function cleanupMaybe() {
+    if (context.skipCleanup) {
+      console.log('Skipped cleanup of sandboxes');
+    } else {
+      debug('Cleaning up sandboxes');
+      try {
+        deleteSandboxesDirectory();
+      } catch (err) {
+        if (err instanceof Error) {
+          console.warn(`Failed cleaning sandboxes: ${err.message}`);
+        } else {
+          throw err;
+        }
+      }
+    }
+  }
+
+  process.once('SIGTERM', cleanupMaybe);
+  process.once('SIGINT', cleanupMaybe);
+  process.once('exit', cleanupMaybe);
+
   debug(`Running tests`);
 
   debug(
diff --git a/packages/compass-smoke-tests/src/directories.ts b/packages/compass-smoke-tests/src/directories.ts
index 26e9bffbbbc..63a619dadfd 100644
--- a/packages/compass-smoke-tests/src/directories.ts
+++ b/packages/compass-smoke-tests/src/directories.ts
@@ -3,12 +3,21 @@ import crypto from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 
+const SANDBOXES_PATH = path.resolve(__dirname, '../.sandboxes');
+
+export function deleteSandboxesDirectory() {
+  fs.rmSync(SANDBOXES_PATH, { recursive: true, force: true });
+}
+
+export function sandboxesDirectoryExists() {
+  return fs.existsSync(SANDBOXES_PATH);
+}
+
 function ensureSandboxesDirectory() {
-  const sandboxesPath = path.resolve(__dirname, '../.sandboxes');
-  if (!fs.existsSync(sandboxesPath)) {
-    fs.mkdirSync(sandboxesPath, { recursive: true });
+  if (!sandboxesDirectoryExists()) {
+    fs.mkdirSync(SANDBOXES_PATH, { recursive: true });
   }
-  return sandboxesPath;
+  return SANDBOXES_PATH;
 }
 
 export function createSandbox() {
diff --git a/packages/compass-smoke-tests/src/downloads.ts b/packages/compass-smoke-tests/src/downloads.ts
index 5c1072dfe7c..edd689fc19c 100644
--- a/packages/compass-smoke-tests/src/downloads.ts
+++ b/packages/compass-smoke-tests/src/downloads.ts
@@ -19,6 +19,7 @@ export async function downloadFile({
   targetFilename,
   clearCache,
 }: DownloadFileOptions): Promise<string> {
+  debug('Fetching', { url });
   const response = await fetch(url);
 
   assert(
@@ -40,7 +41,7 @@ export async function downloadFile({
     if (clearCache) {
       fs.rmSync(cacheDirectoryPath, { recursive: true, force: true });
     } else {
-      debug('Skipped downloading', url, '(cache existed)');
+      debug('Skipped downloading (cache existed)', { url });
       return outputPath;
     }
   }
@@ -51,11 +52,11 @@ export async function downloadFile({
 
   // Write the response to file
   assert(response.body, 'Expected a response body');
-  debug('Downloading', url);
+  debug('Downloading', { url, outputPath });
   await stream.promises.pipeline(
     response.body,
     fs.createWriteStream(outputPath)
   );
-
+  debug('Download completed', { url, outputPath });
   return outputPath;
 }
diff --git a/packages/compass-smoke-tests/src/installers/linux-deb.ts b/packages/compass-smoke-tests/src/installers/linux-deb.ts
index 604860e399a..8b55c17da1c 100644
--- a/packages/compass-smoke-tests/src/installers/linux-deb.ts
+++ b/packages/compass-smoke-tests/src/installers/linux-deb.ts
@@ -1,11 +1,14 @@
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import path from 'node:path';
+import createDebug from 'debug';
 
 import type { InstalledAppInfo, InstallablePackage } from './types';
 import { execute } from '../execute';
 import * as apt from './apt';
 
+const debug = createDebug('compass:smoketests:linux-deb');
+
 export function installLinuxDeb({
   kind,
   filepath,
@@ -18,6 +21,7 @@ export function installLinuxDeb({
   const appPath = path.resolve(installPath, appName);
 
   function uninstall() {
+    debug('Uninstalling %s', filepath);
     execute('sudo', ['apt', 'remove', '--yes', '--purge', packageName]);
   }
 
diff --git a/packages/compass-smoke-tests/src/installers/linux-rpm.ts b/packages/compass-smoke-tests/src/installers/linux-rpm.ts
index 97346607f9d..c34f0a2670e 100644
--- a/packages/compass-smoke-tests/src/installers/linux-rpm.ts
+++ b/packages/compass-smoke-tests/src/installers/linux-rpm.ts
@@ -2,9 +2,13 @@ import assert from 'node:assert/strict';
 import cp from 'node:child_process';
 import path from 'node:path';
 import fs from 'node:fs';
+import createDebug from 'debug';
 
 import type { InstalledAppInfo, InstallablePackage } from './types';
 import { execute } from '../execute';
+
+const debug = createDebug('compass:smoketests:linux-rpm');
+
 /**
  * Call dnf to get the package name
  */
@@ -48,6 +52,7 @@ export function installLinuxRpm({
   const appPath = path.resolve(installPath, appName);
 
   function uninstall() {
+    debug('Uninstalling %s', filepath);
     execute('sudo', ['dnf', 'remove', '-y', packageName]);
   }
 
diff --git a/packages/compass-smoke-tests/src/installers/windows-msi.ts b/packages/compass-smoke-tests/src/installers/windows-msi.ts
index 8838091243d..30135c2317f 100644
--- a/packages/compass-smoke-tests/src/installers/windows-msi.ts
+++ b/packages/compass-smoke-tests/src/installers/windows-msi.ts
@@ -21,6 +21,7 @@ export function installWindowsMSI({
   const appPath = path.resolve(installDirectory, `${appName}.exe`);
 
   function uninstall() {
+    debug('Uninstalling %s', filepath);
     execute('msiexec', ['/uninstall', filepath, '/passive']);
   }
 
diff --git a/packages/compass-smoke-tests/src/installers/windows-setup.ts b/packages/compass-smoke-tests/src/installers/windows-setup.ts
index 8c8dd02cc47..d6332b3716c 100644
--- a/packages/compass-smoke-tests/src/installers/windows-setup.ts
+++ b/packages/compass-smoke-tests/src/installers/windows-setup.ts
@@ -35,6 +35,7 @@ export function installWindowsSetup({
   }
 
   function uninstall({ expectMissing = false }: UninstallOptions = {}) {
+    debug('Uninstalling %s', filepath);
     const entry = queryRegistry();
     if (entry) {
       const {
diff --git a/packages/compass-smoke-tests/src/tests/auto-update-from.ts b/packages/compass-smoke-tests/src/tests/auto-update-from.ts
index 4755ea957cb..e58420dd744 100644
--- a/packages/compass-smoke-tests/src/tests/auto-update-from.ts
+++ b/packages/compass-smoke-tests/src/tests/auto-update-from.ts
@@ -1,12 +1,13 @@
 import assert from 'node:assert/strict';
-import fs from 'node:fs';
+
 import createDebug from 'debug';
+
 import { type SmokeTestsContext } from '../context';
 import { getInstaller } from '../installers';
 import { createSandbox } from '../directories';
 import { getTestSubject } from '../test-subject';
 import { executeAsync } from '../execute';
-import { startAutoUpdateServer } from './update-server';
+import { startAutoUpdateServer, stopAutoUpdateServer } from './update-server';
 
 const debug = createDebug('compass:smoketests:auto-update-from');
 
@@ -19,66 +20,60 @@ export async function testAutoUpdateFrom(context: SmokeTestsContext) {
 
   const { kind, filepath, autoUpdatable } = subject;
 
-  try {
-    const install = getInstaller(kind);
+  const install = getInstaller(kind);
 
-    const { appName, appPath, uninstall } = install({
-      ...subject,
-      filepath,
-      sandboxPath,
-    });
+  debug('Installing');
+  const { appName, appPath, uninstall } = install({
+    ...subject,
+    filepath,
+    sandboxPath,
+  });
 
-    try {
-      process.env.PORT = '0'; // dynamic port
-      process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES = 'true';
+  try {
+    process.env.PORT = '0'; // dynamic port
+    process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES = 'true';
 
-      const server = await startAutoUpdateServer();
+    const server = await startAutoUpdateServer();
 
-      const address = server.address();
-      assert(typeof address === 'object' && address !== null);
-      const port = address.port;
-      const HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE = `http://localhost:${port}`;
+    const address = server.address();
+    assert(typeof address === 'object' && address !== null);
+    const port = address.port;
+    const HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE = `http://localhost:${port}`;
 
-      try {
-        // must be async because the update server is running in the same process
-        await executeAsync(
-          'npm',
-          [
-            'run',
-            '--unsafe-perm',
-            'test-packaged',
-            '--workspace',
-            'compass-e2e-tests',
-            '--',
-            '--test-filter=auto-update',
-          ],
-          {
-            // We need to use a shell to get environment variables setup correctly
-            shell: true,
-            env: {
-              ...process.env,
-              HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE,
-              AUTO_UPDATE_UPDATABLE: (!!autoUpdatable).toString(),
-              TEST_NAME: 'auto-update-from',
-              COMPASS_APP_NAME: appName,
-              COMPASS_APP_PATH: appPath,
-            },
-          }
-        );
-      } finally {
-        debug('Stopping auto-update server');
-        server.close();
-        delete process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES;
-      }
+    try {
+      debug('Executing');
+      // must be async because the update server is running in the same process
+      await executeAsync(
+        'npm',
+        [
+          'run',
+          '--unsafe-perm',
+          'test-packaged',
+          '--workspace',
+          'compass-e2e-tests',
+          '--',
+          '--test-filter=auto-update',
+        ],
+        {
+          // We need to use a shell to get environment variables setup correctly
+          shell: true,
+          env: {
+            ...process.env,
+            HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE,
+            AUTO_UPDATE_UPDATABLE: (!!autoUpdatable).toString(),
+            TEST_NAME: 'auto-update-from',
+            COMPASS_APP_NAME: appName,
+            COMPASS_APP_PATH: appPath,
+          },
+        }
+      );
     } finally {
-      await uninstall();
+      debug('Stopping auto-update server');
+      await stopAutoUpdateServer(server);
+      delete process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES;
     }
   } finally {
-    if (context.skipCleanup) {
-      debug(`Skipped cleaning up sandbox: ${sandboxPath}`);
-    } else {
-      debug(`Cleaning up sandbox: ${sandboxPath}`);
-      fs.rmSync(sandboxPath, { recursive: true });
-    }
+    debug('Uninstalling');
+    await uninstall();
   }
 }
diff --git a/packages/compass-smoke-tests/src/tests/auto-update-to.ts b/packages/compass-smoke-tests/src/tests/auto-update-to.ts
index bce1140c2da..bedfb378ba5 100644
--- a/packages/compass-smoke-tests/src/tests/auto-update-to.ts
+++ b/packages/compass-smoke-tests/src/tests/auto-update-to.ts
@@ -1,5 +1,4 @@
 import assert from 'node:assert/strict';
-import fs from 'node:fs';
 import createDebug from 'debug';
 import { type SmokeTestsContext } from '../context';
 import { getInstaller } from '../installers';
@@ -7,7 +6,7 @@ import { createSandbox } from '../directories';
 import { getTestSubjectDetails } from '../test-subject';
 import { executeAsync } from '../execute';
 import { getLatestRelease, getLatestReleaseKindByKind } from '../releases';
-import { startAutoUpdateServer } from './update-server';
+import { startAutoUpdateServer, stopAutoUpdateServer } from './update-server';
 
 const debug = createDebug('compass:smoketests:auto-update-to');
 
@@ -20,91 +19,95 @@ export async function testAutoUpdateTo(context: SmokeTestsContext) {
   const sandboxPath = createSandbox();
   const subject = getTestSubjectDetails({ ...context, sandboxPath });
   const {
-    kind,
-    autoUpdatable,
     buildInfo: { channel, version },
   } = subject;
 
-  try {
-    const install = getInstaller(getLatestReleaseKindByKind(kind));
-    const filepath = await getLatestRelease(
-      channel,
-      context.arch,
-      kind,
-      context.forceDownload
-    );
+  // Derive the kind of package needed to install the latest release
+  const latestApp = getTestSubjectDetails({
+    ...context,
+    sandboxPath,
+    package: getLatestReleaseKindByKind(subject.kind),
+  });
 
-    const { appPath, appName, uninstall } = install({
-      ...subject,
-      filepath,
-      sandboxPath,
-    });
+  assert.equal(latestApp.buildInfo.channel, subject.buildInfo.channel);
 
-    try {
-      process.env.PORT = '0'; // dynamic port
+  const install = getInstaller(latestApp.kind);
+  const filepath = await getLatestRelease(
+    channel,
+    context.arch,
+    latestApp.kind,
+    context.forceDownload
+  );
 
-      if (channel === 'dev') {
-        process.env.DEV_RELEASE = JSON.stringify({
-          version: version,
-          bucket_key_prefix: context.bucketKeyPrefix,
-        });
-      } else {
-        process.env.PUBLISHED_RELEASES = JSON.stringify({
-          name: version,
-          body: version,
-          bucket_key_prefix: context.bucketKeyPrefix,
-        });
-      }
+  debug('Installing');
+  const { appPath, appName, uninstall } = install({
+    ...latestApp,
+    filepath,
+    filename: subject.filename,
+    sandboxPath,
+  });
 
-      const server = await startAutoUpdateServer();
+  try {
+    process.env.PORT = '0'; // dynamic port
+    process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES = 'true';
 
-      const address = server.address();
-      assert(typeof address === 'object' && address !== null);
-      const port = address.port;
-      const HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE = `http://localhost:${port}`;
+    if (channel === 'dev') {
+      process.env.DEV_RELEASE = JSON.stringify({
+        version: version,
+        bucket_key_prefix: context.bucketKeyPrefix,
+      });
+    } else {
+      process.env.PUBLISHED_RELEASES = JSON.stringify({
+        name: version,
+        body: version,
+        bucket_key_prefix: context.bucketKeyPrefix,
+      });
+    }
 
-      try {
-        // must be async because the update server is running in the same process
-        await executeAsync(
-          'npm',
-          [
-            'run',
-            '--unsafe-perm',
-            'test-packaged',
-            '--workspace',
-            'compass-e2e-tests',
-            '--',
-            '--test-filter=auto-update',
-          ],
-          {
-            // We need to use a shell to get environment variables setup correctly
-            shell: true,
-            env: {
-              ...process.env,
-              HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE,
-              AUTO_UPDATE_UPDATABLE: (!!autoUpdatable).toString(),
-              TEST_NAME: 'auto-update-to',
-              EXPECTED_UPDATE_VERSION: version,
-              COMPASS_APP_NAME: appName,
-              COMPASS_APP_PATH: appPath,
-            },
-          }
-        );
-      } finally {
-        debug('Stopping auto-update server');
-        server.close();
-        delete process.env.DEV_RELEASE;
-        delete process.env.PUBLISHED_RELEASES;
-      }
+    const server = await startAutoUpdateServer();
+
+    const address = server.address();
+    assert(typeof address === 'object' && address !== null);
+    const port = address.port;
+    const HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE = `http://localhost:${port}`;
+
+    try {
+      debug('Executing');
+      // must be async because the update server is running in the same process
+      await executeAsync(
+        'npm',
+        [
+          'run',
+          '--unsafe-perm',
+          'test-packaged',
+          '--workspace',
+          'compass-e2e-tests',
+          '--',
+          '--test-filter=auto-update',
+        ],
+        {
+          // We need to use a shell to get environment variables setup correctly
+          shell: true,
+          env: {
+            ...process.env,
+            HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE,
+            AUTO_UPDATE_UPDATABLE: (!!subject.autoUpdatable).toString(),
+            TEST_NAME: 'auto-update-to',
+            EXPECTED_UPDATE_VERSION: version,
+            COMPASS_APP_NAME: appName,
+            COMPASS_APP_PATH: appPath,
+          },
+        }
+      );
     } finally {
-      await uninstall();
+      debug('Stopping auto-update server');
+      await stopAutoUpdateServer(server);
+      delete process.env.DEV_RELEASE;
+      delete process.env.PUBLISHED_RELEASES;
+      delete process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES;
     }
   } finally {
-    if (context.skipCleanup) {
-      debug(`Skipped cleaning up sandbox: ${sandboxPath}`);
-    } else {
-      debug(`Cleaning up sandbox: ${sandboxPath}`);
-      fs.rmSync(sandboxPath, { recursive: true });
-    }
+    debug('Uninstalling');
+    await uninstall();
   }
 }
diff --git a/packages/compass-smoke-tests/src/tests/time-to-first-query.ts b/packages/compass-smoke-tests/src/tests/time-to-first-query.ts
index 931ca77a2f7..72d555c0f82 100644
--- a/packages/compass-smoke-tests/src/tests/time-to-first-query.ts
+++ b/packages/compass-smoke-tests/src/tests/time-to-first-query.ts
@@ -1,4 +1,3 @@
-import fs from 'node:fs';
 import createDebug from 'debug';
 import { type SmokeTestsContext } from '../context';
 import { execute } from '../execute';
@@ -16,45 +15,39 @@ export async function testTimeToFirstQuery(context: SmokeTestsContext) {
   });
   const { kind } = subject;
 
-  try {
-    const install = getInstaller(kind);
+  const install = getInstaller(kind);
 
-    const { appPath, appName, uninstall } = install({
-      ...subject,
-      sandboxPath,
-    });
+  debug('Installing');
+  const { appPath, appName, uninstall } = install({
+    ...subject,
+    sandboxPath,
+  });
 
-    try {
-      execute(
-        'npm',
-        [
-          'run',
-          '--unsafe-perm',
-          'test-packaged',
-          '--workspace',
-          'compass-e2e-tests',
-          '--',
-          '--test-filter=time-to-first-query',
-        ],
-        {
-          // We need to use a shell to get environment variables setup correctly
-          shell: true,
-          env: {
-            ...process.env,
-            COMPASS_APP_NAME: appName,
-            COMPASS_APP_PATH: appPath,
-          },
-        }
-      );
-    } finally {
-      await uninstall();
-    }
+  try {
+    debug('Executing');
+    execute(
+      'npm',
+      [
+        'run',
+        '--unsafe-perm',
+        'test-packaged',
+        '--workspace',
+        'compass-e2e-tests',
+        '--',
+        '--test-filter=time-to-first-query',
+      ],
+      {
+        // We need to use a shell to get environment variables setup correctly
+        shell: true,
+        env: {
+          ...process.env,
+          COMPASS_APP_NAME: appName,
+          COMPASS_APP_PATH: appPath,
+        },
+      }
+    );
   } finally {
-    if (context.skipCleanup) {
-      debug(`Skipped cleaning up sandbox: ${sandboxPath}`);
-    } else {
-      debug(`Cleaning up sandbox: ${sandboxPath}`);
-      fs.rmSync(sandboxPath, { recursive: true });
-    }
+    debug('Uninstalling');
+    await uninstall();
   }
 }
diff --git a/packages/compass-smoke-tests/src/tests/update-server.ts b/packages/compass-smoke-tests/src/tests/update-server.ts
index 3b1019c526a..6764b1c1390 100644
--- a/packages/compass-smoke-tests/src/tests/update-server.ts
+++ b/packages/compass-smoke-tests/src/tests/update-server.ts
@@ -1,4 +1,7 @@
 import { once } from 'node:events';
+import type http from 'node:http';
+import { promisify } from 'node:util';
+
 import createDebug from 'debug';
 
 const debug = createDebug('compass:smoketests:update-server');
@@ -20,3 +23,7 @@ export async function startAutoUpdateServer() {
 
   return httpServer;
 }
+
+export async function stopAutoUpdateServer(server: http.Server) {
+  await promisify(server.close.bind(server))();
+}