diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml
index 10968ccbc56..b1023484cd2 100644
--- a/.evergreen/functions.yml
+++ b/.evergreen/functions.yml
@@ -667,6 +667,8 @@ functions:
           <<: *compass-env
           DEBUG: ${debug|}
           GITHUB_TOKEN: ${generated_token}
+          GITHUB_PR_NUMBER: ${github_pr_number}
+          EVERGREEN_TASK_URL: https://spruce.mongodb.com/task/${task_id}
         script: |
           set -e
           # Load environment variables
diff --git a/.github/workflows/test-installers.yml b/.github/workflows/test-installers.yml
index d2aeb5171aa..92fbf175bef 100644
--- a/.github/workflows/test-installers.yml
+++ b/.github/workflows/test-installers.yml
@@ -20,10 +20,25 @@ on:
       nonce:
         type: string
         description: 'A random string to track the run from dispatch to watching'
+      github_pr_number:
+        type: string
+        description: 'Number of the PR that triggered this run'
+      evergreen_task_url:
+        type: string
+        description: 'URL to the Evergreen job that triggered this run'
 
 run-name: Test Installers ${{ github.event.inputs.dev_version || github.ref_name }} / (nonce = ${{ github.event.inputs.nonce || 'not set' }})
 
 jobs:
+  summarize:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add URL for the GitHub PR
+        if: ${{ github.event.inputs.github_pr_number }}
+        run: echo "[GitHub PR ${{ github.event.inputs.github_pr_number }}](https://github.com/mongodb-js/compass/pull/${{ github.event.inputs.github_pr_number }})" >> $GITHUB_STEP_SUMMARY
+      - name: Add URL for the Evergreen task
+        if: ${{ github.event.inputs.evergreen_task_url }}
+        run: echo "[Evergreen Task](${{ github.event.inputs.evergreen_task_url }})" >> $GITHUB_STEP_SUMMARY
   test:
     name: ${{ matrix.package }} test ${{ matrix.test }} (${{ matrix.hadron-distribution }})
     strategy:
diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts
index e83aef45ec3..599741f07c8 100755
--- a/packages/compass-smoke-tests/src/cli.ts
+++ b/packages/compass-smoke-tests/src/cli.ts
@@ -71,6 +71,7 @@ yargs(hideBin(process.argv))
   .scriptName('smoke-tests')
   .detectLocale(false)
   .version(false)
+  .showHelpOnFail(false)
   .strict()
   .option('bucketName', {
     type: 'string',
@@ -145,7 +146,13 @@ yargs(hideBin(process.argv))
           default: getDefaultRef(),
         }),
     async ({ bucketName, bucketKeyPrefix, ref, githubPrNumber }) => {
-      const { GITHUB_TOKEN } = process.env;
+      const {
+        GITHUB_TOKEN,
+        DEV_VERSION_IDENTIFIER,
+        GITHUB_PR_NUMBER,
+        EVERGREEN_TASK_URL,
+      } = process.env;
+
       assert(
         typeof GITHUB_TOKEN === 'string',
         'Expected a GITHUB_TOKEN environment variable'
@@ -164,7 +171,9 @@ yargs(hideBin(process.argv))
                 githubPrNumber,
               })
             : ref,
-        devVersion: process.env.DEV_VERSION_IDENTIFIER,
+        devVersion: DEV_VERSION_IDENTIFIER,
+        githubPrNumber: GITHUB_PR_NUMBER,
+        evergreenTaskUrl: EVERGREEN_TASK_URL,
         bucketName,
         bucketKeyPrefix,
       });
diff --git a/packages/compass-smoke-tests/src/dispatch.ts b/packages/compass-smoke-tests/src/dispatch.ts
index ce0bee999dc..cade4f3c4f5 100644
--- a/packages/compass-smoke-tests/src/dispatch.ts
+++ b/packages/compass-smoke-tests/src/dispatch.ts
@@ -1,3 +1,4 @@
+import assert from 'node:assert/strict';
 import crypto from 'node:crypto';
 
 import * as github from '@actions/github';
@@ -70,12 +71,59 @@ export async function getRefFromGithubPr({
   return ref;
 }
 
+type PollToCompletionOptions = {
+  octokit: ReturnType<typeof github.getOctokit>;
+  runId: number;
+  watchTimeoutMs: number;
+  watchPollDelayMs: number;
+};
+
+async function pollToCompletion({
+  octokit,
+  runId,
+  watchTimeoutMs,
+  watchPollDelayMs,
+}: PollToCompletionOptions): Promise<string> {
+  for (
+    const start = new Date();
+    new Date().getTime() - start.getTime() < watchTimeoutMs;
+
+  ) {
+    const {
+      data: { status, conclusion },
+    } = await octokit.rest.actions.getWorkflowRun({
+      owner: GITHUB_OWNER,
+      repo: GITHUB_REPO,
+      run_id: runId,
+    });
+    console.log(`Status: ${status || 'null'}`);
+    if (status === 'completed') {
+      assert(
+        typeof conclusion === 'string',
+        'Expected conclusion when completed'
+      );
+      return conclusion;
+    }
+    await new Promise((resolve) => setTimeout(resolve, watchPollDelayMs));
+  }
+
+  // Cancel the run before timing out
+  await octokit.rest.actions.cancelWorkflowRun({
+    owner: GITHUB_OWNER,
+    repo: GITHUB_REPO,
+    run_id: runId,
+  });
+  return 'timeout';
+}
+
 type DispatchOptions = {
   githubToken: string;
   ref: string;
   bucketName: string;
   bucketKeyPrefix: string;
   devVersion?: string;
+  githubPrNumber?: string;
+  evergreenTaskUrl?: string;
 
   /**
    * Delay in milliseconds to wait between requests when polling while watching the run.
@@ -89,6 +137,8 @@ export async function dispatchAndWait({
   devVersion,
   bucketName,
   bucketKeyPrefix,
+  githubPrNumber,
+  evergreenTaskUrl,
   watchPollDelayMs = 5000,
 }: DispatchOptions) {
   const octokit = github.getOctokit(githubToken);
@@ -103,6 +153,8 @@ export async function dispatchAndWait({
       dev_version: devVersion,
       bucket_name: bucketName,
       bucket_key_prefix: bucketKeyPrefix,
+      github_pr_number: githubPrNumber,
+      evergreen_task_url: evergreenTaskUrl,
       nonce,
     },
   });
@@ -114,27 +166,13 @@ export async function dispatchAndWait({
   );
 
   console.log(`Dispatched run #${run.run_number} (${run.html_url})`);
-  for (
-    const start = new Date();
-    new Date().getTime() - start.getTime() < WATCH_POLL_TIMEOUT_MS;
+  const status = await pollToCompletion({
+    octokit,
+    runId: run.id,
+    watchTimeoutMs: WATCH_POLL_TIMEOUT_MS,
+    watchPollDelayMs,
+  });
 
-  ) {
-    const {
-      data: { status, conclusion },
-    } = await octokit.rest.actions.getWorkflowRun({
-      owner: GITHUB_OWNER,
-      repo: GITHUB_REPO,
-      run_id: run.id,
-    });
-    console.log(
-      `Status = ${status || 'null'}, conclusion = ${conclusion || 'null'}`
-    );
-    if (status === 'completed' && conclusion === 'success') {
-      return;
-    }
-    await new Promise((resolve) => setTimeout(resolve, watchPollDelayMs));
-  }
-  throw new Error(
-    `Run did not complete successfully within ${WATCH_POLL_TIMEOUT_MS}ms: See ${run.html_url} for details.`
-  );
+  console.log(`Run completed: ${run.html_url}`);
+  assert.equal(status, 'success', "Expected a 'success' conclusion");
 }