diff --git a/packages/compass-e2e-tests/helpers/compass.ts b/packages/compass-e2e-tests/helpers/compass.ts
index 923fdbd0382..1f17ed7748b 100644
--- a/packages/compass-e2e-tests/helpers/compass.ts
+++ b/packages/compass-e2e-tests/helpers/compass.ts
@@ -949,18 +949,20 @@ async function getCompassBuildMetadata(): Promise<BinPathOptions> {
 }
 
 export async function buildCompass(
-  force = false,
   compassPath = COMPASS_DESKTOP_PATH
 ): Promise<void> {
-  if (!force) {
-    try {
-      await getCompassBuildMetadata();
-      return;
-    } catch (e) {
-      // No compass build found, let's build it
-    }
+  try {
+    await getCompassBuildMetadata();
+    return;
+  } catch (e) {
+    /* ignore */
+  }
+
+  if (process.env.COMPASS_APP_PATH && process.env.COMPASS_APP_NAME) {
+    throw new Error('We did not expect to have to build Compass');
   }
 
+  debug("No Compass build found, let's build it");
   await packageCompassAsync({
     dir: compassPath,
     skip_installer: true,
diff --git a/packages/compass-e2e-tests/helpers/test-runner-global-fixtures.ts b/packages/compass-e2e-tests/helpers/test-runner-global-fixtures.ts
index 9c163d61b60..a2bd3e751e2 100644
--- a/packages/compass-e2e-tests/helpers/test-runner-global-fixtures.ts
+++ b/packages/compass-e2e-tests/helpers/test-runner-global-fixtures.ts
@@ -139,7 +139,7 @@ export async function mochaGlobalSetup(this: Mocha.Runner) {
 
     if (isTestingDesktop(context)) {
       if (context.testPackagedApp) {
-        debug('Building Compass before running the tests ...');
+        debug('Maybe building Compass before running the tests ...');
         await buildCompass();
       } else {
         debug('Preparing Compass before running the tests');