test: allow e2e suite to run against dist bundle (#164)

Run make build before playwright to provide dist artifacts when needed.
Gate dist mode behind CMUX_E2E_LOAD_DIST and assert required files.
Skip the dev server and launch electron with production env when set.
Enable parallel playwright workers to keep runtimes acceptable.

Author: ThomasK33

Makefile

@@ -133,7 +133,8 @@ test-coverage: ## Run tests with coverage
 	@./scripts/test.sh --coverage
 
 test-e2e: ## Run end-to-end tests
-	@PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 bun x playwright test --project=electron
+	@$(MAKE) build
+	@CMUX_E2E_LOAD_DIST=1 CMUX_E2E_SKIP_BUILD=1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 bun x playwright test --project=electron
 
 ## Distribution
 dist: build ## Build distributable packages

playwright.config.ts

@@ -8,14 +8,13 @@ export default defineConfig({
   expect: {
     timeout: 5_000,
   },
-  fullyParallel: false,
+  fullyParallel: true,
   forbidOnly: isCI,
   retries: isCI ? 1 : 0,
   reporter: [
     ["list"],
     ["html", { outputFolder: "artifacts/playwright-report", open: "never" }],
   ],
-  workers: 1,
   use: {
     trace: isCI ? "on-first-retry" : "retain-on-failure",
     screenshot: "only-on-failure",

tests/e2e/electronTest.ts

@@ -22,7 +22,27 @@ interface ElectronFixtures {
 
 const appRoot = path.resolve(__dirname, "..", "..");
 const defaultTestRoot = path.join(appRoot, "tests", "e2e", "tmp", "cmux-root");
-const DEV_SERVER_PORT = 5173;
+const BASE_DEV_SERVER_PORT = Number(process.env.CMUX_E2E_DEVSERVER_PORT_BASE ?? "5173");
+const shouldLoadDist = process.env.CMUX_E2E_LOAD_DIST === "1";
+
+const REQUIRED_DIST_FILES = [
+  path.join(appRoot, "dist", "index.html"),
+  path.join(appRoot, "dist", "main.js"),
+  path.join(appRoot, "dist", "preload.js"),
+] as const;
+
+function assertDistBundleReady(): void {
+  if (!shouldLoadDist) {
+    return;
+  }
+  for (const filePath of REQUIRED_DIST_FILES) {
+    if (!fs.existsSync(filePath)) {
+      throw new Error(
+        `Missing build artifact at ${filePath}. Run "make build" before executing dist-mode e2e tests.`
+      );
+    }
+  }
+}
 
 async function waitForServerReady(url: string, timeoutMs = 20_000): Promise<void> {
   const start = Date.now();
@@ -70,9 +90,8 @@ export const electronTest = base.extend<ElectronFixtures>({
   workspace: async ({}, use, testInfo) => {
     const envRoot = process.env.CMUX_TEST_ROOT ?? "";
     const baseRoot = envRoot || defaultTestRoot;
-    const testRoot = envRoot
-      ? baseRoot
-      : path.join(baseRoot, sanitizeForPath(testInfo.title ?? testInfo.testId));
+    const uniqueTestId = testInfo.testId || testInfo.title || `test-${Date.now()}`;
+    const testRoot = envRoot ? baseRoot : path.join(baseRoot, sanitizeForPath(uniqueTestId));
 
     const shouldCleanup = !envRoot;
 
@@ -95,34 +114,55 @@ export const electronTest = base.extend<ElectronFixtures>({
   },
   app: async ({ workspace }, use, testInfo) => {
     const { configRoot } = workspace;
-    buildTarget("build-main");
-    buildTarget("build-preload");
-
-    const devServer = spawn("make", ["dev"], {
-      cwd: appRoot,
-      stdio: ["ignore", "ignore", "inherit"],
-      env: {
-        ...process.env,
-        NODE_ENV: "development",
-        VITE_DISABLE_MERMAID: "1",
-      },
-    });
+    const devServerPort = BASE_DEV_SERVER_PORT + testInfo.workerIndex;
+
+    if (shouldLoadDist) {
+      assertDistBundleReady();
+    } else {
+      buildTarget("build-main");
+      buildTarget("build-preload");
+    }
 
+    const shouldStartDevServer = !shouldLoadDist;
+    let devServer: ReturnType<typeof spawn> | undefined;
     let devServerExited = false;
-    const devServerExitPromise = new Promise<void>((resolve) => {
-      const handleExit = () => {
-        devServerExited = true;
-        resolve();
-      };
-
-      if (devServer.exitCode !== null) {
-        handleExit();
-      } else {
-        devServer.once("exit", handleExit);
+    let devServerExitPromise: Promise<void> | undefined;
+
+    if (shouldStartDevServer) {
+      devServer = spawn("make", ["dev"], {
+        cwd: appRoot,
+        stdio: ["ignore", "ignore", "inherit"],
+        env: {
+          ...process.env,
+          NODE_ENV: "development",
+          VITE_DISABLE_MERMAID: "1",
+          CMUX_VITE_PORT: String(devServerPort),
+        },
+      });
+
+      const activeDevServer = devServer;
+      if (!activeDevServer) {
+        throw new Error("Failed to spawn dev server process");
       }
-    });
+
+      devServerExitPromise = new Promise<void>((resolve) => {
+        const handleExit = () => {
+          devServerExited = true;
+          resolve();
+        };
+
+        if (activeDevServer.exitCode !== null) {
+          handleExit();
+        } else {
+          activeDevServer.once("exit", handleExit);
+        }
+      });
+    }
 
     const stopDevServer = async () => {
+      if (!devServer || !devServerExitPromise) {
+        return;
+      }
       if (!devServerExited && devServer.exitCode === null) {
         devServer.kill("SIGTERM");
       }
@@ -134,29 +174,44 @@ export const electronTest = base.extend<ElectronFixtures>({
     let electronApp: ElectronApplication | undefined;
 
     try {
-      await waitForServerReady(`http://127.0.0.1:${DEV_SERVER_PORT}`);
-      if (devServer.exitCode !== null) {
-        throw new Error(`Vite dev server exited early (code ${devServer.exitCode})`);
+      let devHost = "127.0.0.1";
+      if (shouldStartDevServer) {
+        devHost = process.env.CMUX_DEVSERVER_HOST ?? "127.0.0.1";
+        await waitForServerReady(`http://${devHost}:${devServerPort}`);
+        const exitCode = devServer?.exitCode;
+        if (exitCode !== null && exitCode !== undefined) {
+          throw new Error(`Vite dev server exited early (code ${exitCode})`);
+        }
       }
 
       recordVideoDir = testInfo.outputPath("electron-video");
       fs.mkdirSync(recordVideoDir, { recursive: true });
 
-      const devHost = process.env.CMUX_DEVSERVER_HOST ?? "127.0.0.1";
+      const electronEnv: Record<string, string> = {};
+      for (const [key, value] of Object.entries(process.env)) {
+        if (typeof value === "string") {
+          electronEnv[key] = value;
+        }
+      }
+      electronEnv.ELECTRON_DISABLE_SECURITY_WARNINGS = "true";
+      electronEnv.CMUX_MOCK_AI = electronEnv.CMUX_MOCK_AI ?? "1";
+      electronEnv.CMUX_TEST_ROOT = configRoot;
+      electronEnv.CMUX_E2E = "1";
+      electronEnv.CMUX_E2E_LOAD_DIST = shouldLoadDist ? "1" : "0";
+      electronEnv.VITE_DISABLE_MERMAID = "1";
+
+      if (shouldStartDevServer) {
+        electronEnv.CMUX_DEVSERVER_PORT = String(devServerPort);
+        electronEnv.CMUX_DEVSERVER_HOST = devHost;
+        electronEnv.NODE_ENV = electronEnv.NODE_ENV ?? "development";
+      } else {
+        electronEnv.NODE_ENV = electronEnv.NODE_ENV ?? "production";
+      }
+
       electronApp = await electron.launch({
         args: ["."],
         cwd: appRoot,
-        env: {
-          ...process.env,
-          ELECTRON_DISABLE_SECURITY_WARNINGS: "true",
-          CMUX_MOCK_AI: process.env.CMUX_MOCK_AI ?? "1",
-          CMUX_TEST_ROOT: configRoot,
-          CMUX_E2E: "1",
-          CMUX_E2E_LOAD_DIST: "0",
-          CMUX_DEVSERVER_PORT: String(DEV_SERVER_PORT),
-          CMUX_DEVSERVER_HOST: devHost,
-          VITE_DISABLE_MERMAID: "1",
-        },
+        env: electronEnv,
         recordVideo: {
           dir: recordVideoDir,
           size: { width: 1280, height: 720 },
@@ -170,14 +225,17 @@ export const electronTest = base.extend<ElectronFixtures>({
           await electronApp.close();
         }
 
+        const displayName = testInfo.title ?? testInfo.testId;
         if (recordVideoDir) {
           try {
             const videoFiles = await fsPromises.readdir(recordVideoDir);
             if (electronApp && videoFiles.length) {
               const videosDir = path.join(appRoot, "artifacts", "videos");
               await fsPromises.mkdir(videosDir, { recursive: true });
               const orderedFiles = [...videoFiles].sort();
-              const baseName = testInfo.title.replace(/\s+/g, "-").toLowerCase();
+              const baseName = sanitizeForPath(
+                testInfo.testId || testInfo.title || "cmux-e2e-video"
+              );
               for (const [index, file] of orderedFiles.entries()) {
                 const ext = path.extname(file) || ".webm";
                 const suffix = orderedFiles.length > 1 ? `-${index}` : "";
@@ -187,19 +245,19 @@ export const electronTest = base.extend<ElectronFixtures>({
                 console.log(`[video] saved to ${destination}`); // eslint-disable-line no-console
               }
             } else if (electronApp) {
-              console.warn(
-                `[video] no video captured for "${testInfo.title}" at ${recordVideoDir}`
-              ); // eslint-disable-line no-console
+              console.warn(`[video] no video captured for "${displayName}" at ${recordVideoDir}`); // eslint-disable-line no-console
             }
           } catch (error) {
-            console.error(`[video] failed to process video for "${testInfo.title}":`, error); // eslint-disable-line no-console
+            console.error(`[video] failed to process video for "${displayName}":`, error); // eslint-disable-line no-console
           } finally {
             await fsPromises.rm(recordVideoDir, { recursive: true, force: true });
           }
         }
       }
     } finally {
-      await stopDevServer();
+      if (shouldStartDevServer) {
+        await stopDevServer();
+      }
     }
   },
   page: async ({ app }, use) => {

vite.config.ts

@@ -6,6 +6,8 @@ import { fileURLToPath } from "url";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const disableMermaid = process.env.VITE_DISABLE_MERMAID === "1";
+const devServerPort = Number(process.env.CMUX_VITE_PORT ?? "5173");
+const previewPort = Number(process.env.CMUX_VITE_PREVIEW_PORT ?? "4173");
 
 const alias: Record<string, string> = {
   "@": path.resolve(__dirname, "./src"),
@@ -29,8 +31,6 @@ export default defineConfig(({ mode }) => ({
     sourcemap: true,
     minify: "esbuild",
     rollupOptions: {
-      // Exclude ai-tokenizer from renderer bundle - it's never used there (only in main process)
-      external: ["ai-tokenizer"],
       output: {
         format: "es",
         inlineDynamicImports: false,
@@ -46,13 +46,13 @@ export default defineConfig(({ mode }) => ({
   },
   server: {
     host: "127.0.0.1",
-    port: 5173,
+    port: devServerPort,
     strictPort: true,
     allowedHosts: ["localhost", "127.0.0.1"],
   },
   preview: {
     host: "127.0.0.1",
-    port: 4173,
+    port: previewPort,
     strictPort: true,
     allowedHosts: ["localhost", "127.0.0.1"],
   },