test: add e2e tests of the build+preview use case (un-revert of #8384) (#8436)

* test: add e2e tests of the build+preview use case (#8384)

* test: add e2e tests of the build+preview use case
* fixups: refactor tests to avoid boilerplate and make it easier to test all the combinations
* Apply suggestions from code review

Co-authored-by: James Opstad <[email protected]>

---------

Co-authored-by: James Opstad <[email protected]>

* avoid running binaries directly, which doesn't seem to work with yarn on Windows

---------

Co-authored-by: James Opstad <[email protected]>

Author: petebacondarwin

packages/vite-plugin-cloudflare/e2e/README.md

@@ -1,6 +1,6 @@
 # vite-plugin e2e tests
 
-This directory contains e2e test that give more confidence that the plugin will work in real world scenarios outside the comfort of this monorepo.
+This directory contains e2e tests that give more confidence that the plugin will work in real world scenarios outside the comfort of this monorepo.
 
 In general, these tests create test projects by copying a fixture from the `fixtures` directory into a temporary directory and then installing the local builds of the plugin along with its dependencies.
 
@@ -9,8 +9,10 @@ In general, these tests create test projects by copying a fixture from the `fixt
 Simply use turbo to run the tests from the root of the monorepo.
 This will also ensure that the required dependencies have all been built before running the tests.
 
+You will need to provide CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN for the Workers AI tests to pass.
+
 ```sh
-pnpm test:e2e -F @cloudflare/vite-plugin
+CLOUDFLARE_ACCOUNT_ID=xxxx CLOUDFLARE_API_TOKEN=yyyy pnpm test:e2e -F @cloudflare/vite-plugin
 ```
 
 ## Developing e2e tests
@@ -19,7 +21,7 @@ These tests use a mock npm registry where the built plugin has been published.
 
 The registry is booted up and loaded with the local build of the plugin and its local dependencies in the global-setup.ts file that runs once at the start of the e2e test run, and the server is killed and its caches removed at the end of the test run.
 
-The Vite `test` function is an extended with additional helpers to setup clean copies of fixtures outside of the monorepo so that they can be isolated from any other dependencies in the project.
+The Vitest `test` function is extended with additional helpers to setup clean copies of fixtures outside of the monorepo so that they can be isolated from any other dependencies in the project.
 
 The simplest test looks like:
 
@@ -28,14 +30,23 @@ test("can serve a Worker request", async ({ expect, seed, viteDev }) => {
 	const projectPath = await seed("basic");
 	runCommand(`pnpm install`, projectPath);
 
-	const proc = await viteDev(projectPath);
+	const proc = await viteDev("pnpm", "dev", projectPath);
 	const url = await waitForReady(proc);
 	expect(await fetchJson(url + "/api/")).toEqual({ name: "Cloudflare" });
 });
 ```
 
 - The `seed()` helper makes a copy of the named fixture into a temporary directory. It returns the path to the directory containing the copy (`projectPath` above). This directory will be deleted at the end of the test.
 - The `runCommand()` helper simply executes a one-shot command and resolves when it has exited. You can use this to install the dependencies of the fixture from the mock npm registry, as in the example above.
-- The `viteDev()` helper boots up the `vite dev` command and returns an object that can be used to monitor its output. The process will be killed at the end of the test.
-- The `waitForReady()` helper will resolve when the `vite dev` process has output its ready message, from which it will parse the url that can be fetched in the test.
+- The `viteCommand()` helper boots up the given long-lived command and returns an object that can be used to monitor its output. The process will be killed at the end of the test.
+- The `waitForReady()` helper will resolve when the `proc` process has output its ready message, from which it will parse the url that can be fetched in the test.
 - The `fetchJson()` helper makes an Undici fetch to the url parsing the response into JSON. It will retry every 250ms for up to 10 secs to minimize flakes.
+
+## Debugging e2e tests
+
+You can control the logging and cleanup via environment variables:
+
+- Keep the temporary directory after the tests have completed: `CLOUDFLARE_VITE_E2E_KEEP_TEMP_DIRS=true`
+- See debug logs for the tests: `NODE_DEBUG=vite-plugin:test`
+- See debug logs for the mock npm registry: `NODE_DEBUG=mock-npm-registry`
+- See debug logs for Vite: `DEBUG="vite:*"`

packages/vite-plugin-cloudflare/e2e/basic.test.ts

@@ -1,30 +1,53 @@
 import { describe } from "vitest";
 import { fetchJson, runCommand, test, waitForReady } from "./helpers.js";
 
-describe("node compatibility", () => {
-	describe.each(["pnpm --no-store", "npm", "yarn"])("using %s", (pm) => {
-		test("can serve a Worker request", async ({ expect, seed, viteDev }) => {
-			const projectPath = await seed("basic");
-			runCommand(`${pm} install`, projectPath);
+const isWindows = process.platform === "win32";
 
-			const proc = await viteDev(projectPath);
-			const url = await waitForReady(proc);
-			expect(await fetchJson(url + "/api/")).toEqual({ name: "Cloudflare" });
-		});
-	});
-});
+const packageManagers = ["pnpm", "npm", "yarn"] as const;
+const commands = ["dev", "preview"] as const;
+
+describe("basic e2e tests", () => {
+	describe.each(commands)('with "%s" command', (command) => {
+		// TODO: re-enable `vite preview` tests on Windows (DEVX-1748)
+		describe.skipIf(command === "preview" && isWindows).each(packageManagers)(
+			'with "%s" package manager',
+			(pm) => {
+				describe("node compatibility", () => {
+					test("can serve a Worker request", async ({
+						expect,
+						seed,
+						viteCommand,
+					}) => {
+						const projectPath = await seed("basic");
+						runCommand(`${pm} install`, projectPath);
+
+						const proc = await viteCommand(pm, command, projectPath);
+						const url = await waitForReady(proc);
+						expect(await fetchJson(url + "/api/")).toEqual({
+							name: "Cloudflare",
+						});
+					});
+				});
 
-// This test checks that wrapped bindings which rely on additional workers with an authed connection to the CF API work
-describe("Workers AI", () => {
-	test("can serve a Worker request", async ({ expect, seed, viteDev }) => {
-		const projectPath = await seed("basic");
-		runCommand(`npm install`, projectPath);
+				// This test checks that wrapped bindings which rely on additional workers with an authed connection to the CF API work
+				describe("Workers AI", () => {
+					test("can serve a Worker request", async ({
+						expect,
+						seed,
+						viteCommand,
+					}) => {
+						const projectPath = await seed("basic");
+						runCommand(`${pm} install`, projectPath);
 
-		const proc = await viteDev(projectPath);
-		const url = await waitForReady(proc);
+						const proc = await viteCommand(pm, command, projectPath);
+						const url = await waitForReady(proc);
 
-		expect(await fetchJson(url + "/ai/")).toEqual({
-			response: expect.stringContaining("Workers AI"),
-		});
+						expect(await fetchJson(url + "/ai/")).toEqual({
+							response: expect.stringContaining("Workers AI"),
+						});
+					});
+				});
+			}
+		);
 	});
 });

packages/vite-plugin-cloudflare/e2e/fixtures/basic/package.json

@@ -4,7 +4,7 @@
 	"private": true,
 	"type": "module",
 	"scripts": {
-		"build": "tsc -b && vite build",
+		"build": "vite build",
 		"dev": "vite",
 		"lint": "eslint .",
 		"preview": "vite preview"

packages/vite-plugin-cloudflare/e2e/global-setup.ts

@@ -3,7 +3,9 @@ import os from "node:os";
 import path from "node:path";
 import util from "node:util";
 import { startMockNpmRegistry } from "@cloudflare/mock-npm-registry";
-import type { GlobalSetupContext } from "vitest/node";
+import type { TestProject } from "vitest/node";
+
+const debuglog = util.debuglog("vite-plugin:test");
 
 declare module "vitest" {
 	export interface ProvidedContext {
@@ -13,22 +15,27 @@ declare module "vitest" {
 
 // Using a global setup means we can modify tests without having to re-install
 // packages into our temporary directory
-// Typings for the GlobalSetupContext are augmented in `global-setup.d.ts`.
-export default async function ({ provide }: GlobalSetupContext) {
+export default async function ({ provide }: TestProject) {
 	const stopMockNpmRegistry = await startMockNpmRegistry(
 		"@cloudflare/vite-plugin"
 	);
 
 	// Create temporary directory to host projects used for testing
 	const root = await fs.mkdtemp(path.join(os.tmpdir(), "vite-plugin-"));
+	debuglog("Created temporary directory at " + root);
 
+	// The type of the provided `root` is defined in the `ProvidedContent` type above.
 	provide("root", root);
 
 	// Cleanup temporary directory on teardown
 	return async () => {
 		await stopMockNpmRegistry();
 
-		console.log("Cleaning up temporary directory...");
-		await fs.rm(root, { recursive: true, maxRetries: 10 });
+		if (process.env.CLOUDFLARE_VITE_E2E_KEEP_TEMP_DIRS) {
+			debuglog("Temporary directory left in-place at " + root);
+		} else {
+			debuglog("Cleaning up temporary directory...");
+			await fs.rm(root, { recursive: true, maxRetries: 10 });
+		}
 	};
 }

packages/vite-plugin-cloudflare/e2e/helpers.ts

@@ -5,20 +5,32 @@ import path from "node:path";
 import util from "node:util";
 import { stripAnsi } from "miniflare";
 import kill from "tree-kill";
-import { test as baseTest, inject, vi } from "vitest";
+import { test as baseTest, inject, onTestFailed, vi } from "vitest";
 
 const debuglog = util.debuglog("vite-plugin:test");
 
+const testEnv = {
+	...process.env,
+	// The following env vars are set to ensure that package managers
+	// do not use the same global cache and accidentally hit race conditions.
+	YARN_CACHE_FOLDER: "./.yarn/cache",
+	YARN_ENABLE_GLOBAL_CACHE: "false",
+	PNPM_HOME: "./.pnpm",
+	npm_config_cache: "./.npm/cache",
+};
+
 /**
  * Extends the Vitest `test()` function to support running vite in
  * well defined environments that represent real-world usage.
  */
 export const test = baseTest.extend<{
 	seed: (fixture: string) => Promise<string>;
-	viteDev: (
+	viteCommand: (
+		pm: "pnpm" | "npm" | "yarn",
+		command: "dev" | "preview",
 		projectPath: string,
 		options?: { flags?: string[]; maxBuffer?: number }
-	) => Process;
+	) => Promise<Process>;
 }>({
 	/** Seed a test project from a fixture. */
 	async seed({}, use) {
@@ -45,18 +57,24 @@ export const test = baseTest.extend<{
 			}
 		}
 	},
-	/** Start a `vite dev` command and wraps its outputs. */
-	async viteDev({}, use) {
+	/** Starts a command and wraps its outputs. */
+	async viteCommand({}, use) {
 		const processes: ChildProcess[] = [];
-		await use((projectPath) => {
-			debuglog("starting vite for " + projectPath);
-			const proc = childProcess.exec(`pnpm exec vite dev`, {
+		await use(async (pm, command, projectPath) => {
+			if (command === "preview") {
+				// We must first run the build command to generate the Worker that is to be previewed.
+				await runCommand(`${pm} run build`, projectPath);
+			}
+
+			debuglog(`starting "${command}" with ${pm} in ${projectPath}`);
+			const proc = childProcess.exec(`${pm} run ${command}`, {
 				cwd: projectPath,
+				env: testEnv,
 			});
 			processes.push(proc);
 			return wrap(proc);
 		});
-		debuglog("Closing down vite dev processes", processes.length);
+		debuglog("Closing down command processes", processes.length);
 		processes.forEach((proc) => proc.pid && kill(proc.pid));
 	},
 });
@@ -68,7 +86,7 @@ export interface Process {
 }
 
 /**
- * Wrap a long running child process to capture its stdio and make it available programmatically.
+ * Wraps a long running child process to capture its stdio and make it available programmatically.
  */
 function wrap(proc: childProcess.ChildProcess): Process {
 	let stdout = "";
@@ -88,7 +106,7 @@ function wrap(proc: childProcess.ChildProcess): Process {
 		stderr += chunk;
 	});
 	const closePromise = events.once(proc, "close");
-	return {
+	const wrappedProc = {
 		get stdout() {
 			return stripAnsi(stdout);
 		},
@@ -99,12 +117,24 @@ function wrap(proc: childProcess.ChildProcess): Process {
 			return closePromise.then(([exitCode]) => exitCode ?? -1);
 		},
 	};
+
+	onTestFailed(() => {
+		console.log(
+			`Wrapped process logs (${proc.spawnfile} ${proc.spawnargs.join(" ")}):`
+		);
+		console.log(wrappedProc.stdout);
+		console.error(wrappedProc.stderr);
+	});
+
+	return wrappedProc;
 }
 
 export function runCommand(command: string, cwd: string) {
+	debuglog(`Running "${command}"`);
 	childProcess.execSync(command, {
 		cwd,
 		stdio: debuglog.enabled ? "inherit" : "ignore",
+		env: testEnv,
 	});
 }