test: add e2e tests of the build+preview use case (#8905)

* test: add e2e tests of the build+preview use case

* fixups

* add npm install retries

* clarify mock registry usage

* fix retry logic

* ensure seeded directory is cleaned between tests

* don't rely upon pre/post npm hooks

* faster e2e tests

* skip AI e2e tests if there is no account id or api token

* fix up README

* allow package install beforeAll hook to run for up to 200 secs

* don't crash if killing test process fails

* skip e2e build tests on Windows

Author: petebacondarwin

packages/vite-plugin-cloudflare/e2e/README.md

@@ -1,6 +1,6 @@
 # vite-plugin e2e tests
 
-This directory contains e2e test that give more confidence that the plugin will work in real world scenarios outside the comfort of this monorepo.
+This directory contains e2e tests that give more confidence that the plugin will work in real world scenarios outside the comfort of this monorepo.
 
 In general, these tests create test projects by copying a fixture from the `fixtures` directory into a temporary directory and then installing the local builds of the plugin along with its dependencies.
 
@@ -9,44 +9,49 @@ In general, these tests create test projects by copying a fixture from the `fixt
 Simply use turbo to run the tests from the root of the monorepo.
 This will also ensure that the required dependencies have all been built before running the tests.
 
+You will need to provide CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN for the Workers AI tests to pass.
+
 ```sh
-pnpm test:e2e -F @cloudflare/vite-plugin
+CLOUDFLARE_ACCOUNT_ID=xxxx CLOUDFLARE_API_TOKEN=yyyy pnpm test:e2e -F @cloudflare/vite-plugin
 ```
 
 ## Developing e2e tests
 
 These tests use a mock npm registry where the built plugin has been published.
 
-The registry is booted up and loaded with the local build of the plugin and its local dependencies in the global-setup.ts file that runs once at the start of the e2e test run, and the server is killed and its caches removed at the end of the test run.
+The registry is booted up and loaded with the local build of the plugin and its local dependencies in the global-setup.ts file that runs once at the start of the e2e test run, and the server is killed and its caches removed at the end of the test run. The mock npm registry will delegate to the standard public npm registry for non-local dependency requests (e.g. vite, typescript, etc).
 
-The Vite `test` function is an extended with additional helpers to setup clean copies of fixtures outside of the monorepo so that they can be isolated from any other dependencies in the project.
+There is a `seed()` helper to setup a clean copy of a fixture outside of the monorepo so that it can be isolated from any other dependencies in the project.
 
 The simplest test looks like:
 
 ```ts
-test("can serve a Worker request", async ({ expect, seed, viteDev }) => {
-	const projectPath = await seed("basic", "pnpm");
+const projectPath = await seed("basic", "pnpm");
 
-	const proc = await viteDev(projectPath);
+test("can serve a Worker request", async ({ expect, seed, runLongLived }) => {
+	const proc = await runLongLived("npm", "dev", projectPath);
 	const url = await waitForReady(proc);
 	expect(await fetchJson(url + "/api/")).toEqual({ name: "Cloudflare" });
 });
 ```
 
-- The `seed()` helper does the following:
-  - makes a copy of the named fixture into a temporary directory,
+- The `seed()` helper creates a `beforeAll()` block:
+  - make a copy of the named fixture into a temporary directory,
   - updates the vite-plugin dependency in the package.json to match the local version
-  - runs `npm install` (or equivalent package manager command) in the temporary project
+  - runs `npm install` (or equivalent package manager command) in the temporary project - the dependencies are installed from the mock npm registry mentioned above.
   - returns the path to the directory containing the copy (`projectPath` above)
+- The `seed()` helper also creates an `afterAll()` block:
   - the temporary directory will be deleted at the end of the test.
 - The `runCommand()` helper simply executes a one-shot command and resolves when it has exited. You can use this to install the dependencies of the fixture from the mock npm registry.
-- The `viteDev()` helper boots up the `vite dev` command and returns an object that can be used to monitor its output. The process will be killed at the end of the test.
-- The `waitForReady()` helper will resolve when the `vite dev` process has output its ready message, from which it will parse the url that can be fetched in the test.
+- The `runLongLived()` helper runs an npm script (from the package.json scripts section) and returns an object that can be used to monitor its output. The process will be killed at the end of the test.
+- The `waitForReady()` helper will resolve when a `vite dev` long lived process has output a ready message, from which it will parse the url that can be fetched in the test.
 - The `fetchJson()` helper makes an Undici fetch to the url parsing the response into JSON. It will retry every 250ms for up to 10 secs to minimize flakes.
 
-## Debugging the tests
+## Debugging e2e tests
 
-You can provide the following environment variables to get access to the logs and the actual files being tested:
+You can control the logging and cleanup via environment variables:
 
-- `NODE_DEBUG=vite-plugin:test` - this will display debugging log messages as well as the streamed output from the commands being run.
-- `CLOUDFLARE_VITE_E2E_KEEP_TEMP_DIRS=1` - this will prevent the temporary directory containing the test project from being deleted, so that you can go and play with it manually.
+- Keep the temporary directory after the tests have completed: `CLOUDFLARE_VITE_E2E_KEEP_TEMP_DIRS=true`
+- See debug logs for the tests: `NODE_DEBUG=vite-plugin:test`
+- See debug logs for the mock npm registry: `NODE_DEBUG=mock-npm-registry`
+- See debug logs for Vite: `DEBUG="vite:*"`

packages/vite-plugin-cloudflare/e2e/basic.test.ts

@@ -1,28 +1,45 @@
-import { describe } from "vitest";
-import { fetchJson, test, waitForReady } from "./helpers.js";
+import { describe, test } from "vitest";
+import { fetchJson, runLongLived, seed, waitForReady } from "./helpers.js";
 
-describe("node compatibility", () => {
-	describe.each(["pnpm", "npm", "yarn"])("using %s", (pm) => {
-		test("can serve a Worker request", async ({ expect, seed, viteDev }) => {
-			const projectPath = await seed("basic", pm);
+const isWindows = process.platform === "win32";
+const packageManagers = ["pnpm", , "npm", "yarn"] as const;
+const commands = ["dev", "buildAndPreview"] as const;
 
-			const proc = await viteDev(projectPath);
-			const url = await waitForReady(proc);
-			expect(await fetchJson(url + "/api/")).toEqual({ name: "Cloudflare" });
-		});
-	});
-});
+describe("basic e2e tests", () => {
+	describe.each(packageManagers)('with "%s" package manager', async (pm) => {
+		const projectPath = seed("basic", pm);
 
-// This test checks that wrapped bindings which rely on additional workers with an authed connection to the CF API work
-describe("Workers AI", () => {
-	test("can serve a Worker request", async ({ expect, seed, viteDev }) => {
-		const projectPath = await seed("basic", "npm");
+		describe.each(commands)('with "%s" command', (command) => {
+			describe("node compatibility", () => {
+				test.skipIf(isWindows && command === "buildAndPreview")(
+					"can serve a Worker request",
+					async ({ expect }) => {
+						const proc = await runLongLived(pm, command, projectPath);
+						const url = await waitForReady(proc);
+						expect(await fetchJson(url + "/api/")).toEqual({
+							name: "Cloudflare",
+						});
+					}
+				);
+			});
 
-		const proc = await viteDev(projectPath);
-		const url = await waitForReady(proc);
+			// This test checks that wrapped bindings which rely on additional workers with an authed connection to the CF API work
+			// They are skipped if you have not provided the necessary account id and api token.
+			describe.skipIf(
+				!process.env.CLOUDFLARE_ACCOUNT_ID || !process.env.CLOUDFLARE_API_TOKEN
+			)("Workers AI", () => {
+				test.skipIf(isWindows && command === "buildAndPreview")(
+					"can serve a Worker request",
+					async ({ expect }) => {
+						const proc = await runLongLived(pm, command, projectPath);
+						const url = await waitForReady(proc);
 
-		expect(await fetchJson(url + "/ai/")).toEqual({
-			response: expect.stringContaining("Workers AI"),
+						expect(await fetchJson(url + "/ai/")).toEqual({
+							response: expect.stringContaining("Workers AI"),
+						});
+					}
+				);
+			});
 		});
 	});
 });

packages/vite-plugin-cloudflare/e2e/dynamic.test.ts

@@ -1,16 +1,16 @@
-import { describe } from "vitest";
-import { fetchJson, test, waitForReady } from "./helpers.js";
+import { describe, test } from "vitest";
+import { fetchJson, runLongLived, seed, waitForReady } from "./helpers.js";
+
+const packageManagers = ["pnpm", "npm", "yarn"] as const;
 
 describe("prebundling Node.js compatibility", () => {
-	describe.each(["pnpm", "npm", "yarn"])("using %s", (pm) => {
+	describe.each(packageManagers)('with "%s" package manager', (pm) => {
+		const projectPath = seed("dynamic", pm);
+
 		test("will not cause a reload on a dynamic import of a Node.js module", async ({
 			expect,
-			seed,
-			viteDev,
 		}) => {
-			const projectPath = await seed("dynamic", pm);
-
-			const proc = await viteDev(projectPath);
+			const proc = await runLongLived(pm, "dev", projectPath);
 			const url = await waitForReady(proc);
 			expect(await fetchJson(url)).toEqual("OK!");
 			expect(proc.stdout).not.toContain(

packages/vite-plugin-cloudflare/e2e/fixtures/basic/package.json

@@ -4,7 +4,8 @@
 	"private": true,
 	"type": "module",
 	"scripts": {
-		"build": "tsc -b && vite build",
+		"build": "vite build",
+		"buildAndPreview": "vite build && vite preview",
 		"dev": "vite",
 		"lint": "eslint .",
 		"preview": "vite preview"
@@ -26,7 +27,6 @@
 		"globals": "^15.14.0",
 		"typescript": "~5.7.2",
 		"typescript-eslint": "^8.22.0",
-		"vite": "^6.1.0",
-		"wrangler": "^3.108.1"
+		"vite": "^6.1.0"
 	}
 }

packages/vite-plugin-cloudflare/e2e/fixtures/dynamic/package.json

@@ -5,6 +5,7 @@
 	"type": "module",
 	"scripts": {
 		"build": "vite build",
+		"buildAndPreview": "vite build && vite preview",
 		"dev": "vite",
 		"lint": "eslint .",
 		"preview": "vite preview"
@@ -19,7 +20,6 @@
 		"globals": "^15.14.0",
 		"typescript": "~5.7.2",
 		"typescript-eslint": "^8.22.0",
-		"vite": "^6.1.0",
-		"wrangler": "^3.108.1"
+		"vite": "^6.1.0"
 	}
 }

packages/vite-plugin-cloudflare/e2e/global-setup.ts

@@ -5,6 +5,8 @@ import util from "node:util";
 import { startMockNpmRegistry } from "@cloudflare/mock-npm-registry";
 import type { TestProject } from "vitest/node";
 
+const debuglog = util.debuglog("vite-plugin:test");
+
 declare module "vitest" {
 	export interface ProvidedContext {
 		root: string;
@@ -20,15 +22,19 @@ export default async function ({ provide }: TestProject) {
 
 	// Create temporary directory to host projects used for testing
 	const root = await fs.mkdtemp(path.join(os.tmpdir(), "vite-plugin-"));
+	debuglog("Created temporary directory at " + root);
 
+	// The type of the provided `root` is defined in the `ProvidedContent` type above.
 	provide("root", root);
 
 	// Cleanup temporary directory on teardown
 	return async () => {
 		await stopMockNpmRegistry();
 
-		if (!process.env.CLOUDFLARE_VITE_E2E_KEEP_TEMP_DIRS) {
-			console.log("Cleaning up temporary directory...", root);
+		if (process.env.CLOUDFLARE_VITE_E2E_KEEP_TEMP_DIRS) {
+			debuglog("Temporary directory left in-place at " + root);
+		} else {
+			debuglog("Cleaning up temporary directory...");
 			await fs.rm(root, { recursive: true, maxRetries: 10 });
 		}
 	};