backport: improve e2e testing approach to reduce flakiness (#10309)

devin-ai-integration[bot] · petebacondarwin · web-flow · commit bb14da5cedbd · 2025-08-13T12:55:29.000+02:00
* backport: improve e2e testing approach from main branch

- Add retry mechanism for failed tests
- Use environment variable isolation with WRANGLER_E2E_TEST_FILE
- Implement glob-based test file discovery
- Add better error handling and reporting
- Include GitHub Actions console grouping

This backports the improved e2e testing approach from main branch
to reduce flakiness in e2e tests on v3-maintenance branch.

Co-Authored-By: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;

* add glob dependency to tools package

The improved e2e runner imports glob directly, so it needs to be
added as a dependency to the tools package.json.

Co-Authored-By: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;

* fix: update pnpm-lock.yaml after adding glob dependency

This resolves the CI failure:
ERR_PNPM_OUTDATED_LOCKFILE Cannot install with "frozen-lockfile" because pnpm-lock.yaml is not up to date with tools/package.json

Co-Authored-By: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;

* fix: backport turbo cache configuration to resolve E2E test timeouts

- Add globalPassThroughEnv to root turbo.json with essential environment variables
- Update packages/wrangler/turbo.json with WRANGLER_E2E_TEST_FILE and other missing env vars
- Add passThroughEnv array to build task with comprehensive environment variable list
- Update build inputs to use $TURBO_DEFAULT$ pattern from main branch
- Add create-cloudflare#build dependency to test:e2e task

These changes ensure the backported e2e runner can properly utilize turbo cache
by recognizing the WRANGLER_E2E_TEST_FILE environment variable and other
essential configuration, preventing cache misses that cause test timeouts.

Co-Authored-By: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;

* fix: remove non-existent create-cloudflare#build dependency from test:e2e task

The create-cloudflare#build dependency doesn't exist in v3-maintenance branch,
causing E2E tests to fail with 'Could not find package create-cloudflare from task create-cloudflare#build'.
This fix removes the problematic dependency while keeping the essential WRANGLER_E2E_TEST_FILE
environment variable that enables proper turbo caching for the backported e2e runner.

Co-Authored-By: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;

* fix: add dotenv prefix to test:e2e:wrangler script for consistent env handling

This ensures the e2e test runner uses the same environment variable handling
pattern as other turbo tasks, optimizing turbo cache behavior.

Co-Authored-By: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;

* fix: revert dotenv prefix from test:e2e:wrangler to resolve env var passing issues

The dotenv prefix was causing the WRANGLER environment variable to not be
passed through properly, causing E2E test validation failures. The core
turbo cache optimizations (env var configuration, dependency fixes) remain
in place and provide the intended performance improvements.

Co-Authored-By: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;

* fix: add missing WRANGLER_E2E_TEST_FILE support to vitest config

Backport the vitest configuration from main branch to properly handle
the WRANGLER_E2E_TEST_FILE environment variable for individual test
file execution. This completes the e2e testing improvements backport.

- Add retry: 1 configuration for automatic test retries
- Add dynamic include pattern using WRANGLER_E2E_TEST_FILE env var
- Fixes E2E test failures by enabling proper test file filtering

Co-Authored-By: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;

---------

Co-authored-by: Devin AI &lt;158243242+devin-ai-integration[bot]@users.noreply.github.com&gt;
Co-authored-by: pbacondarwin@cloudflare.com &lt;pete@bacondarwin.com&gt;
diff --git a/packages/wrangler/e2e/vitest.config.mts b/packages/wrangler/e2e/vitest.config.mts
@@ -9,7 +9,9 @@ export default defineConfig({
 				singleThread: true,
 			},
 		},
-		include: ["e2e/**/*.test.ts"],
+		retry: 1,
+		// eslint-disable-next-line turbo/no-undeclared-env-vars
+		include: [process.env.WRANGLER_E2E_TEST_FILE || "e2e/**/*.test.ts"],
 		// eslint-disable-next-line turbo/no-undeclared-env-vars
 		outputFile: process.env.TEST_REPORT_PATH ?? ".e2e-test-report/index.html",
 		globalSetup: path.resolve(__dirname, "./validate-environment.ts"),
diff --git a/packages/wrangler/turbo.json b/packages/wrangler/turbo.json
@@ -3,73 +3,76 @@
 	"extends": ["//"],
 	"tasks": {
 		"build": {
-			"inputs": [
-				"!*/**/__tests__/**",
-				"!*/**/.wrangler/**",
-				"bin/**",
-				"src/**",
-				"scripts/**",
-				"templates/**",
-				"*.json",
-				"*.js",
-				"*.ts"
-			],
+			"inputs": ["$TURBO_DEFAULT$", "!**/__tests__/**", "!e2e/**"],
 			"outputs": [
 				"miniflare-dist/**",
 				"emitted-types/**",
 				"wrangler-dist/**",
 				"config-schema.json"
 			],
 			"env": [
-				"NODE_EXTRA_CA_CERTS",
-				"CLOUDFLARE_ACCOUNT_ID",
-				"SOURCEMAPS",
-				"NODE_ENV",
-				"SPARROW_SOURCE_KEY",
 				"ALGOLIA_APP_ID",
 				"ALGOLIA_PUBLIC_KEY",
-				"CLOUDFLARE_API_TOKEN",
-				"CLOUDFLARE_ACCOUNT_ID",
-				"WRANGLER_AUTH_DOMAIN",
-				"PATH",
-				"WRANGLER_LOG",
-				"EXPERIMENTAL_MIDDLEWARE",
-				"FORMAT_WRANGLER_ERRORS",
+				"NODE_ENV",
+				"SOURCEMAPS",
+				"SPARROW_SOURCE_KEY",
+				"SENTRY_DSN",
+				"WRANGLER_PRERELEASE_LABEL"
+			],
+			"passThroughEnv": [
+				"CF_PAGES_UPLOAD_JWT",
 				"CF_PAGES",
-				"WORKERS_CI",
 				"CI",
-				"CF_PAGES_UPLOAD_JWT",
+				"CLOUDFLARE_ACCOUNT_ID",
+				"CLOUDFLARE_API_TOKEN",
+				"CUSTOM_BUILD_VAR",
 				"EXPERIMENTAL_MIDDLEWARE",
+				"FORMAT_WRANGLER_ERRORS",
+				"http_proxy",
+				"HTTP_PROXY",
+				"https_proxy",
+				"HTTPS_PROXY",
+				"HYPERDRIVE_DATABASE_URL",
+				"WRANGLER_DOCKER_BIN",
+				"WRANGLER_DOCKER_HOST",
+				"LC_ALL",
 				"NO_D1_WARNING",
 				"NO_HYPERDRIVE_WARNING",
-				"WRANGLER",
-				"WRANGLER_IMPORT",
-				"CUSTOM_BUILD_VAR",
+				"PATH",
 				"PWD",
-				"LC_ALL",
-				"WRANGLER_SEND_METRICS",
-				"https_proxy",
-				"HTTPS_PROXY",
-				"http_proxy",
-				"HTTP_PROXY",
-				"CI_OS",
-				"SENTRY_DSN",
 				"SYSTEMROOT",
 				"TZ",
+				"WORKERS_CI",
+				"WRANGLER_API_ENVIRONMENT",
+				"WRANGLER_AUTH_DOMAIN",
+				"WRANGLER_D1_EXTRA_LOCATION_CHOICES",
 				"WRANGLER_DISABLE_EXPERIMENTAL_WARNING",
 				"WRANGLER_DISABLE_REQUEST_BODY_DRAINING",
+				"WRANGLER_LOG",
+				"WRANGLER_SEND_METRICS",
 				"WRANGLER_WORKER_REGISTRY_PORT",
-				"WRANGLER_API_ENVIRONMENT",
-				"HYPERDRIVE_DATABASE_URL"
+				"DOCKER_HOST",
+				"WRANGLER_DOCKER_HOST"
 			]
 		},
 		"test:ci": {
-			"inputs": ["!*/**/.wrangler/**", "**/__tests__/**"],
-			"dependsOn": ["build"]
+			"dependsOn": ["build"],
+			"env": ["VITEST", "NODE_DEBUG", "MINIFLARE_WORKERD_PATH"]
 		},
 		"test:e2e": {
 			"inputs": ["e2e/**"],
-			"dependsOn": ["build"]
+			"dependsOn": ["build"],
+			"env": [
+				"VITEST",
+				"NODE_DEBUG",
+				"MINIFLARE_WORKERD_PATH",
+				"WRANGLER",
+				"WRANGLER_IMPORT",
+				"MINIFLARE_IMPORT",
+				"CLOUDFLARE_ACCOUNT_ID",
+				"CLOUDFLARE_API_TOKEN",
+				"WRANGLER_E2E_TEST_FILE"
+			]
 		}
 	}
 }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/tools/e2e/runIndividualE2EFiles.ts b/tools/e2e/runIndividualE2EFiles.ts
@@ -1,32 +1,68 @@
-/**
- * Turbo only supports caching on the individual task level, but for Wrangler's
- * e2e tests we want to support caching on a more granular basis—at the file level.
- *
- * As such, we run the `test:e2e` turbo task multiple times—once per e2e test file
- * with different arguments, ensuring that each file's tests can be cached individually.
- *
- * The intended flow here is that CI will run this file, which will trigger turbo to run
- * an individual task for each Wrangler e2e test file, using `execSync`.
- */
+import assert from "assert";
 import { execSync } from "child_process";
-import { readdirSync } from "fs";
+import { statSync } from "fs";
+import path from "path";
+import { globIterateSync } from "glob";
+import type { ExecSyncOptionsWithBufferEncoding } from "child_process";
 
-// Get a list of e2e test files, each of which should have an associated script
-const e2eTests = readdirSync("packages/wrangler/e2e");
+// Turbo only supports caching on the individual task level, but for Wrangler's
+// e2e tests we want to support caching on a more granular basis - at the file level.
+//
+// As such, we run the `test:e2e` turbo task multiple times — once per e2e test file so that each file's tests can be cached individually.
+// We use the `WRANGLER_E2E_TEST_FILE` environment variable to pass the specific test file to the e2e test runner so that it reuses the cached build tasks.
+// If you use a command line argument to do this turbo will create a different cache key for the build tasks.
+//
+// The intended flow here is that CI will run this file, which will trigger turbo to run
+// an individual task for each Wrangler e2e test file, using `execSync`.
+//
+// Any params after a `--` will be passed to the Vitest runner, so you can use this to configure the test run.
+// For example to update the snapshots for all Wrangler e2e tests, you can run:
+//
+// ```bash
+// pnpm test:e2e:wrangler -- -u
+// ```
 
-const tasks = new Set<string>();
+const extraParamsIndex = process.argv.indexOf("--");
+const extraParams =
+	extraParamsIndex === -1 ? [] : process.argv.slice(extraParamsIndex);
+const command =
+	`pnpm test:e2e --log-order=stream --output-logs=new-only --summarize --filter wrangler ` +
+	extraParams.join(" ");
 
-for (const file of e2eTests) {
-	// Ignore other files in the e2e directory (the README, for instance)
-	if (file.endsWith(".test.ts")) {
-		tasks.add(
-			`pnpm test:e2e --log-order=stream --output-logs=new-only --summarize --filter wrangler --concurrency 1 -- run ./e2e/${file}`
-		);
+const failed: string[] = [];
+
+const wranglerPath = path.join(__dirname, "../../packages/wrangler");
+assert(statSync(wranglerPath).isDirectory());
+
+for (const testFile of globIterateSync("e2e/**/*.test.ts", {
+	cwd: wranglerPath,
+	// Return `/` delimited paths, even on Windows.
+	posix: true,
+})) {
+	const options: ExecSyncOptionsWithBufferEncoding = {
+		stdio: "inherit",
+		env: { ...process.env, WRANGLER_E2E_TEST_FILE: testFile },
+	};
+
+	console.log(`::group::Testing: ${testFile}`);
+
+	try {
+		execSync(command, options);
+	} catch {
+		console.error("Task failed - retrying");
+		try {
+			execSync(command, options);
+		} catch {
+			console.error("Still failed, moving on");
+			failed.push(testFile);
+		}
 	}
+	console.log("::endgroup::");
 }
 
-for (const task of tasks.values()) {
-	execSync(task, {
-		stdio: "inherit",
-	});
+if (failed.length > 0) {
+	throw new Error(
+		"At least one task failed (even on retry):" +
+			failed.map((file) => `\n - ${file}`)
+	);
 }
diff --git a/tools/package.json b/tools/package.json
@@ -14,6 +14,7 @@
 		"@typescript-eslint/eslint-plugin": "^6.9.0",
 		"@typescript-eslint/parser": "^6.9.0",
 		"find-up": "^6.3.0",
+		"glob": "^10.4.5",
 		"ts-dedent": "^2.2.0",
 		"undici": "catalog:default"
 	}
diff --git a/turbo.json b/turbo.json
@@ -3,13 +3,16 @@
 	"remoteCache": {
 		"signature": true
 	},
-	"globalEnv": [
+	"globalEnv": ["CI_OS", "NODE_VERSION"],
+	"globalPassThroughEnv": [
+		"NODE_EXTRA_CA_CERTS",
 		"CI",
-		"CI_OS",
-		"NODE_VERSION",
-		"VITEST",
-		"NODE_DEBUG",
-		"NODE_EXTRA_CA_CERTS"
+		"VSCODE_INSPECTOR_OPTIONS",
+		"WRANGLER_LOG_PATH",
+		"TEST_REPORT_PATH",
+		"CLOUDFLARE_CONTAINER_REGISTRY",
+		"DOCKER_HOST",
+		"WRANGLER_DOCKER_HOST"
 	],
 	"tasks": {
 		"dev": {
@@ -27,24 +30,20 @@
 			"persistent": true,
 			"cache": false
 		},
-		"topological": {
-			"dependsOn": ["^topological"]
-		},
 		"check:lint": {
-			"dependsOn": ["topological"]
+			"dependsOn": ["^check:lint"]
 		},
 		"check:type": {
-			"dependsOn": ["topological"]
+			"dependsOn": ["build"]
 		},
 		"type:tests": {
-			"dependsOn": ["topological"]
+			"dependsOn": ["build"]
 		},
 		"test:ci": {
 			"dependsOn": ["build"],
 			"outputLogs": "new-only"
 		},
 		"test:e2e": {
-			"env": ["WRANGLER", "CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_TOKEN"],
 			"dependsOn": ["build"],
 			"outputLogs": "new-only"
 		},

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,7 @@`
`14`	`14`	`"@typescript-eslint/eslint-plugin": "^6.9.0",`
`15`	`15`	`"@typescript-eslint/parser": "^6.9.0",`
`16`	`16`	`"find-up": "^6.3.0",`
	`17`	`+ "glob": "^10.4.5",`
`17`	`18`	`"ts-dedent": "^2.2.0",`
`18`	`19`	`"undici": "catalog:default"`
`19`	`20`	`}`