Configurable rules for exclusions in bundling of python_modules

.changeset/itchy-radios-drive.md

@@ -0,0 +1,5 @@
+---
+"wrangler": minor
+---
+
+Implements python_modules_excludes wrangler config field

packages/wrangler/e2e/dev.test.ts

@@ -534,6 +534,54 @@ describe.each([{ cmd: "wrangler dev" }])(
 
 			await worker.waitForReady();
 		});
+
+		it(`can exclude vendored module during ${cmd}`, async () => {
+			const helper = new WranglerE2ETestHelper();
+			await helper.seed({
+				"wrangler.toml": dedent`
+					name = "${workerName}"
+					main = "src/index.py"
+					compatibility_date = "2023-01-01"
+					compatibility_flags = ["python_workers"]
+					python_modules_excludes = ["excluded_module.py"]
+			`,
+				"src/arithmetic.py": dedent`
+					def mul(a,b):
+						return a*b`,
+				"python_modules/excluded_module.py": dedent`
+					def excluded(a,b):
+						return a*b`,
+				"src/index.py": dedent`
+					from arithmetic import mul
+
+					from js import Response
+					def on_fetch(request):
+						print(f"hello {mul(2,3)}")
+						try:
+							import excluded_module
+							print("excluded_module found")
+						except ImportError:
+							print("excluded_module not found")
+						print(f"end")
+						return Response.new(f"py hello world {mul(2,3)}")`,
+				"package.json": dedent`
+					{
+						"name": "worker",
+						"version": "0.0.0",
+						"private": true
+					}
+					`,
+			});
+			const worker = helper.runLongLived(cmd);
+
+			const { url } = await worker.waitForReady();
+
+			await expect(fetchText(url)).resolves.toBe("py hello world 6");
+
+			await worker.readUntil(/hello 6/);
+			await worker.readUntil(/excluded_module not found/);
+			await worker.readUntil(/end/);
+		});
 	}
 );
 

packages/wrangler/src/__tests__/config/configuration.test.ts

@@ -136,6 +136,7 @@ describe("normalizeAndValidateConfig()", () => {
 			data_blobs: undefined,
 			workers_dev: undefined,
 			preview_urls: undefined,
+			python_modules_excludes: ["**/*.pyc"],
 			no_bundle: undefined,
 			minify: undefined,
 			first_party_worker: undefined,

packages/wrangler/src/__tests__/deploy.test.ts

@@ -13484,6 +13484,7 @@ export default{
 			writeWranglerConfig({
 				main: "src/index.py",
 				compatibility_flags: ["python_workers"],
+				// python_modules_excludes is set to `**/*.pyc` by default
 			});
 
 			// Create main Python file
@@ -13503,6 +13504,16 @@ export default{
 				"# Python vendor module 2\nprint('hello')"
 			);
 
+			await fs.promises.writeFile(
+				"python_modules/test.pyc",
+				"this shouldn't be deployed"
+			);
+			await fs.promises.mkdir("python_modules/other", { recursive: true });
+			await fs.promises.writeFile(
+				"python_modules/other/test.pyc",
+				"this shouldn't be deployed"
+			);
+
 			// Create a regular Python module
 			await fs.promises.writeFile(
 				"src/helper.py",
@@ -13512,12 +13523,18 @@ export default{
 			const expectedModules = {
 				"index.py": mainPython,
 				"helper.py": "# Helper module\ndef helper(): pass",
+				"python_modules/module1.so": "binary content for module 1",
+				"python_modules/module2.py": "# Python vendor module 2\nprint('hello')",
 			};
 
 			mockSubDomainRequest();
 			mockUploadWorkerRequest({
 				expectedMainModule: "index.py",
 				expectedModules,
+				excludedModules: [
+					"python_modules/test.pyc",
+					"python_modules/other/test.pyc",
+				],
 			});
 
 			await runWrangler("deploy");

packages/wrangler/src/__tests__/helpers/mock-upload-worker.ts

@@ -30,6 +30,7 @@ export function mockUploadWorkerRequest(
 		expectedType?: "esm" | "sw" | "none";
 		expectedBindings?: unknown;
 		expectedModules?: Record<string, string | null>;
+		excludedModules?: string[];
 		expectedCompatibilityDate?: string;
 		expectedCompatibilityFlags?: string[];
 		expectedMigrations?: CfWorkerInit["migrations"];
@@ -142,6 +143,9 @@ export function mockUploadWorkerRequest(
 		for (const [name, content] of Object.entries(expectedModules)) {
 			expect(await serialize(formBody.get(name))).toEqual(content);
 		}
+		for (const name of excludedModules) {
+			expect(formBody.get(name)).toBeNull();
+		}
 
 		if (useOldUploadApi) {
 			return HttpResponse.json(
@@ -180,6 +184,7 @@ export function mockUploadWorkerRequest(
 		expectedType = "esm",
 		expectedBindings,
 		expectedModules = {},
+		excludedModules = [],
 		expectedCompatibilityDate,
 		expectedCompatibilityFlags,
 		env = undefined,

packages/wrangler/src/api/startDevWorker/BundlerController.ts

@@ -106,7 +106,8 @@ export class BundlerController extends Controller<BundlerControllerEventMap> {
 				? await noBundleWorker(
 						entry,
 						config.build.moduleRules,
-						this.#tmpDir.path
+						this.#tmpDir.path,
+						config.pythonModulesExcludes ?? []
 					)
 				: await bundleWorker(entry, this.#tmpDir.path, {
 						bundle: true,
@@ -279,6 +280,7 @@ export class BundlerController extends Controller<BundlerControllerEventMap> {
 					config.compatibilityDate,
 					config.compatibilityFlags
 				),
+				pythonModulesExcludes: config.pythonModulesExcludes,
 			},
 			(cb) => {
 				const newBundle = cb(this.#currentBundle);

packages/wrangler/src/api/startDevWorker/ConfigController.ts

@@ -342,6 +342,8 @@ async function resolveConfig(
 		compatibilityDate: getDevCompatibilityDate(config, input.compatibilityDate),
 		compatibilityFlags: input.compatibilityFlags ?? config.compatibility_flags,
 		complianceRegion: input.complianceRegion ?? config.compliance_region,
+		pythonModulesExcludes:
+			input.pythonModulesExcludes ?? config.python_modules_excludes,
 		entrypoint: entry.file,
 		projectRoot: entry.projectRoot,
 		bindings,

packages/wrangler/src/api/startDevWorker/types.ts

@@ -88,6 +88,9 @@ export interface StartDevWorkerInput {
 	/** Specify the compliance region mode of the Worker. */
 	complianceRegion?: Config["compliance_region"];
 
+	/** A list of glob patterns to exclude files from the python_modules directory when bundling. */
+	pythonModulesExcludes?: string[];
+
 	env?: string;
 
 	/**

packages/wrangler/src/config/config.ts

@@ -381,6 +381,7 @@ export const defaultWranglerConfig: Config = {
 	observability: { enabled: true },
 	/** The default here is undefined so that we can delegate to the CLOUDFLARE_COMPLIANCE_REGION environment variable. */
 	compliance_region: undefined,
+	python_modules_excludes: ["**/*.pyc"],
 
 	/** NON-INHERITABLE ENVIRONMENT FIELDS **/
 	define: {},

packages/wrangler/src/config/environment.ts

@@ -589,6 +589,16 @@ interface EnvironmentInheritable {
 	 * it can be set to `undefined` in configuration to delegate to the CLOUDFLARE_COMPLIANCE_REGION environment variable.
 	 */
 	compliance_region: "public" | "fedramp_high" | undefined;
+
+	/**
+	 * A list of glob patterns to exclude files from the python_modules directory when bundling.
+	 *
+	 * Patterns are relative to the python_modules directory and use glob syntax.
+	 *
+	 * @default ["**\*.pyc"]
+	 * @inheritable
+	 */
+	python_modules_excludes: string[];
 }
 
 export type DurableObjectBindings = {

packages/wrangler/src/config/validation.ts

@@ -1591,6 +1591,14 @@ function normalizeAndValidateEnvironment(
 			isOneOf("public", "fedramp_high"),
 			undefined
 		),
+		python_modules_excludes: inheritable(
+			diagnostics,
+			topLevelEnv,
+			rawEnv,
+			"python_modules_excludes",
+			isStringArray,
+			["**/*.pyc"]
+		),
 	};
 
 	warnIfDurableObjectsHaveNoMigrations(

packages/wrangler/src/deploy/deploy.ts

@@ -613,7 +613,12 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 			bundleType,
 			...bundle
 		} = props.noBundle
-			? await noBundleWorker(props.entry, props.rules, props.outDir)
+			? await noBundleWorker(
+					props.entry,
+					props.rules,
+					props.outDir,
+					config.python_modules_excludes
+				)
 			: await bundleWorker(
 					props.entry,
 					typeof destination === "string" ? destination : destination.path,

packages/wrangler/src/deployment-bundle/find-additional-modules.ts

@@ -50,7 +50,7 @@ function isValidPythonPackageName(name: string): boolean {
 	return regex.test(name);
 }
 
-function filterPythonVendorModules(
+function removePythonVendorModules(
 	isPythonEntrypoint: boolean,
 	modules: CfModule[]
 ): CfModule[] {
@@ -74,7 +74,8 @@ function getPythonVendorModulesSize(modules: CfModule[]): number {
 export async function findAdditionalModules(
 	entry: Entry,
 	rules: Rule[] | ParsedRules,
-	attachSourcemaps = false
+	attachSourcemaps = false,
+	pythonModulesExcludes: string[] = []
 ): Promise<CfModule[]> {
 	const files = getFiles(
 		entry.configPath,
@@ -176,13 +177,24 @@ export async function findAdditionalModules(
 					pythonModulesDir,
 					parseRules(vendoredRules)
 				)
-			).map((m) => {
-				const prefixedPath = path.join("python_modules", m.name);
-				return {
-					...m,
-					name: prefixedPath,
-				};
-			});
+			)
+				.filter((m) => {
+					// Check if the file matches any exclusion pattern
+					for (const pattern of pythonModulesExcludes) {
+						const regexp = globToRegExp(pattern, { globstar: true });
+						if (regexp.test(m.name)) {
+							return false; // Exclude this file
+						}
+					}
+					return true; // Include this file
+				})
+				.map((m) => {
+					const prefixedPath = path.join("python_modules", m.name);
+					return {
+						...m,
+						name: prefixedPath,
+					};
+				});
 
 			modules.push(...vendoredModules);
 		} else {
@@ -200,7 +212,7 @@ export async function findAdditionalModules(
 
 	if (modules.length > 0) {
 		logger.info(`Attaching additional modules:`);
-		const filteredModules = filterPythonVendorModules(
+		const filteredModules = removePythonVendorModules(
 			isPythonEntrypoint,
 			modules
 		);

packages/wrangler/src/deployment-bundle/no-bundle-worker.ts

@@ -9,9 +9,15 @@ import type { Entry } from "./entry";
 export async function noBundleWorker(
 	entry: Entry,
 	rules: Rule[],
-	outDir: string | undefined
+	outDir: string | undefined,
+	pythonModulesExcludes: string[] = []
 ) {
-	const modules = await findAdditionalModules(entry, rules);
+	const modules = await findAdditionalModules(
+		entry,
+		rules,
+		false,
+		pythonModulesExcludes
+	);
 	if (outDir) {
 		await writeAdditionalModules(modules, outDir);
 	}

packages/wrangler/src/dev/use-esbuild.ts

@@ -59,6 +59,7 @@ export function runBuild(
 		onStart,
 		defineNavigatorUserAgent,
 		checkFetch,
+		pythonModulesExcludes,
 	}: {
 		entry: Entry;
 		destination: string | undefined;
@@ -86,6 +87,7 @@ export function runBuild(
 		onStart: () => void;
 		defineNavigatorUserAgent: boolean;
 		checkFetch: boolean;
+		pythonModulesExcludes?: string[];
 	},
 	setBundle: (
 		cb: (previous: EsbuildBundle | undefined) => EsbuildBundle
@@ -110,7 +112,12 @@ export function runBuild(
 	async function getAdditionalModules() {
 		return noBundle
 			? dedupeModulesByName([
-					...((await doFindAdditionalModules(entry, rules)) ?? []),
+					...((await doFindAdditionalModules(
+						entry,
+						rules,
+						false,
+						pythonModulesExcludes ?? []
+					)) ?? []),
 					...additionalModules,
 				])
 			: additionalModules;

packages/wrangler/src/versions/upload.ts

@@ -588,7 +588,12 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 			bundleType,
 			...bundle
 		} = props.noBundle
-			? await noBundleWorker(props.entry, props.rules, props.outDir)
+			? await noBundleWorker(
+					props.entry,
+					props.rules,
+					props.outDir,
+					config.python_modules_excludes
+				)
 			: await bundleWorker(
 					props.entry,
 					typeof destination === "string" ? destination : destination.path,