From 93c0915834f2ccdecded2fe87c0b3acaf2a6f8cb Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 29 Sep 2025 11:49:41 +0100 Subject: [PATCH] Configurable rules for exclusions in bundling of python_modules --- .changeset/itchy-radios-drive.md | 5 ++ packages/wrangler/e2e/dev.test.ts | 48 +++++++++++++++++++ .../__tests__/config/configuration.test.ts | 1 + .../wrangler/src/__tests__/deploy.test.ts | 17 +++++++ .../__tests__/helpers/mock-upload-worker.ts | 5 ++ .../api/startDevWorker/BundlerController.ts | 4 +- .../api/startDevWorker/ConfigController.ts | 2 + .../wrangler/src/api/startDevWorker/types.ts | 3 ++ packages/wrangler/src/config/config.ts | 1 + packages/wrangler/src/config/environment.ts | 10 ++++ packages/wrangler/src/config/validation.ts | 8 ++++ packages/wrangler/src/deploy/deploy.ts | 7 ++- .../find-additional-modules.ts | 32 +++++++++---- .../src/deployment-bundle/no-bundle-worker.ts | 10 +++- packages/wrangler/src/dev/use-esbuild.ts | 9 +++- packages/wrangler/src/versions/upload.ts | 7 ++- 16 files changed, 153 insertions(+), 16 deletions(-) create mode 100644 .changeset/itchy-radios-drive.md diff --git a/.changeset/itchy-radios-drive.md b/.changeset/itchy-radios-drive.md new file mode 100644 index 000000000000..52ed92ff28ee --- /dev/null +++ b/.changeset/itchy-radios-drive.md @@ -0,0 +1,5 @@ +--- +"wrangler": minor +--- + +Implements python_modules_excludes wrangler config field diff --git a/packages/wrangler/e2e/dev.test.ts b/packages/wrangler/e2e/dev.test.ts index f4f90935c539..f435ee4527da 100644 --- a/packages/wrangler/e2e/dev.test.ts +++ b/packages/wrangler/e2e/dev.test.ts @@ -534,6 +534,54 @@ describe.each([{ cmd: "wrangler dev" }])( await worker.waitForReady(); }); + + it(`can exclude vendored module during ${cmd}`, async () => { + const helper = new WranglerE2ETestHelper(); + await helper.seed({ + "wrangler.toml": dedent` + name = "${workerName}" + main = "src/index.py" + compatibility_date = "2023-01-01" + compatibility_flags = ["python_workers"] + python_modules_excludes = ["excluded_module.py"] + `, + "src/arithmetic.py": dedent` + def mul(a,b): + return a*b`, + "python_modules/excluded_module.py": dedent` + def excluded(a,b): + return a*b`, + "src/index.py": dedent` + from arithmetic import mul + + from js import Response + def on_fetch(request): + print(f"hello {mul(2,3)}") + try: + import excluded_module + print("excluded_module found") + except ImportError: + print("excluded_module not found") + print(f"end") + return Response.new(f"py hello world {mul(2,3)}")`, + "package.json": dedent` + { + "name": "worker", + "version": "0.0.0", + "private": true + } + `, + }); + const worker = helper.runLongLived(cmd); + + const { url } = await worker.waitForReady(); + + await expect(fetchText(url)).resolves.toBe("py hello world 6"); + + await worker.readUntil(/hello 6/); + await worker.readUntil(/excluded_module not found/); + await worker.readUntil(/end/); + }); } ); diff --git a/packages/wrangler/src/__tests__/config/configuration.test.ts b/packages/wrangler/src/__tests__/config/configuration.test.ts index 605e80759245..4e7ff23b4cdc 100644 --- a/packages/wrangler/src/__tests__/config/configuration.test.ts +++ b/packages/wrangler/src/__tests__/config/configuration.test.ts @@ -136,6 +136,7 @@ describe("normalizeAndValidateConfig()", () => { data_blobs: undefined, workers_dev: undefined, preview_urls: undefined, + python_modules_excludes: ["**/*.pyc"], no_bundle: undefined, minify: undefined, first_party_worker: undefined, diff --git a/packages/wrangler/src/__tests__/deploy.test.ts b/packages/wrangler/src/__tests__/deploy.test.ts index 40a4445c55de..ecd3a13dea2e 100644 --- a/packages/wrangler/src/__tests__/deploy.test.ts +++ b/packages/wrangler/src/__tests__/deploy.test.ts @@ -13484,6 +13484,7 @@ export default{ writeWranglerConfig({ main: "src/index.py", compatibility_flags: ["python_workers"], + // python_modules_excludes is set to `**/*.pyc` by default }); // Create main Python file @@ -13503,6 +13504,16 @@ export default{ "# Python vendor module 2\nprint('hello')" ); + await fs.promises.writeFile( + "python_modules/test.pyc", + "this shouldn't be deployed" + ); + await fs.promises.mkdir("python_modules/other", { recursive: true }); + await fs.promises.writeFile( + "python_modules/other/test.pyc", + "this shouldn't be deployed" + ); + // Create a regular Python module await fs.promises.writeFile( "src/helper.py", @@ -13512,12 +13523,18 @@ export default{ const expectedModules = { "index.py": mainPython, "helper.py": "# Helper module\ndef helper(): pass", + "python_modules/module1.so": "binary content for module 1", + "python_modules/module2.py": "# Python vendor module 2\nprint('hello')", }; mockSubDomainRequest(); mockUploadWorkerRequest({ expectedMainModule: "index.py", expectedModules, + excludedModules: [ + "python_modules/test.pyc", + "python_modules/other/test.pyc", + ], }); await runWrangler("deploy"); diff --git a/packages/wrangler/src/__tests__/helpers/mock-upload-worker.ts b/packages/wrangler/src/__tests__/helpers/mock-upload-worker.ts index 0305ad0aba05..13ae1d8ad8d8 100644 --- a/packages/wrangler/src/__tests__/helpers/mock-upload-worker.ts +++ b/packages/wrangler/src/__tests__/helpers/mock-upload-worker.ts @@ -30,6 +30,7 @@ export function mockUploadWorkerRequest( expectedType?: "esm" | "sw" | "none"; expectedBindings?: unknown; expectedModules?: Record; + excludedModules?: string[]; expectedCompatibilityDate?: string; expectedCompatibilityFlags?: string[]; expectedMigrations?: CfWorkerInit["migrations"]; @@ -142,6 +143,9 @@ export function mockUploadWorkerRequest( for (const [name, content] of Object.entries(expectedModules)) { expect(await serialize(formBody.get(name))).toEqual(content); } + for (const name of excludedModules) { + expect(formBody.get(name)).toBeNull(); + } if (useOldUploadApi) { return HttpResponse.json( @@ -180,6 +184,7 @@ export function mockUploadWorkerRequest( expectedType = "esm", expectedBindings, expectedModules = {}, + excludedModules = [], expectedCompatibilityDate, expectedCompatibilityFlags, env = undefined, diff --git a/packages/wrangler/src/api/startDevWorker/BundlerController.ts b/packages/wrangler/src/api/startDevWorker/BundlerController.ts index d48a1c728fc8..e5c99f4f6039 100644 --- a/packages/wrangler/src/api/startDevWorker/BundlerController.ts +++ b/packages/wrangler/src/api/startDevWorker/BundlerController.ts @@ -106,7 +106,8 @@ export class BundlerController extends Controller { ? await noBundleWorker( entry, config.build.moduleRules, - this.#tmpDir.path + this.#tmpDir.path, + config.pythonModulesExcludes ?? [] ) : await bundleWorker(entry, this.#tmpDir.path, { bundle: true, @@ -279,6 +280,7 @@ export class BundlerController extends Controller { config.compatibilityDate, config.compatibilityFlags ), + pythonModulesExcludes: config.pythonModulesExcludes, }, (cb) => { const newBundle = cb(this.#currentBundle); diff --git a/packages/wrangler/src/api/startDevWorker/ConfigController.ts b/packages/wrangler/src/api/startDevWorker/ConfigController.ts index c432ec3ffd32..4e3c6bbde4ad 100644 --- a/packages/wrangler/src/api/startDevWorker/ConfigController.ts +++ b/packages/wrangler/src/api/startDevWorker/ConfigController.ts @@ -342,6 +342,8 @@ async function resolveConfig( compatibilityDate: getDevCompatibilityDate(config, input.compatibilityDate), compatibilityFlags: input.compatibilityFlags ?? config.compatibility_flags, complianceRegion: input.complianceRegion ?? config.compliance_region, + pythonModulesExcludes: + input.pythonModulesExcludes ?? config.python_modules_excludes, entrypoint: entry.file, projectRoot: entry.projectRoot, bindings, diff --git a/packages/wrangler/src/api/startDevWorker/types.ts b/packages/wrangler/src/api/startDevWorker/types.ts index 99df995d6fcd..2c4bd4e57393 100644 --- a/packages/wrangler/src/api/startDevWorker/types.ts +++ b/packages/wrangler/src/api/startDevWorker/types.ts @@ -88,6 +88,9 @@ export interface StartDevWorkerInput { /** Specify the compliance region mode of the Worker. */ complianceRegion?: Config["compliance_region"]; + /** A list of glob patterns to exclude files from the python_modules directory when bundling. */ + pythonModulesExcludes?: string[]; + env?: string; /** diff --git a/packages/wrangler/src/config/config.ts b/packages/wrangler/src/config/config.ts index 05b05d402390..cb384c258f7d 100644 --- a/packages/wrangler/src/config/config.ts +++ b/packages/wrangler/src/config/config.ts @@ -381,6 +381,7 @@ export const defaultWranglerConfig: Config = { observability: { enabled: true }, /** The default here is undefined so that we can delegate to the CLOUDFLARE_COMPLIANCE_REGION environment variable. */ compliance_region: undefined, + python_modules_excludes: ["**/*.pyc"], /** NON-INHERITABLE ENVIRONMENT FIELDS **/ define: {}, diff --git a/packages/wrangler/src/config/environment.ts b/packages/wrangler/src/config/environment.ts index 9780482d4710..e8af39f93f9a 100644 --- a/packages/wrangler/src/config/environment.ts +++ b/packages/wrangler/src/config/environment.ts @@ -589,6 +589,16 @@ interface EnvironmentInheritable { * it can be set to `undefined` in configuration to delegate to the CLOUDFLARE_COMPLIANCE_REGION environment variable. */ compliance_region: "public" | "fedramp_high" | undefined; + + /** + * A list of glob patterns to exclude files from the python_modules directory when bundling. + * + * Patterns are relative to the python_modules directory and use glob syntax. + * + * @default ["**\*.pyc"] + * @inheritable + */ + python_modules_excludes: string[]; } export type DurableObjectBindings = { diff --git a/packages/wrangler/src/config/validation.ts b/packages/wrangler/src/config/validation.ts index d56a2259645c..0a472eb59010 100644 --- a/packages/wrangler/src/config/validation.ts +++ b/packages/wrangler/src/config/validation.ts @@ -1591,6 +1591,14 @@ function normalizeAndValidateEnvironment( isOneOf("public", "fedramp_high"), undefined ), + python_modules_excludes: inheritable( + diagnostics, + topLevelEnv, + rawEnv, + "python_modules_excludes", + isStringArray, + ["**/*.pyc"] + ), }; warnIfDurableObjectsHaveNoMigrations( diff --git a/packages/wrangler/src/deploy/deploy.ts b/packages/wrangler/src/deploy/deploy.ts index 5453d7e8a6ef..a6cc3e6132cf 100644 --- a/packages/wrangler/src/deploy/deploy.ts +++ b/packages/wrangler/src/deploy/deploy.ts @@ -613,7 +613,12 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m bundleType, ...bundle } = props.noBundle - ? await noBundleWorker(props.entry, props.rules, props.outDir) + ? await noBundleWorker( + props.entry, + props.rules, + props.outDir, + config.python_modules_excludes + ) : await bundleWorker( props.entry, typeof destination === "string" ? destination : destination.path, diff --git a/packages/wrangler/src/deployment-bundle/find-additional-modules.ts b/packages/wrangler/src/deployment-bundle/find-additional-modules.ts index 77a4641225fa..9eba85e3a166 100644 --- a/packages/wrangler/src/deployment-bundle/find-additional-modules.ts +++ b/packages/wrangler/src/deployment-bundle/find-additional-modules.ts @@ -50,7 +50,7 @@ function isValidPythonPackageName(name: string): boolean { return regex.test(name); } -function filterPythonVendorModules( +function removePythonVendorModules( isPythonEntrypoint: boolean, modules: CfModule[] ): CfModule[] { @@ -74,7 +74,8 @@ function getPythonVendorModulesSize(modules: CfModule[]): number { export async function findAdditionalModules( entry: Entry, rules: Rule[] | ParsedRules, - attachSourcemaps = false + attachSourcemaps = false, + pythonModulesExcludes: string[] = [] ): Promise { const files = getFiles( entry.configPath, @@ -176,13 +177,24 @@ export async function findAdditionalModules( pythonModulesDir, parseRules(vendoredRules) ) - ).map((m) => { - const prefixedPath = path.join("python_modules", m.name); - return { - ...m, - name: prefixedPath, - }; - }); + ) + .filter((m) => { + // Check if the file matches any exclusion pattern + for (const pattern of pythonModulesExcludes) { + const regexp = globToRegExp(pattern, { globstar: true }); + if (regexp.test(m.name)) { + return false; // Exclude this file + } + } + return true; // Include this file + }) + .map((m) => { + const prefixedPath = path.join("python_modules", m.name); + return { + ...m, + name: prefixedPath, + }; + }); modules.push(...vendoredModules); } else { @@ -200,7 +212,7 @@ export async function findAdditionalModules( if (modules.length > 0) { logger.info(`Attaching additional modules:`); - const filteredModules = filterPythonVendorModules( + const filteredModules = removePythonVendorModules( isPythonEntrypoint, modules ); diff --git a/packages/wrangler/src/deployment-bundle/no-bundle-worker.ts b/packages/wrangler/src/deployment-bundle/no-bundle-worker.ts index 2438eddf856f..7e1d455d3593 100644 --- a/packages/wrangler/src/deployment-bundle/no-bundle-worker.ts +++ b/packages/wrangler/src/deployment-bundle/no-bundle-worker.ts @@ -9,9 +9,15 @@ import type { Entry } from "./entry"; export async function noBundleWorker( entry: Entry, rules: Rule[], - outDir: string | undefined + outDir: string | undefined, + pythonModulesExcludes: string[] = [] ) { - const modules = await findAdditionalModules(entry, rules); + const modules = await findAdditionalModules( + entry, + rules, + false, + pythonModulesExcludes + ); if (outDir) { await writeAdditionalModules(modules, outDir); } diff --git a/packages/wrangler/src/dev/use-esbuild.ts b/packages/wrangler/src/dev/use-esbuild.ts index 5cb3fd66f962..5d99cd2ff533 100644 --- a/packages/wrangler/src/dev/use-esbuild.ts +++ b/packages/wrangler/src/dev/use-esbuild.ts @@ -59,6 +59,7 @@ export function runBuild( onStart, defineNavigatorUserAgent, checkFetch, + pythonModulesExcludes, }: { entry: Entry; destination: string | undefined; @@ -86,6 +87,7 @@ export function runBuild( onStart: () => void; defineNavigatorUserAgent: boolean; checkFetch: boolean; + pythonModulesExcludes?: string[]; }, setBundle: ( cb: (previous: EsbuildBundle | undefined) => EsbuildBundle @@ -110,7 +112,12 @@ export function runBuild( async function getAdditionalModules() { return noBundle ? dedupeModulesByName([ - ...((await doFindAdditionalModules(entry, rules)) ?? []), + ...((await doFindAdditionalModules( + entry, + rules, + false, + pythonModulesExcludes ?? [] + )) ?? []), ...additionalModules, ]) : additionalModules; diff --git a/packages/wrangler/src/versions/upload.ts b/packages/wrangler/src/versions/upload.ts index badba4bcea83..82c9b8942ae6 100644 --- a/packages/wrangler/src/versions/upload.ts +++ b/packages/wrangler/src/versions/upload.ts @@ -588,7 +588,12 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m bundleType, ...bundle } = props.noBundle - ? await noBundleWorker(props.entry, props.rules, props.outDir) + ? await noBundleWorker( + props.entry, + props.rules, + props.outDir, + config.python_modules_excludes + ) : await bundleWorker( props.entry, typeof destination === "string" ? destination : destination.path,