feat: experimental workers assets can be ignored by adding a .assetsignore file (#6640)

* feat: experimental workers assets can be ignored by adding a .cfassetsignore file

* fixup! feat: experimental workers assets can be ignored by adding a .cfassetsignore file

Author: petebacondarwin

.changeset/nasty-hats-rhyme.md

@@ -0,0 +1,10 @@
+---
+"wrangler": minor
+---
+
+feat: experimental workers assets can be ignored by adding a .assetsignore file
+
+This file can be added to the root of the assets directory that is to be uploaded alongside the Worker
+when using `experimental_assets`.
+
+The file follows the `.gitignore` syntax, and any matching paths will not be included in the upload.

.vscode/settings.json

@@ -2,6 +2,7 @@
 	"editor.defaultFormatter": "esbenp.prettier-vscode",
 	"cSpell.words": [
 		"Abortable",
+		"assetsignore",
 		"cfetch",
 		"chatgpt",
 		"clipboardy",
@@ -11,6 +12,7 @@
 		"esbuild",
 		"eslintcache",
 		"execa",
+		"filestat",
 		"haikunate",
 		"haikunator",
 		"httplogs",

packages/wrangler/src/__tests__/deploy.test.ts

@@ -4381,6 +4381,47 @@ addEventListener('fetch', event => {});`
 			});
 		});
 
+		it("should ignore assets that match patterns in an .assetsignore file in the root of the assets directory", async () => {
+			const assets = [
+				{ filePath: ".assetsignore", content: "*.bak\nsub-dir" },
+				{ filePath: "file-1.txt", content: "Content of file-1" },
+				{ filePath: "file-2.bak", content: "Content of file-2" },
+				{ filePath: "file-3.txt", content: "Content of file-3" },
+				{ filePath: "sub-dir/file-4.bak", content: "Content of file-4" },
+				{ filePath: "sub-dir/file-5.txt", content: "Content of file-5" },
+			];
+			writeAssets(assets, "some/path/assets");
+			writeWranglerToml(
+				{
+					experimental_assets: { directory: "assets" },
+				},
+				"some/path/wrangler.toml"
+			);
+			const bodies: AssetManifest[] = [];
+			await mockAUSRequest(bodies);
+			mockSubDomainRequest();
+			mockUploadWorkerRequest({
+				expectedExperimentalAssets: true,
+				expectedType: "none",
+			});
+			await runWrangler("deploy --config some/path/wrangler.toml");
+			expect(bodies.length).toBe(1);
+			expect(bodies[0]).toMatchInlineSnapshot(`
+				Object {
+				  "manifest": Object {
+				    "/file-1.txt": Object {
+				      "hash": "0de3dd5df907418e9730fd2bd747bd5e",
+				      "size": 17,
+				    },
+				    "/file-3.txt": Object {
+				      "hash": "ff5016e92f039aa743a4ff7abb3180fa",
+				      "size": 17,
+				    },
+				  },
+				}
+			`);
+		});
+
 		it("should resolve assets directory relative to cwd if using cli", async () => {
 			const assets = [{ filePath: "file-1.txt", content: "Content of file-1" }];
 			writeAssets(assets, "some/path/assets");

packages/wrangler/src/experimental-assets.ts

@@ -14,6 +14,7 @@ import { logger, LOGGER_LEVELS } from "./logger";
 import { hashFile } from "./pages/hash";
 import { isJwtExpired } from "./pages/upload";
 import { APIError } from "./parse";
+import { createPatternMatcher } from "./utils/filesystem";
 import type { Config } from "./config";
 import type { ExperimentalAssets } from "./config/environment";
 
@@ -220,10 +221,20 @@ export const buildAssetsManifest = async (dir: string) => {
 	const files = await readdir(dir, { recursive: true });
 	const manifest: AssetManifest = {};
 	let counter = 0;
+
+	const ignoreFn = await createAssetIgnoreFunction(dir);
+
 	await Promise.all(
 		files.map(async (file) => {
 			const filepath = path.join(dir, file);
 			const relativeFilepath = path.relative(dir, filepath);
+
+			if (ignoreFn?.(relativeFilepath)) {
+				logger.debug("Ignoring asset:", relativeFilepath);
+				// This file should not be included in the manifest.
+				return;
+			}
+
 			const filestat = await stat(filepath);
 
 			if (filestat.isSymbolicLink() || filestat.isDirectory()) {
@@ -361,3 +372,28 @@ const decodeFilepath = (filePath: string) => {
 		.map((segment) => decodeURIComponent(segment))
 		.join(path.sep);
 };
+
+/**
+ * Create a function for filtering out ignored assets.
+ *
+ * The generated function takes an asset path, relative to the asset directory,
+ * and returns true if the asset should not be ignored.
+ */
+async function createAssetIgnoreFunction(dir: string) {
+	const CF_ASSETS_IGNORE_FILENAME = ".assetsignore";
+
+	const cfAssetIgnorePath = path.resolve(dir, CF_ASSETS_IGNORE_FILENAME);
+
+	if (!existsSync(cfAssetIgnorePath)) {
+		return null;
+	}
+
+	const ignorePatterns = (
+		await readFile(cfAssetIgnorePath, { encoding: "utf8" })
+	).split("\n");
+
+	// Always ignore the `.assetsignore` file.
+	ignorePatterns.push(CF_ASSETS_IGNORE_FILENAME);
+
+	return createPatternMatcher(ignorePatterns, true);
+}

packages/wrangler/src/sites.ts

@@ -2,7 +2,6 @@ import assert from "node:assert";
 import { readdir, readFile, stat } from "node:fs/promises";
 import * as path from "node:path";
 import chalk from "chalk";
-import ignore from "ignore";
 import xxhash from "xxhash-wasm";
 import { UserError } from "./errors";
 import {
@@ -17,6 +16,7 @@ import {
 	putKVKeyValue,
 } from "./kv/helpers";
 import { logger, LOGGER_LEVELS } from "./logger";
+import { createPatternMatcher } from "./utils/filesystem";
 import type { Config } from "./config";
 import type { KeyValue } from "./kv/helpers";
 import type { XXHashAPI } from "xxhash-wasm";
@@ -391,18 +391,6 @@ export async function syncLegacyAssets(
 	return { manifest, namespace };
 }
 
-function createPatternMatcher(
-	patterns: string[],
-	exclude: boolean
-): (filePath: string) => boolean {
-	if (patterns.length === 0) {
-		return (_filePath) => !exclude;
-	} else {
-		const ignorer = ignore().add(patterns);
-		return (filePath) => ignorer.test(filePath).ignored;
-	}
-}
-
 /**
  * validate that the passed-in file is below 25 MiB
  * **PRIOR** to base64 encoding. 25 MiB is a KV limit

packages/wrangler/src/utils/filesystem.ts

@@ -1,6 +1,7 @@
 import { mkdirSync } from "fs";
 import { mkdir } from "fs/promises";
 import path from "path";
+import ignore from "ignore";
 
 export async function ensureDirectoryExists(filepath: string) {
 	const dirpath = path.dirname(filepath);
@@ -13,3 +14,18 @@ export function ensureDirectoryExistsSync(filepath: string) {
 
 	mkdirSync(dirpath, { recursive: true });
 }
+
+/**
+ * Generate a function that can match relative filepaths against a list of gitignore formatted patterns.
+ */
+export function createPatternMatcher(
+	patterns: string[],
+	exclude: boolean
+): (filePath: string) => boolean {
+	if (patterns.length === 0) {
+		return (_filePath) => !exclude;
+	} else {
+		const ignorer = ignore().add(patterns);
+		return (filePath) => ignorer.test(filePath).ignored;
+	}
+}