handle folders in uploadFiles

Author: coyotte508

packages/hub/README.md

@@ -57,6 +57,8 @@ await hub.uploadFiles({
     },
     // Local file URL
     pathToFileURL("./pytorch-model.bin"),
+    // Local folder URL
+    pathToFileURL("./models"),
     // Web URL
     new URL("https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json"),
     // Path + Web URL

packages/hub/package.json

@@ -19,6 +19,7 @@
 	},
 	"browser": {
 		"./src/utils/sha256-node.ts": false,
+		"./src/utils/sub-paths.ts": false,
 		"./src/utils/FileBlob.ts": false,
 		"./src/lib/cache-management.ts": false,
 		"./src/lib/download-file-to-cache-dir.ts": false,

packages/hub/src/lib/commit.ts

@@ -18,10 +18,10 @@ import { promisesQueueStreaming } from "../utils/promisesQueueStreaming";
 import { sha256 } from "../utils/sha256";
 import { toRepoId } from "../utils/toRepoId";
 import { WebBlob } from "../utils/WebBlob";
-import { createBlob } from "../utils/createBlob";
 import { eventToGenerator } from "../utils/eventToGenerator";
 import { base64FromBytes } from "../utils/base64FromBytes";
 import { isFrontend } from "../utils/isFrontend";
+import { createBlobs } from "../utils/createBlobs";
 
 const CONCURRENT_SHAS = 5;
 const CONCURRENT_LFS_UPLOADS = 5;
@@ -73,9 +73,15 @@ export type CommitParams = {
 	/**
 	 * Whether to use web workers to compute SHA256 hashes.
 	 *
-	 * We load hash-wasm from a CDN inside the web worker. Not sure how to do otherwise and still have a "clean" bundle.
+	 * @default false
 	 */
 	useWebWorkers?: boolean | { minSize?: number; poolSize?: number };
+	/**
+	 * Maximum depth of folders to upload. Files deeper than this will be ignored
+	 *
+	 * @default 5
+	 */
+	maxFolderDepth?: number;
 	/**
 	 * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
 	 */
@@ -144,27 +150,32 @@ export async function* commitIter(params: CommitParams): AsyncGenerator<CommitPr
 	}
 
 	try {
-		const allOperations = await Promise.all(
-			params.operations.map(async (operation) => {
-				if (operation.operation !== "addOrUpdate") {
-					return operation;
-				}
+		const allOperations = (
+			await Promise.all(
+				params.operations.map(async (operation) => {
+					if (operation.operation !== "addOrUpdate") {
+						return operation;
+					}
 
-				if (!(operation.content instanceof URL)) {
-					/** TS trick to enforce `content` to be a `Blob` */
-					return { ...operation, content: operation.content };
-				}
+					if (!(operation.content instanceof URL)) {
+						/** TS trick to enforce `content` to be a `Blob` */
+						return { ...operation, content: operation.content };
+					}
 
-				const lazyBlob = await createBlob(operation.content, { fetch: params.fetch });
+					const lazyBlobs = await createBlobs(operation.content, {
+						fetch: params.fetch,
+						maxFolderDepth: params.maxFolderDepth,
+					});
 
-				abortSignal?.throwIfAborted();
+					abortSignal?.throwIfAborted();
 
-				return {
-					...operation,
-					content: lazyBlob,
-				};
-			})
-		);
+					return lazyBlobs.map((blob) => ({
+						...operation,
+						content: blob,
+					}));
+				})
+			)
+		).flat(1);
 
 		const gitAttributes = allOperations.filter(isFileOperation).find((op) => op.path === ".gitattributes")?.content;
 

packages/hub/src/lib/upload-files-with-progress.ts

@@ -28,6 +28,7 @@ export async function* uploadFilesWithProgress(
 		isPullRequest?: CommitParams["isPullRequest"];
 		parentCommit?: CommitParams["parentCommit"];
 		abortSignal?: CommitParams["abortSignal"];
+		maxFolderDepth?: CommitParams["maxFolderDepth"];
 		/**
 		 * Set this to true in order to have progress events for hashing
 		 */

packages/hub/src/lib/upload-files.ts

@@ -14,6 +14,7 @@ export function uploadFiles(
 		parentCommit?: CommitParams["parentCommit"];
 		fetch?: CommitParams["fetch"];
 		useWebWorkers?: CommitParams["useWebWorkers"];
+		maxFolderDepth?: CommitParams["maxFolderDepth"];
 		abortSignal?: CommitParams["abortSignal"];
 	} & Partial<CredentialsParams>
 ): Promise<CommitOutput> {

packages/hub/src/utils/createBlobs.ts

@@ -0,0 +1,33 @@
+import { WebBlob } from "./WebBlob";
+import { isFrontend } from "./isFrontend";
+
+/**
+ * This function allow to retrieve either a FileBlob or a WebBlob from a URL.
+ *
+ * From the backend:
+ *   - support local files
+ *   - support local folders
+ *   - support http resources with absolute URLs
+ *
+ * From the frontend:
+ *   - support http resources with absolute or relative URLs
+ */
+export async function createBlobs(url: URL, opts?: { fetch?: typeof fetch; maxFolderDepth?: number }): Promise<Blob[]> {
+	if (url.protocol === "http:" || url.protocol === "https:") {
+		return [await WebBlob.create(url, { fetch: opts?.fetch })];
+	}
+
+	if (isFrontend) {
+		throw new TypeError(`Unsupported URL protocol "${url.protocol}"`);
+	}
+
+	if (url.protocol === "file:") {
+		const { FileBlob } = await import("./FileBlob");
+		const { subPaths } = await import("./sub-paths");
+		const paths = await subPaths(url, opts?.maxFolderDepth);
+
+		return Promise.all(paths.map((path) => FileBlob.create(path)));
+	}
+
+	throw new TypeError(`Unsupported URL protocol "${url.protocol}"`);
+}

packages/hub/src/utils/sub-paths.ts

@@ -0,0 +1,24 @@
+import { readdir, stat } from "node:fs/promises";
+
+/**
+ * Recursively retrieves all sub-paths of a given directory up to a specified depth.
+ */
+export async function subPaths(path: URL, maxDepth = 10): Promise<URL[]> {
+	const state = await stat(path);
+	if (!state.isDirectory()) {
+		return [path];
+	}
+
+	const files = await readdir(path, { withFileTypes: true });
+	const ret: URL[] = [];
+	for (const file of files) {
+		const filePath = new URL(file.name, path);
+		if (file.isDirectory()) {
+			ret.push(...(await subPaths(filePath, maxDepth - 1)));
+		} else {
+			ret.push(filePath);
+		}
+	}
+
+	return ret;
+}