Merge branch 'main' into asr

Author: Vaibhavs10

.github/workflows/test.yml

@@ -104,7 +104,7 @@ jobs:
         working-directory: e2e
         run: |
           sleep 3
-          pnpm i --filter root --filter inference... --filter hub... --frozen-lockfile
+          pnpm i --filter root --filter inference... --filter hub... --filter tasks-gen --frozen-lockfile
           pnpm --filter inference --filter hub --filter tasks publish --force --no-git-checks --registry http://localhost:4874/
 
       - name: E2E test - test yarn install

README.md

@@ -93,7 +93,7 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
 ```html
 <script type="module">
     import { HfInference } from 'https://cdn.jsdelivr.net/npm/@huggingface/[email protected]/+esm';
-    import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/hub@0.19.0/+esm";
+    import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/hub@0.20.0/+esm";
 </script>
 ```
 

package.json

@@ -23,6 +23,7 @@
 		"prettier": "^3.1.0",
 		"prettier-plugin-svelte": "^3.1.2",
 		"semver": "^7.5.0",
+		"tshy": "^3.0.2",
 		"tsup": "^6.7.0",
 		"tsx": "^4.7.0",
 		"typescript": "^5.4.2",

packages/gguf/src/gguf.spec.ts

@@ -33,7 +33,7 @@ describe("gguf", () => {
 			const arrayBuf = await res.arrayBuffer();
 			fs.writeFileSync(".cache/model.gguf", Buffer.from(arrayBuf));
 		}
-	});
+	}, 30_000);
 
 	it("should parse a llama2 7b", async () => {
 		const { metadata, tensorInfos } = await gguf(URL_LLAMA);

packages/hub/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/hub",
 	"packageManager": "[email protected]",
-	"version": "0.19.0",
+	"version": "0.20.0",
 	"description": "Utilities to interact with the Hugging Face hub",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/hub/src/lib/index.ts

@@ -19,6 +19,7 @@ export * from "./model-info";
 export * from "./oauth-handle-redirect";
 export * from "./oauth-login-url";
 export * from "./parse-safetensors-metadata";
+export * from "./paths-info";
 export * from "./space-info";
 export * from "./upload-file";
 export * from "./upload-files";

packages/hub/src/lib/paths-info.spec.ts

@@ -0,0 +1,75 @@
+import { expect, it, describe } from "vitest";
+import type { CommitInfo, PathInfo, SecurityFileStatus } from "./paths-info";
+import { pathsInfo } from "./paths-info";
+
+describe("pathsInfo", () => {
+	it("should fetch LFS path info", async () => {
+		const result: PathInfo[] = await pathsInfo({
+			repo: {
+				name: "bert-base-uncased",
+				type: "model",
+			},
+			paths: ["tf_model.h5"],
+			revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7",
+		});
+
+		expect(result).toHaveLength(1);
+
+		const modelPathInfo = result[0];
+		expect(modelPathInfo.path).toBe('tf_model.h5');
+		expect(modelPathInfo.type).toBe('file');
+		// lfs pointer, therefore lfs should be defined
+		expect(modelPathInfo?.lfs).toBeDefined();
+		expect(modelPathInfo?.lfs?.oid).toBe("a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2");
+		expect(modelPathInfo?.lfs?.size).toBe(536063208);
+		expect(modelPathInfo?.lfs?.pointerSize).toBe(134);
+
+		// should not include expand info
+		expect(modelPathInfo.lastCommit).toBeUndefined();
+		expect(modelPathInfo.securityFileStatus).toBeUndefined();
+	});
+
+	it("expand parmas should fetch lastCommit and securityFileStatus", async () => {
+		const result: (PathInfo & {
+			lastCommit: CommitInfo,
+			securityFileStatus: SecurityFileStatus,
+		})[] = await pathsInfo({
+			repo: {
+				name: "bert-base-uncased",
+				type: "model",
+			},
+			paths: ["tf_model.h5"],
+			revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7",
+			expand: true, // include
+		});
+
+		expect(result).toHaveLength(1);
+
+		const modelPathInfo = result[0];
+
+		// should include expand info
+		expect(modelPathInfo.lastCommit).toBeDefined();
+		expect(modelPathInfo.securityFileStatus).toBeDefined();
+
+		expect(modelPathInfo.lastCommit.id).toBe("dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7");
+		expect(modelPathInfo.lastCommit.title).toBe("Update tf_model.h5");
+		expect(modelPathInfo.lastCommit.date.getTime()).toBe(1569268124000); // 2019-09-23T19:48:44.000Z
+	});
+
+	it("non-LFS pointer should have lfs undefined", async () => {
+		const result: (PathInfo)[] = await pathsInfo({
+			repo: {
+				name: "bert-base-uncased",
+				type: "model",
+			},
+			paths: ["config.json"],
+			revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7",
+		});
+
+		expect(result).toHaveLength(1);
+
+		const modelPathInfo = result[0];
+		expect(modelPathInfo.path).toBe("config.json");
+		expect(modelPathInfo.lfs).toBeUndefined();
+	});
+});

packages/hub/src/lib/paths-info.ts

@@ -0,0 +1,120 @@
+import type { CredentialsParams, RepoDesignation } from "../types/public";
+import { checkCredentials } from "../utils/checkCredentials";
+import { toRepoId } from "../utils/toRepoId";
+import { HUB_URL } from "../consts";
+import { createApiError } from "../error";
+
+export interface LfsPathInfo {
+	"oid": string,
+	"size": number,
+	"pointerSize": number
+}
+
+export  interface CommitInfo {
+	"id": string,
+	"title": string,
+	"date": Date,
+}
+
+export interface SecurityFileStatus {
+	"status": string,
+}
+
+export interface PathInfo {
+	path: string,
+	type: string,
+	oid: string,
+	size: number,
+	/**
+	 * Only defined when path is LFS pointer
+	 */
+	lfs?: LfsPathInfo,
+	lastCommit?: CommitInfo,
+	securityFileStatus?: SecurityFileStatus
+}
+
+// Define the overloaded signatures
+export function pathsInfo(
+	params: {
+		repo: RepoDesignation;
+		paths: string[];
+		expand: true; // if expand true
+		revision?: string;
+		hubUrl?: string;
+		/**
+		 * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
+		 */
+		fetch?: typeof fetch;
+	}  & Partial<CredentialsParams>
+): Promise<(PathInfo & {lastCommit: CommitInfo, securityFileStatus: SecurityFileStatus })[]>;
+export function pathsInfo(
+	params: {
+		repo: RepoDesignation;
+		paths: string[];
+		expand?: boolean;
+		revision?: string;
+		hubUrl?: string;
+		/**
+		 * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
+		 */
+		fetch?: typeof fetch;
+	}  & Partial<CredentialsParams>
+): Promise<(PathInfo)[]>;
+
+export async function pathsInfo(
+	params: {
+		repo: RepoDesignation;
+		paths: string[];
+		expand?: boolean;
+		revision?: string;
+		hubUrl?: string;
+		/**
+		 * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
+		 */
+		fetch?: typeof fetch;
+	}  & Partial<CredentialsParams>
+): Promise<PathInfo[]> {
+	const accessToken = checkCredentials(params);
+	const repoId = toRepoId(params.repo);
+
+	const hubUrl = params.hubUrl ?? HUB_URL;
+
+	const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info/${encodeURIComponent(params.revision ?? "main")}`;
+
+	const resp = await (params.fetch ?? fetch)(url, {
+		method: "POST",
+		headers: {
+			...(params.credentials && {
+				Authorization: `Bearer ${accessToken}`,
+			}),
+			'Accept': 'application/json',
+			'Content-Type': 'application/json'
+		},
+		body: JSON.stringify({
+			paths: params.paths,
+			expand: params.expand,
+		}),
+	});
+
+	if (!resp.ok) {
+		throw await createApiError(resp);
+	}
+
+	const json: unknown = await resp.json();
+	if(!Array.isArray(json)) throw new Error('malformed response: expected array');
+
+	return json.map((item: PathInfo) => ({
+		path: item.path,
+		lfs: item.lfs,
+		type: item.type,
+		oid: item.oid,
+		size: item.size,
+		// expand fields
+		securityFileStatus: item.securityFileStatus,
+		lastCommit: item.lastCommit ? {
+			date: new Date(item.lastCommit.date),
+			title: item.lastCommit.title,
+			id: item.lastCommit.id,
+		}: undefined,
+	}));
+}

packages/tasks-gen/package.json

@@ -0,0 +1,26 @@
+{
+	"name": "@huggingface/tasks-gen",
+	"packageManager": "[email protected]",
+	"version": "0.13.1",
+	"description": "Utilities to generate files for @huggingface/tasks, and avoid unnecessary deps in the published package",
+	"repository": "https://github.com/huggingface/huggingface.js.git",
+	"private": true,
+	"scripts": {
+		"lint": "eslint --quiet --fix --ext .cjs,.ts .",
+		"lint:check": "eslint --ext .cjs,.ts .",
+		"format": "prettier --write .",
+		"format:check": "prettier --check .",
+		"check": "tsc",
+		"inference-codegen": "tsx scripts/inference-codegen.ts && prettier --write ../tasks/src/tasks/*/inference.ts",
+		"inference-tgi-import": "tsx scripts/inference-tgi-import.ts && prettier --write ../tasks/src/tasks/text-generation/spec/*.json && prettier --write ../tasks/src/tasks/chat-completion/spec/*.json",
+		"inference-tei-import": "tsx scripts/inference-tei-import.ts && prettier --write ../tasks/src/tasks/feature-extraction/spec/*.json"
+	},
+	"type": "module",
+	"author": "Hugging Face",
+	"license": "MIT",
+	"devDependencies": {
+		"@types/node": "^20.11.5",
+		"quicktype-core": "https://github.com/huggingface/quicktype/raw/pack-18.0.17/packages/quicktype-core/quicktype-core-18.0.17.tgz",
+		"type-fest": "^3.13.1"
+	}
+}