XetBlob util

CONTRIBUTING.md

@@ -18,7 +18,9 @@ It's not a hard requirement, but please consider using an icon from [Gitmoji](ht
 
 ## Tests
 
-If you want to run only specific tests, you can do `pnpm test -- -t "test name"`
+If you want to run only specific tests, you can do `pnpm test -- -t "test name"`.
+
+You can also do `npx vitest ./packages/hub/src/utils/XetBlob.spec.ts` to run a specific test file.
 
 ## Adding a package
 

packages/hub/README.md

@@ -174,3 +174,4 @@ Under the hood, `@huggingface/hub` uses a lazy blob implementation to load the f
 ## Dependencies
 
 - `@huggingface/tasks` : Typings only
+- `@huggingface/lz4` : URL join utility

packages/hub/package.json

@@ -59,9 +59,11 @@
 	"author": "Hugging Face",
 	"license": "MIT",
 	"devDependencies": {
+		"@types/lz4js": "^0.2.1",
 		"@types/node": "^20.11.28"
 	},
 	"dependencies": {
-		"@huggingface/tasks": "workspace:^"
+		"@huggingface/tasks": "workspace:^",
+		"lz4js": "^0.2.0"
 	}
 }

packages/hub/src/utils/WebBlob.ts

@@ -60,7 +60,7 @@ export class WebBlob extends Blob {
 
 	override slice(start = 0, end = this.size): WebBlob {
 		if (start < 0 || end < 0) {
-			new TypeError("Unsupported negative start/end on FileBlob.slice");
+			new TypeError("Unsupported negative start/end on WebBlob.slice");
 		}
 
 		const slice = new WebBlob(

packages/hub/src/utils/XetBlob.spec.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from "vitest";
+import { XetBlob } from "./XetBlob";
+
+describe("XetBlob", () => {
+	it("should lazy load the first 22 bytes", async () => {
+		const blob = new XetBlob({
+			repo: {
+				type: "model",
+				name: "celinah/xet-experiments",
+			},
+			hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b",
+			size: 5_234_139_343,
+		});
+
+		expect(await blob.slice(10, 22).text()).toBe("__metadata__");
+	}, 30_000);
+});