more failures

Author: coyotte508

packages/hub/src/utils/XetBlob.spec.ts

@@ -101,6 +101,55 @@ describe("XetBlob", () => {
 		expect(new Uint8Array(xetDownload)).toEqual(new Uint8Array(bridgeDownload));
 	});
 
+	it("should load text correctly when offset_into_range starts in a chunk further than the first", async () => {
+		const blob = new XetBlob({
+			repo: {
+				type: "model",
+				name: "celinah/xet-experiments",
+			},
+			hash: "794efea76d8cb372bbe1385d9e51c3384555f3281e629903ecb6abeff7d54eec",
+			size: 62_914_580,
+		});
+
+		// Reconstruction info
+		// {
+		// 	"offset_into_first_range": 600000,
+		// 	"terms":
+		// 		[
+		// 			{
+		// 				"hash": "be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2",
+		// 				"unpacked_length": 655360,
+		// 				"range": { "start": 0, "end": 5 },
+		// 			},
+		// 			{
+		// 				"hash": "be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2",
+		// 				"unpacked_length": 655360,
+		// 				"range": { "start": 0, "end": 5 },
+		// 			},
+		// 		],
+		// 	"fetch_info":
+		// 		{
+		// 			"be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2":
+		// 				[
+		// 					{
+		// 						"range": { "start": 0, "end": 5 },
+		// 						"url": "...",
+		// 						"url_range": { "start": 0, "end": 2839 },
+		// 					},
+		// 				],
+		// 		},
+		// }
+
+		const text = await blob.slice(600_000, 700_000).text();
+		const bridgeDownload = await fetch("https://huggingface.co/celinah/xet-experiments/resolve/main/large_text.txt", {
+			headers: {
+				Range: "bytes=600000-699999",
+			},
+		}).then((res) => res.text());
+
+		expect(text.length).toBe(bridgeDownload.length);
+	});
+
 	describe("bg4_regoup_bytes", () => {
 		it("should regroup bytes when the array is %4 length", () => {
 			expect(bg4_regoup_bytes(new Uint8Array([1, 5, 2, 6, 3, 7, 4, 8]))).toEqual(

packages/hub/src/utils/XetBlob.ts

@@ -183,6 +183,7 @@ export class XetBlob extends Blob {
 			let readBytesToSkip = reconstructionInfo.offset_into_first_range;
 
 			for (const term of reconstructionInfo.terms) {
+				// console.log("term", term, readBytesToSkip, totalBytesRead);
 				if (totalBytesRead >= maxBytes) {
 					break;
 				}
@@ -277,14 +278,14 @@ export class XetBlob extends Blob {
 
 							if (chunksToSkip) {
 								chunksToSkip--;
-								leftoverBytes = result.value.slice(CHUNK_HEADER_BYTES);
+								result.value = result.value.slice(CHUNK_HEADER_BYTES);
 								bytesToSkip = chunkHeader.compressed_length;
 								continue;
 							}
 
 							if (readBytesToSkip >= chunkHeader.uncompressed_length) {
 								readBytesToSkip -= chunkHeader.uncompressed_length;
-								leftoverBytes = result.value.slice(CHUNK_HEADER_BYTES);
+								result.value = result.value.slice(CHUNK_HEADER_BYTES);
 								bytesToSkip = chunkHeader.compressed_length;
 								chunksToRead--;
 								continue;