Merge branch 'main' into inference-providers

Author: SBrandeis

.github/workflows/inference-publish.yml

@@ -54,7 +54,7 @@ jobs:
           git tag "inference-v$BUMPED_VERSION"
 
       - name: "Check Deps are published before publishing this package"
-        run: pnpm -w check-deps gguf
+        run: pnpm -w check-deps tasks
 
       - run: pnpm publish --no-git-checks .
         env:

packages/gguf/src/gguf.spec.ts

@@ -283,4 +283,9 @@ describe("gguf", () => {
 		expect(parseGGUFQuantLabel("Codestral-22B-v0.1-IQ3_XS.gguf")).toEqual(undefined); // TODO: investigate IQ3_XS
 		expect(parseGGUFQuantLabel("Codestral-22B-v0.1-Q4_0_4_4.gguf")).toEqual("Q4_0"); // TODO: investigate Q4_0_4_4
 	});
+
+	it("calculate tensor data offset", async () => {
+		const { tensorDataOffset } = await gguf(URL_LLAMA);
+		expect(tensorDataOffset).toEqual(741056n);
+	});
 });

packages/gguf/src/gguf.ts

@@ -10,6 +10,8 @@ export { parseGGUFQuantLabel, GGUF_QUANT_RE, GGUF_QUANT_RE_GLOBAL } from "@huggi
 
 export const RE_GGUF_FILE = /\.gguf$/;
 export const RE_GGUF_SHARD_FILE = /^(?<prefix>.*?)-(?<shard>\d{5})-of-(?<total>\d{5})\.gguf$/;
+const GGUF_DEFAULT_ALIGNMENT = 32; // defined in ggml.h
+const GGML_PAD = (x: number, n: number) => (x + n - 1) & ~(n - 1); // defined in ggml.h
 const PARALLEL_DOWNLOADS = 20;
 
 export interface GgufShardFileInfo {
@@ -384,14 +386,18 @@ export async function gguf(
 		});
 	}
 
+	// calculate absolute offset of tensor data
+	const alignment: number = Number(metadata["general.alignment"] ?? GGUF_DEFAULT_ALIGNMENT);
+	const tensorDataOffset = BigInt(GGML_PAD(offset, alignment));
+
 	if (params?.computeParametersCount) {
 		const parameterCount = tensorInfos
 			.map(({ shape }) => shape.reduce((acc, val) => acc * Number(val), 1))
 			.reduce((acc, val) => acc + val, 0);
 
-		return { metadata, tensorInfos, parameterCount };
+		return { metadata, tensorInfos, tensorDataOffset, parameterCount };
 	} else {
-		return { metadata, tensorInfos };
+		return { metadata, tensorInfos, tensorDataOffset };
 	}
 }
 
@@ -429,7 +435,10 @@ export async function ggufAllShards(
 			parameterCount: shards.map(({ parameterCount }) => parameterCount).reduce((acc, val) => acc + val, 0),
 		};
 	} else {
-		const { metadata, tensorInfos, parameterCount } = await gguf(url, { ...params, computeParametersCount: true });
-		return { shards: [{ metadata, tensorInfos }], parameterCount };
+		const { metadata, tensorInfos, tensorDataOffset, parameterCount } = await gguf(url, {
+			...params,
+			computeParametersCount: true,
+		});
+		return { shards: [{ metadata, tensorInfos, tensorDataOffset }], parameterCount };
 	}
 }

packages/gguf/src/types.ts

@@ -155,4 +155,5 @@ export interface GGUFTensorInfo {
 export interface GGUFParseOutput<Options extends GGUFMetadataOptions = { strict: true }> {
 	metadata: GGUFMetadata<Options>;
 	tensorInfos: GGUFTensorInfo[];
+	tensorDataOffset: bigint;
 }

packages/tasks/src/hardware.ts

@@ -307,6 +307,24 @@ export const SKUS = {
 				memory: [16],
 			},
 		},
+		INTEL: {
+			"Arc A750": {
+				tflops: 34.41,
+				memory: [8],
+			},
+			"Arc A770": {
+				tflops: 39.32,
+				memory: [8, 16],
+			},
+			"Arc B570": {
+				tflops: 23.04,
+				memory: [10],
+			},
+			"Arc B580": {
+				tflops: 27.34,
+				memory: [12],
+			},
+		},
 		QUALCOMM: {
 			"Snapdragon X Elite X1E-00-1DE": {
 				tflops: 4.6,
@@ -485,7 +503,7 @@ export const SKUS = {
 			},
 			"Apple M4 Pro": {
 				tflops: 9.2,
-				memory: [24, 48],
+				memory: [24, 48, 64],
 			},
 			"Apple M4 Max": {
 				tflops: 18.4,

packages/tasks/src/model-libraries.ts

@@ -166,6 +166,12 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		filter: false,
 		countDownloads: `path:"adapter_config.json"`,
 	},
+	cosmos: {
+		prettyLabel: "Cosmos",
+		repoName: "Cosmos",
+		repoUrl: "https://github.com/NVIDIA/Cosmos",
+		countDownloads: `path:"config.json" OR path_extension:"pt"`,
+	},
 	"cxr-foundation": {
 		prettyLabel: "CXR Foundation",
 		repoName: "cxr-foundation",
@@ -249,6 +255,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		filter: false,
 		countDownloads: `path_extension:"pt"`,
 	},
+	cosyvoice: {
+		prettyLabel: "CosyVoice",
+		repoName: "CosyVoice",
+		repoUrl: "https://github.com/FunAudioLLM/CosyVoice",
+		filter: false,
+		countDownloads: `path_extension:"onnx" OR path_extension:"pt"`,	
+	},
 	cotracker: {
 		prettyLabel: "CoTracker",
 		repoName: "CoTracker",