Merge remote-tracking branch 'origin/main' into provider-featherless-ai

Author: wxgeorge

README.md

@@ -97,7 +97,7 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
 
 ```html
 <script type="module">
-    import { InferenceClient } from 'https://cdn.jsdelivr.net/npm/@huggingface/inference@3.7.1/+esm';
+    import { InferenceClient } from 'https://cdn.jsdelivr.net/npm/@huggingface/inference@3.8.0/+esm';
     import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/[email protected]/+esm";
 </script>
 ```

packages/inference/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@huggingface/inference",
-	"version": "3.7.1",
+	"version": "3.8.0",
 	"packageManager": "[email protected]",
 	"license": "MIT",
 	"author": "Hugging Face and Tim Mikeladze <[email protected]>",

packages/inference/src/lib/getInferenceProviderMapping.ts

@@ -0,0 +1,96 @@
+import type { WidgetType } from "@huggingface/tasks";
+import type { InferenceProvider, ModelId } from "../types";
+import { HF_HUB_URL } from "../config";
+import { HARDCODED_MODEL_INFERENCE_MAPPING } from "../providers/consts";
+import { EQUIVALENT_SENTENCE_TRANSFORMERS_TASKS } from "../providers/hf-inference";
+import { typedInclude } from "../utils/typedInclude";
+
+export const inferenceProviderMappingCache = new Map<ModelId, InferenceProviderMapping>();
+
+export type InferenceProviderMapping = Partial<
+	Record<InferenceProvider, Omit<InferenceProviderModelMapping, "hfModelId" | "adapterWeightsPath">>
+>;
+
+export interface InferenceProviderModelMapping {
+	adapter?: string;
+	adapterWeightsPath?: string;
+	hfModelId: ModelId;
+	providerId: string;
+	status: "live" | "staging";
+	task: WidgetType;
+}
+
+export async function getInferenceProviderMapping(
+	params: {
+		accessToken?: string;
+		modelId: ModelId;
+		provider: InferenceProvider;
+		task: WidgetType;
+	},
+	options: {
+		fetch?: (input: RequestInfo, init?: RequestInit) => Promise<Response>;
+	}
+): Promise<InferenceProviderModelMapping | null> {
+	if (HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId]) {
+		return HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId];
+	}
+	let inferenceProviderMapping: InferenceProviderMapping | null;
+	if (inferenceProviderMappingCache.has(params.modelId)) {
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		inferenceProviderMapping = inferenceProviderMappingCache.get(params.modelId)!;
+	} else {
+		const resp = await (options?.fetch ?? fetch)(
+			`${HF_HUB_URL}/api/models/${params.modelId}?expand[]=inferenceProviderMapping`,
+			{
+				headers: params.accessToken?.startsWith("hf_") ? { Authorization: `Bearer ${params.accessToken}` } : {},
+			}
+		);
+		if (resp.status === 404) {
+			throw new Error(`Model ${params.modelId} does not exist`);
+		}
+		inferenceProviderMapping = await resp
+			.json()
+			.then((json) => json.inferenceProviderMapping)
+			.catch(() => null);
+	}
+
+	if (!inferenceProviderMapping) {
+		throw new Error(`We have not been able to find inference provider information for model ${params.modelId}.`);
+	}
+
+	const providerMapping = inferenceProviderMapping[params.provider];
+	if (providerMapping) {
+		const equivalentTasks =
+			params.provider === "hf-inference" && typedInclude(EQUIVALENT_SENTENCE_TRANSFORMERS_TASKS, params.task)
+				? EQUIVALENT_SENTENCE_TRANSFORMERS_TASKS
+				: [params.task];
+		if (!typedInclude(equivalentTasks, providerMapping.task)) {
+			throw new Error(
+				`Model ${params.modelId} is not supported for task ${params.task} and provider ${params.provider}. Supported task: ${providerMapping.task}.`
+			);
+		}
+		if (providerMapping.status === "staging") {
+			console.warn(
+				`Model ${params.modelId} is in staging mode for provider ${params.provider}. Meant for test purposes only.`
+			);
+		}
+		if (providerMapping.adapter === "lora") {
+			const treeResp = await (options?.fetch ?? fetch)(`${HF_HUB_URL}/api/models/${params.modelId}/tree/main`);
+			if (!treeResp.ok) {
+				throw new Error(`Unable to fetch the model tree for ${params.modelId}.`);
+			}
+			const tree: Array<{ type: "file" | "directory"; path: string }> = await treeResp.json();
+			const adapterWeightsPath = tree.find(({ type, path }) => type === "file" && path.endsWith(".safetensors"))?.path;
+			if (!adapterWeightsPath) {
+				throw new Error(`No .safetensors file found in the model tree for ${params.modelId}.`);
+			}
+			return {
+				...providerMapping,
+				hfModelId: params.modelId,
+				adapterWeightsPath,
+			};
+		}
+		return { ...providerMapping, hfModelId: params.modelId };
+	}
+	return null;
+}

packages/inference/src/lib/getProviderModelId.ts

@@ -1,8 +1,9 @@
 import { name as packageName, version as packageVersion } from "../../package.json";
 import { HF_HEADER_X_BILL_TO, HF_HUB_URL } from "../config";
 import type { InferenceTask, Options, RequestArgs } from "../types";
+import type { InferenceProviderModelMapping } from "./getInferenceProviderMapping";
+import { getInferenceProviderMapping } from "./getInferenceProviderMapping";
 import type { getProviderHelper } from "./getProviderHelper";
-import { getProviderModelId } from "./getProviderModelId";
 import { isUrl } from "./isUrl";
 
 /**
@@ -40,7 +41,13 @@ export async function makeRequestOptions(
 
 	if (args.endpointUrl) {
 		// No need to have maybeModel, or to load default model for a task
-		return makeRequestOptionsFromResolvedModel(maybeModel ?? args.endpointUrl, providerHelper, args, options);
+		return makeRequestOptionsFromResolvedModel(
+			maybeModel ?? args.endpointUrl,
+			providerHelper,
+			args,
+			undefined,
+			options
+		);
 	}
 
 	if (!maybeModel && !task) {
@@ -54,16 +61,38 @@ export async function makeRequestOptions(
 		throw new Error(`Provider ${provider} requires a model ID to be passed directly.`);
 	}
 
-	const resolvedModel = providerHelper.clientSideRoutingOnly
-		? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		  removeProviderPrefix(maybeModel!, provider)
-		: await getProviderModelId({ model: hfModel, provider }, args, {
-				task,
-				fetch: options?.fetch,
-		  });
+	const inferenceProviderMapping = providerHelper.clientSideRoutingOnly
+		? ({
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				providerId: removeProviderPrefix(maybeModel!, provider),
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				hfModelId: maybeModel!,
+				status: "live",
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				task: task!,
+		  } satisfies InferenceProviderModelMapping)
+		: await getInferenceProviderMapping(
+				{
+					modelId: hfModel,
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					task: task!,
+					provider,
+					accessToken: args.accessToken,
+				},
+				{ fetch: options?.fetch }
+		  );
+	if (!inferenceProviderMapping) {
+		throw new Error(`We have not been able to find inference provider information for model ${hfModel}.`);
+	}
 
 	// Use the sync version with the resolved model
-	return makeRequestOptionsFromResolvedModel(resolvedModel, providerHelper, args, options);
+	return makeRequestOptionsFromResolvedModel(
+		inferenceProviderMapping.providerId,
+		providerHelper,
+		args,
+		inferenceProviderMapping,
+		options
+	);
 }
 
 /**
@@ -77,6 +106,7 @@ export function makeRequestOptionsFromResolvedModel(
 		data?: Blob | ArrayBuffer;
 		stream?: boolean;
 	},
+	mapping: InferenceProviderModelMapping | undefined,
 	options?: Options & {
 		task?: InferenceTask;
 	}
@@ -138,6 +168,7 @@ export function makeRequestOptionsFromResolvedModel(
 		args: remainingArgs as Record<string, unknown>,
 		model: resolvedModel,
 		task,
+		mapping,
 	});
 	/**
 	 * For edge runtimes, leave 'credentials' undefined, otherwise cloudflare workers will error

packages/inference/src/lib/makeRequestOptions.ts

@@ -1,15 +1,18 @@
+import type { InferenceProviderModelMapping } from "../lib/getInferenceProviderMapping";
 import type { InferenceProvider } from "../types";
 import { type ModelId } from "../types";
 
-type ProviderId = string;
 /**
  * If you want to try to run inference for a new model locally before it's registered on huggingface.co
  * for a given Inference Provider,
  * you can add it to the following dictionary, for dev purposes.
  *
  * We also inject into this dictionary from tests.
  */
-export const HARDCODED_MODEL_ID_MAPPING: Record<InferenceProvider, Record<ModelId, ProviderId>> = {
+export const HARDCODED_MODEL_INFERENCE_MAPPING: Record<
+	InferenceProvider,
+	Record<ModelId, InferenceProviderModelMapping>
+> = {
 	/**
 	 * "HF model ID" => "Model ID on Inference Provider's side"
 	 *

packages/inference/src/providers/consts.ts

@@ -17,7 +17,7 @@
 import type { AutomaticSpeechRecognitionOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../lib/InferenceOutputError";
 import { isUrl } from "../lib/isUrl";
-import type { BodyParams, HeaderParams, UrlParams } from "../types";
+import type { BodyParams, HeaderParams, ModelId, UrlParams } from "../types";
 import { delay } from "../utils/delay";
 import { omit } from "../utils/omit";
 import {
@@ -26,6 +26,7 @@ import {
 	type TextToImageTaskHelper,
 	type TextToVideoTaskHelper,
 } from "./providerHelper";
+import { HF_HUB_URL } from "../config";
 
 export interface FalAiQueueOutput {
 	request_id: string;
@@ -74,14 +75,42 @@ abstract class FalAITask extends TaskProviderHelper {
 	}
 }
 
+function buildLoraPath(modelId: ModelId, adapterWeightsPath: string): string {
+	return `${HF_HUB_URL}/${modelId}/resolve/main/${adapterWeightsPath}`;
+}
+
 export class FalAITextToImageTask extends FalAITask implements TextToImageTaskHelper {
 	override preparePayload(params: BodyParams): Record<string, unknown> {
-		return {
+		const payload: Record<string, unknown> = {
 			...omit(params.args, ["inputs", "parameters"]),
 			...(params.args.parameters as Record<string, unknown>),
 			sync_mode: true,
 			prompt: params.args.inputs,
+			...(params.mapping?.adapter === "lora" && params.mapping.adapterWeightsPath
+				? {
+						loras: [
+							{
+								path: buildLoraPath(params.mapping.hfModelId, params.mapping.adapterWeightsPath),
+								scale: 1,
+							},
+						],
+				  }
+				: undefined),
 		};
+
+		if (params.mapping?.adapter === "lora" && params.mapping.adapterWeightsPath) {
+			payload.loras = [
+				{
+					path: buildLoraPath(params.mapping.hfModelId, params.mapping.adapterWeightsPath),
+					scale: 1,
+				},
+			];
+			if (params.mapping.providerId === "fal-ai/lora") {
+				payload.model_name = "stabilityai/stable-diffusion-xl-base-1.0";
+			}
+		}
+
+		return payload;
 	}
 
 	override async getResponse(response: FalAITextToImageOutput, outputType?: "url" | "blob"): Promise<string | Blob> {

packages/inference/src/providers/fal-ai.ts

@@ -87,6 +87,8 @@ interface AudioToAudioOutput {
 	label: string;
 }
 
+export const EQUIVALENT_SENTENCE_TRANSFORMERS_TASKS = ["feature-extraction", "sentence-similarity"] as const;
+
 export class HFInferenceTask extends TaskProviderHelper {
 	constructor() {
 		super("hf-inference", `${HF_ROUTER_URL}/hf-inference`);