Merge branch 'main' into v2ark/add_centml

Author: hanouticelina

packages/inference/src/lib/getInferenceProviderMapping.ts

@@ -1,8 +1,8 @@
 import type { WidgetType } from "@huggingface/tasks";
-import type { InferenceProvider, ModelId } from "../types";
 import { HF_HUB_URL } from "../config";
 import { HARDCODED_MODEL_INFERENCE_MAPPING } from "../providers/consts";
 import { EQUIVALENT_SENTENCE_TRANSFORMERS_TASKS } from "../providers/hf-inference";
+import type { InferenceProvider, InferenceProviderOrPolicy, ModelId } from "../types";
 import { typedInclude } from "../utils/typedInclude";
 
 export const inferenceProviderMappingCache = new Map<ModelId, InferenceProviderMapping>();
@@ -20,44 +20,62 @@ export interface InferenceProviderModelMapping {
 	task: WidgetType;
 }
 
-export async function getInferenceProviderMapping(
-	params: {
-		accessToken?: string;
-		modelId: ModelId;
-		provider: InferenceProvider;
-		task: WidgetType;
-	},
-	options: {
+export async function fetchInferenceProviderMappingForModel(
+	modelId: ModelId,
+	accessToken?: string,
+	options?: {
 		fetch?: (input: RequestInfo, init?: RequestInit) => Promise<Response>;
 	}
-): Promise<InferenceProviderModelMapping | null> {
-	if (HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId]) {
-		return HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId];
-	}
+): Promise<InferenceProviderMapping> {
 	let inferenceProviderMapping: InferenceProviderMapping | null;
-	if (inferenceProviderMappingCache.has(params.modelId)) {
+	if (inferenceProviderMappingCache.has(modelId)) {
 		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		inferenceProviderMapping = inferenceProviderMappingCache.get(params.modelId)!;
+		inferenceProviderMapping = inferenceProviderMappingCache.get(modelId)!;
 	} else {
 		const resp = await (options?.fetch ?? fetch)(
-			`${HF_HUB_URL}/api/models/${params.modelId}?expand[]=inferenceProviderMapping`,
+			`${HF_HUB_URL}/api/models/${modelId}?expand[]=inferenceProviderMapping`,
 			{
-				headers: params.accessToken?.startsWith("hf_") ? { Authorization: `Bearer ${params.accessToken}` } : {},
+				headers: accessToken?.startsWith("hf_") ? { Authorization: `Bearer ${accessToken}` } : {},
 			}
 		);
 		if (resp.status === 404) {
-			throw new Error(`Model ${params.modelId} does not exist`);
+			throw new Error(`Model ${modelId} does not exist`);
 		}
 		inferenceProviderMapping = await resp
 			.json()
 			.then((json) => json.inferenceProviderMapping)
 			.catch(() => null);
+
+		if (inferenceProviderMapping) {
+			inferenceProviderMappingCache.set(modelId, inferenceProviderMapping);
+		}
 	}
 
 	if (!inferenceProviderMapping) {
-		throw new Error(`We have not been able to find inference provider information for model ${params.modelId}.`);
+		throw new Error(`We have not been able to find inference provider information for model ${modelId}.`);
 	}
+	return inferenceProviderMapping;
+}
 
+export async function getInferenceProviderMapping(
+	params: {
+		accessToken?: string;
+		modelId: ModelId;
+		provider: InferenceProvider;
+		task: WidgetType;
+	},
+	options: {
+		fetch?: (input: RequestInfo, init?: RequestInit) => Promise<Response>;
+	}
+): Promise<InferenceProviderModelMapping | null> {
+	if (HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId]) {
+		return HARDCODED_MODEL_INFERENCE_MAPPING[params.provider][params.modelId];
+	}
+	const inferenceProviderMapping = await fetchInferenceProviderMappingForModel(
+		params.modelId,
+		params.accessToken,
+		options
+	);
 	const providerMapping = inferenceProviderMapping[params.provider];
 	if (providerMapping) {
 		const equivalentTasks =
@@ -78,3 +96,23 @@ export async function getInferenceProviderMapping(
 	}
 	return null;
 }
+
+export async function resolveProvider(
+	provider?: InferenceProviderOrPolicy,
+	modelId?: string
+): Promise<InferenceProvider> {
+	if (!provider) {
+		console.log(
+			"Defaulting to 'auto' which will select the first provider available for the model, sorted by the user's order in https://hf.co/settings/inference-providers."
+		);
+		provider = "auto";
+	}
+	if (provider === "auto") {
+		if (!modelId) {
+			throw new Error("Specifying a model is required when provider is 'auto'");
+		}
+		const inferenceProviderMapping = await fetchInferenceProviderMappingForModel(modelId);
+		provider = Object.keys(inferenceProviderMapping)[0] as InferenceProvider;
+	}
+	return provider;
+}

packages/inference/src/lib/makeRequestOptions.ts

@@ -27,8 +27,8 @@ export async function makeRequestOptions(
 		task?: InferenceTask;
 	}
 ): Promise<{ url: string; info: RequestInit }> {
-	const { provider: maybeProvider, model: maybeModel } = args;
-	const provider = maybeProvider ?? "hf-inference";
+	const { model: maybeModel } = args;
+	const provider = providerHelper.provider;
 	const { task } = options ?? {};
 
 	// Validate inputs
@@ -113,8 +113,9 @@ export function makeRequestOptionsFromResolvedModel(
 ): { url: string; info: RequestInit } {
 	const { accessToken, endpointUrl, provider: maybeProvider, model, ...remainingArgs } = args;
 	void model;
+	void maybeProvider;
 
-	const provider = maybeProvider ?? "hf-inference";
+	const provider = providerHelper.provider;
 
 	const { includeCredentials, task, signal, billTo } = options ?? {};
 	const authMethod = (() => {

packages/inference/src/providers/providerHelper.ts

@@ -56,7 +56,7 @@ import { toArray } from "../utils/toArray";
  */
 export abstract class TaskProviderHelper {
 	constructor(
-		private provider: InferenceProvider,
+		readonly provider: InferenceProvider,
 		private baseUrl: string,
 		readonly clientSideRoutingOnly: boolean = false
 	) {}

packages/inference/src/snippets/getInferenceSnippets.ts

@@ -272,7 +272,7 @@ const prepareConversationalInput = (
 	return {
 		messages: opts?.messages ?? getModelInputSnippet(model),
 		...(opts?.temperature ? { temperature: opts?.temperature } : undefined),
-		max_tokens: opts?.max_tokens ?? 512,
+		...(opts?.max_tokens ? { max_tokens: opts?.max_tokens } : undefined),
 		...(opts?.top_p ? { top_p: opts?.top_p } : undefined),
 	};
 };

packages/inference/src/tasks/audio/audioClassification.ts

@@ -1,4 +1,5 @@
 import type { AudioClassificationInput, AudioClassificationOutput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -15,7 +16,8 @@ export async function audioClassification(
 	args: AudioClassificationArgs,
 	options?: Options
 ): Promise<AudioClassificationOutput> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "audio-classification");
+	const provider = await resolveProvider(args.provider, args.model);
+	const providerHelper = getProviderHelper(provider, "audio-classification");
 	const payload = preparePayload(args);
 	const { data: res } = await innerRequest<AudioClassificationOutput>(payload, providerHelper, {
 		...options,

packages/inference/src/tasks/audio/audioToAudio.ts

@@ -1,3 +1,4 @@
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -36,7 +37,9 @@ export interface AudioToAudioOutput {
  * Example model: speechbrain/sepformer-wham does audio source separation.
  */
 export async function audioToAudio(args: AudioToAudioArgs, options?: Options): Promise<AudioToAudioOutput[]> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "audio-to-audio");
+	const model = "inputs" in args ? args.model : undefined;
+	const provider = await resolveProvider(args.provider, model);
+	const providerHelper = getProviderHelper(provider, "audio-to-audio");
 	const payload = preparePayload(args);
 	const { data: res } = await innerRequest<AudioToAudioOutput>(payload, providerHelper, {
 		...options,

packages/inference/src/tasks/audio/automaticSpeechRecognition.ts

@@ -1,4 +1,5 @@
 import type { AutomaticSpeechRecognitionInput, AutomaticSpeechRecognitionOutput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import { FAL_AI_SUPPORTED_BLOB_TYPES } from "../../providers/fal-ai";
@@ -18,7 +19,8 @@ export async function automaticSpeechRecognition(
 	args: AutomaticSpeechRecognitionArgs,
 	options?: Options
 ): Promise<AutomaticSpeechRecognitionOutput> {
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", "automatic-speech-recognition");
+	const provider = await resolveProvider(args.provider, args.model);
+	const providerHelper = getProviderHelper(provider, "automatic-speech-recognition");
 	const payload = await buildPayload(args);
 	const { data: res } = await innerRequest<AutomaticSpeechRecognitionOutput>(payload, providerHelper, {
 		...options,

packages/inference/src/tasks/audio/textToSpeech.ts

@@ -1,4 +1,5 @@
 import type { TextToSpeechInput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { BaseArgs, Options } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -12,7 +13,7 @@ interface OutputUrlTextToSpeechGeneration {
  * Recommended model: espnet/kan-bayashi_ljspeech_vits
  */
 export async function textToSpeech(args: TextToSpeechArgs, options?: Options): Promise<Blob> {
-	const provider = args.provider ?? "hf-inference";
+	const provider = await resolveProvider(args.provider, args.model);
 	const providerHelper = getProviderHelper(provider, "text-to-speech");
 	const { data: res } = await innerRequest<Blob | OutputUrlTextToSpeechGeneration>(args, providerHelper, {
 		...options,

packages/inference/src/tasks/custom/request.ts

@@ -1,3 +1,4 @@
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { InferenceTask, Options, RequestArgs } from "../../types";
 import { innerRequest } from "../../utils/request";
@@ -16,7 +17,8 @@ export async function request<T>(
 	console.warn(
 		"The request method is deprecated and will be removed in a future version of huggingface.js. Use specific task functions instead."
 	);
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", options?.task);
+	const provider = await resolveProvider(args.provider, args.model);
+	const providerHelper = getProviderHelper(provider, options?.task);
 	const result = await innerRequest<T>(args, providerHelper, options);
 	return result.data;
 }

packages/inference/src/tasks/custom/streamingRequest.ts

@@ -1,3 +1,4 @@
+import { resolveProvider } from "../../lib/getInferenceProviderMapping";
 import { getProviderHelper } from "../../lib/getProviderHelper";
 import type { InferenceTask, Options, RequestArgs } from "../../types";
 import { innerStreamingRequest } from "../../utils/request";
@@ -16,6 +17,7 @@ export async function* streamingRequest<T>(
 	console.warn(
 		"The streamingRequest method is deprecated and will be removed in a future version of huggingface.js. Use specific task functions instead."
 	);
-	const providerHelper = getProviderHelper(args.provider ?? "hf-inference", options?.task);
+	const provider = await resolveProvider(args.provider, args.model);
+	const providerHelper = getProviderHelper(provider, options?.task);
 	yield* innerStreamingRequest(args, providerHelper, options);
 }