Merge branch 'main' into fix-inference-search-param

Author: Wauplin

README.md

@@ -97,7 +97,7 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
 
 ```html
 <script type="module">
-    import { InferenceClient } from 'https://cdn.jsdelivr.net/npm/@huggingface/[email protected].0/+esm';
+    import { InferenceClient } from 'https://cdn.jsdelivr.net/npm/@huggingface/[email protected].3/+esm';
     import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/[email protected]/+esm";
 </script>
 ```

packages/inference/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@huggingface/inference",
-	"version": "4.0.0",
+	"version": "4.0.3",
 	"packageManager": "[email protected]",
 	"license": "MIT",
 	"author": "Hugging Face and Tim Mikeladze <[email protected]>",

packages/inference/src/package.ts

@@ -1,3 +1,3 @@
 // Generated file from package.json. Issues importing JSON directly when publishing on commonjs/ESM - see https://github.com/microsoft/TypeScript/issues/51783
-export const PACKAGE_VERSION = "4.0.0";
+export const PACKAGE_VERSION = "4.0.3";
 export const PACKAGE_NAME = "@huggingface/inference";

packages/inference/src/snippets/getInferenceSnippets.ts

@@ -14,10 +14,12 @@ import { makeRequestOptionsFromResolvedModel } from "../lib/makeRequestOptions.j
 import type { InferenceProviderOrPolicy, InferenceTask, RequestArgs } from "../types.js";
 import { templates } from "./templates.exported.js";
 
-export type InferenceSnippetOptions = { streaming?: boolean; billTo?: string; accessToken?: string } & Record<
-	string,
-	unknown
->;
+export type InferenceSnippetOptions = {
+	streaming?: boolean;
+	billTo?: string;
+	accessToken?: string;
+	directRequest?: boolean;
+} & Record<string, unknown>;
 
 const PYTHON_CLIENTS = ["huggingface_hub", "fal_client", "requests", "openai"] as const;
 const JS_CLIENTS = ["fetch", "huggingface.js", "openai"] as const;
@@ -124,7 +126,10 @@ const HF_JS_METHODS: Partial<Record<WidgetType, string>> = {
 	translation: "translation",
 };
 
-const ACCESS_TOKEN_PLACEHOLDER = "<ACCESS_TOKEN>"; // Placeholder to replace with env variable in snippets
+// Placeholders to replace with env variable in snippets
+// little hack to support both direct requests and routing => routed requests should start with "hf_"
+const ACCESS_TOKEN_ROUTING_PLACEHOLDER = "hf_token_placeholder";
+const ACCESS_TOKEN_DIRECT_REQUEST_PLACEHOLDER = "not_hf_token_placeholder";
 
 // Snippet generators
 const snippetGenerator = (templateName: string, inputPreparationFn?: InputPreparationFn) => {
@@ -153,7 +158,11 @@ const snippetGenerator = (templateName: string, inputPreparationFn?: InputPrepar
 			console.error(`Failed to get provider helper for ${provider} (${task})`, e);
 			return [];
 		}
-		const accessTokenOrPlaceholder = opts?.accessToken ?? ACCESS_TOKEN_PLACEHOLDER;
+
+		const placeholder = opts?.directRequest
+			? ACCESS_TOKEN_DIRECT_REQUEST_PLACEHOLDER
+			: ACCESS_TOKEN_ROUTING_PLACEHOLDER;
+		const accessTokenOrPlaceholder = opts?.accessToken ?? placeholder;
 
 		/// Prepare inputs + make request
 		const inputs = inputPreparationFn ? inputPreparationFn(model, opts) : { inputs: getModelInputSnippet(model) };
@@ -255,8 +264,8 @@ const snippetGenerator = (templateName: string, inputPreparationFn?: InputPrepar
 						}
 
 						/// Replace access token placeholder
-						if (snippet.includes(ACCESS_TOKEN_PLACEHOLDER)) {
-							snippet = replaceAccessTokenPlaceholder(snippet, language, provider);
+						if (snippet.includes(placeholder)) {
+							snippet = replaceAccessTokenPlaceholder(opts?.directRequest, placeholder, snippet, language, provider);
 						}
 
 						/// Snippet is ready!
@@ -431,6 +440,8 @@ function removeSuffix(str: string, suffix: string) {
 }
 
 function replaceAccessTokenPlaceholder(
+	directRequest: boolean | undefined,
+	placeholder: string,
 	snippet: string,
 	language: InferenceSnippetLanguage,
 	provider: InferenceProviderOrPolicy
@@ -439,46 +450,57 @@ function replaceAccessTokenPlaceholder(
 	// Once snippets are rendered, we replace the placeholder with code to fetch the access token from an environment variable.
 
 	// Determine if HF_TOKEN or specific provider token should be used
-	const accessTokenEnvVar =
-		!snippet.includes("https://") || // no URL provided => using a client => use $HF_TOKEN
-		snippet.includes("https://router.huggingface.co") || // explicit routed request => use $HF_TOKEN
-		provider == "hf-inference" // hf-inference provider => use $HF_TOKEN
-			? "HF_TOKEN"
-			: provider.toUpperCase().replace("-", "_") + "_API_KEY"; // e.g. "REPLICATE_API_KEY"
+	const useHfToken =
+		provider == "hf-inference" || // hf-inference provider => use $HF_TOKEN
+		(!directRequest && // if explicit directRequest => use provider-specific token
+			(!snippet.includes("https://") || // no URL provided => using a client => use $HF_TOKEN
+				snippet.includes("https://router.huggingface.co"))); // explicit routed request => use $HF_TOKEN
+
+	const accessTokenEnvVar = useHfToken
+		? "HF_TOKEN" // e.g. routed request or hf-inference
+		: provider.toUpperCase().replace("-", "_") + "_API_KEY"; // e.g. "REPLICATE_API_KEY"
 
 	// Replace the placeholder with the env variable
 	if (language === "sh") {
 		snippet = snippet.replace(
-			`'Authorization: Bearer ${ACCESS_TOKEN_PLACEHOLDER}'`,
+			`'Authorization: Bearer ${placeholder}'`,
 			`"Authorization: Bearer $${accessTokenEnvVar}"` // e.g. "Authorization: Bearer $HF_TOKEN"
 		);
 	} else if (language === "python") {
 		snippet = "import os\n" + snippet;
 		snippet = snippet.replace(
-			`"${ACCESS_TOKEN_PLACEHOLDER}"`,
+			`"${placeholder}"`,
 			`os.environ["${accessTokenEnvVar}"]` // e.g. os.environ["HF_TOKEN")
 		);
 		snippet = snippet.replace(
-			`"Bearer ${ACCESS_TOKEN_PLACEHOLDER}"`,
+			`"Bearer ${placeholder}"`,
 			`f"Bearer {os.environ['${accessTokenEnvVar}']}"` // e.g. f"Bearer {os.environ['HF_TOKEN']}"
 		);
 		snippet = snippet.replace(
-			`"Key ${ACCESS_TOKEN_PLACEHOLDER}"`,
+			`"Key ${placeholder}"`,
 			`f"Key {os.environ['${accessTokenEnvVar}']}"` // e.g. f"Key {os.environ['FAL_AI_API_KEY']}"
 		);
+		snippet = snippet.replace(
+			`"X-Key ${placeholder}"`,
+			`f"X-Key {os.environ['${accessTokenEnvVar}']}"` // e.g. f"X-Key {os.environ['BLACK_FOREST_LABS_API_KEY']}"
+		);
 	} else if (language === "js") {
 		snippet = snippet.replace(
-			`"${ACCESS_TOKEN_PLACEHOLDER}"`,
+			`"${placeholder}"`,
 			`process.env.${accessTokenEnvVar}` // e.g. process.env.HF_TOKEN
 		);
 		snippet = snippet.replace(
-			`Authorization: "Bearer ${ACCESS_TOKEN_PLACEHOLDER}",`,
+			`Authorization: "Bearer ${placeholder}",`,
 			`Authorization: \`Bearer $\{process.env.${accessTokenEnvVar}}\`,` // e.g. Authorization: `Bearer ${process.env.HF_TOKEN}`,
 		);
 		snippet = snippet.replace(
-			`Authorization: "Key ${ACCESS_TOKEN_PLACEHOLDER}",`,
+			`Authorization: "Key ${placeholder}",`,
 			`Authorization: \`Key $\{process.env.${accessTokenEnvVar}}\`,` // e.g. Authorization: `Key ${process.env.FAL_AI_API_KEY}`,
 		);
+		snippet = snippet.replace(
+			`Authorization: "X-Key ${placeholder}",`,
+			`Authorization: \`X-Key $\{process.env.${accessTokenEnvVar}}\`,` // e.g. Authorization: `X-Key ${process.env.BLACK_FOREST_LABS_AI_API_KEY}`,
+		);
 	}
 	return snippet;
 }

packages/inference/src/snippets/templates/python/huggingface_hub/tableQuestionAnswering.jinja

@@ -1,4 +1,4 @@
-answer = client.question_answering(
+answer = client.table_question_answering(
     query="{{ inputs.asObj.query }}",
     table={{ inputs.asObj.table }},
     model="{{ model.id }}",

packages/mcp-client/src/Agent.ts

@@ -46,6 +46,7 @@ const exitLoopTools = [taskCompletionTool, askQuestionTool];
 
 export class Agent extends McpClient {
 	private readonly servers: (ServerConfig | StdioServerParameters)[];
+	public readonly prompt: string;
 	protected messages: ChatCompletionInputMessage[];
 
 	constructor({
@@ -73,10 +74,11 @@ export class Agent extends McpClient {
 		super(provider ? { provider, endpointUrl, model, apiKey } : { provider, endpointUrl, model, apiKey });
 		/// ^This shenanigan is just here to please an overzealous TS type-checker.
 		this.servers = servers;
+		this.prompt = prompt ?? DEFAULT_SYSTEM_PROMPT;
 		this.messages = [
 			{
 				role: "system",
-				content: prompt ?? DEFAULT_SYSTEM_PROMPT,
+				content: this.prompt,
 			},
 		];
 	}
@@ -86,19 +88,27 @@ export class Agent extends McpClient {
 	}
 
 	async *run(
-		input: string,
+		input: string | ChatCompletionInputMessage[],
 		opts: { abortSignal?: AbortSignal } = {}
 	): AsyncGenerator<ChatCompletionStreamOutput | ChatCompletionInputMessageTool> {
-		this.messages.push({
-			role: "user",
-			content: input,
-		});
+		let messages: ChatCompletionInputMessage[];
+		if (typeof input === "string") {
+			/// Use internal array of messages
+			this.messages.push({
+				role: "user",
+				content: input,
+			});
+			messages = this.messages;
+		} else {
+			/// Use the passed messages directly
+			messages = input;
+		}
 
 		let numOfTurns = 0;
 		let nextTurnShouldCallTools = true;
 		while (true) {
 			try {
-				yield* this.processSingleTurnWithTools(this.messages, {
+				yield* this.processSingleTurnWithTools(messages, {
 					exitLoopTools,
 					exitIfFirstChunkNoTool: numOfTurns > 0 && nextTurnShouldCallTools,
 					abortSignal: opts.abortSignal,
@@ -111,7 +121,7 @@ export class Agent extends McpClient {
 			}
 			numOfTurns++;
 			// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-			const currentLast = this.messages.at(-1)!;
+			const currentLast = messages.at(-1)!;
 			debug("current role", currentLast.role);
 			if (
 				currentLast.role === "tool" &&

packages/ollama-utils/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/ollama-utils",
 	"packageManager": "[email protected]",
-	"version": "0.0.11",
+	"version": "0.0.12",
 	"description": "Various utilities for maintaining Ollama compatibility with models on Hugging Face hub",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/ollama-utils/src/chat-template-automap.ts

@@ -5,9 +5,14 @@ import type { OllamaChatTemplateMapEntry } from "./types";
 
 /**
  * Skipped these models due to error:
- * - library/llama4:latest
- * - library/llama3.1:latest
- * - library/granite3.2:latest
+ * - library/llama4:16x17b
+ * - library/llama3.3:latest
+ * - library/dolphin3:8b
+ * - library/qwen2-math:latest
+ * - library/qwen2-math:1.5b
+ * - library/marco-o1:latest
+ * - library/bespoke-minicheck:7b
+ * - library/falcon2:11b
  */
 
 export const OLLAMA_CHAT_TEMPLATE_MAPPING: OllamaChatTemplateMapEntry[] = [
@@ -269,6 +274,33 @@ export const OLLAMA_CHAT_TEMPLATE_MAPPING: OllamaChatTemplateMapEntry[] = [
 			},
 		},
 	},
+	{
+		model: "library/deepseek-r1:latest",
+		gguf: "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{% set content = message['content'] %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{%- set ns.is_first = false -%}{%- set ns.is_last_user = true -%}{{'<｜User｜>' + content + '<｜Assistant｜>'}}{%- endif %}{%- if message['role'] == 'assistant' %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{% endif %}{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}{%- set ns.is_last_user = false -%}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{%- endif %}{%- set ns.is_first = false %}{%- set ns.is_tool = false -%}{%- set ns.is_output_first = true %}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if content is none %}{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- else %}{{content + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- endfor %}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}{%- set ns.is_last_user = false -%}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + content + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{{content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_last_user = false -%}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}",
+		ollama: {
+			template:
+				'{{- if .System }}{{ .System }}{{ end }}\n{{- range $i, $_ := .Messages }}\n{{- $last := eq (len (slice $.Messages $i)) 1}}\n{{- if eq .Role "user" }}<｜User｜>{{ .Content }}\n{{- else if eq .Role "assistant" }}<｜Assistant｜>\n  {{- if and $.IsThinkSet (and $last .Thinking) -}}\n<think>\n{{ .Thinking }}\n</think>\n{{- end }}{{ .Content }}{{- if not $last }}<｜end▁of▁sentence｜>{{- end }}\n{{- end }}\n{{- if and $last (ne .Role "assistant") }}<｜Assistant｜>\n{{- if and $.IsThinkSet (not $.Think) -}}\n<think>\n\n</think>\n\n{{ end }}\n{{- end -}}\n{{- end }}',
+			tokens: [
+				"<｜User｜>",
+				"<｜Assistant｜>",
+				"<｜tool▁outputs▁end｜>",
+				"<｜tool▁calls▁begin｜>",
+				"<｜tool▁call▁begin｜>",
+				"<｜tool▁sep｜>",
+				"<｜tool▁call▁end｜>",
+				"<｜tool▁calls▁end｜>",
+				"<｜end▁of▁sentence｜>",
+				"<｜tool▁outputs▁begin｜>",
+				"<｜tool▁output▁begin｜>",
+				"<｜tool▁output▁end｜>",
+			],
+			params: {
+				stop: ["<｜begin▁of▁sentence｜>", "<｜end▁of▁sentence｜>", "<｜User｜>", "<｜Assistant｜>"],
+				temperature: 0.6,
+				top_p: 0.95,
+			},
+		},
+	},
 	{
 		model: "library/deepseek-v2.5:236b",
 		gguf: "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}    {%- if message['role'] == 'system' %}        {% set ns.system_prompt = message['content'] %}    {%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}    {%- if message['role'] == 'user' %}    {%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}    {%- endif %}    {%- if message['role'] == 'assistant' and message['content'] is none %}        {%- set ns.is_tool = false -%}        {%- for tool in message['tool_calls']%}            {%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}            {%- set ns.is_first = true -%}            {%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}                   {%- endif %}        {%- endfor %}    {%- endif %}    {%- if message['role'] == 'assistant' and message['content'] is not none %}        {%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}        {%- set ns.is_tool = false -%}        {%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}        {%- endif %}    {%- endif %}    {%- if message['role'] == 'tool' %}        {%- set ns.is_tool = true -%}        {%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}        {%- set ns.is_output_first = false %}        {%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}        {%- endif %}    {%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}",

packages/tasks-gen/scripts/generate-snippets-fixtures.ts

@@ -252,6 +252,18 @@ const TEST_CASES: {
 		providers: ["hf-inference"],
 		opts: { accessToken: "hf_xxx" },
 	},
+	{
+		testName: "explicit-direct-request",
+		task: "conversational",
+		model: {
+			id: "meta-llama/Llama-3.1-8B-Instruct",
+			pipeline_tag: "text-generation",
+			tags: ["conversational"],
+			inference: "",
+		},
+		providers: ["together"],
+		opts: { directRequest: true },
+	},
 	{
 		testName: "text-to-speech",
 		task: "text-to-speech",

packages/tasks-gen/snippets-fixtures/conversational-llm-non-stream/js/openai/0.together.js

@@ -1,8 +1,8 @@
 import { OpenAI } from "openai";
 
 const client = new OpenAI({
-	baseURL: "https://api.together.xyz/v1",
-	apiKey: process.env.TOGETHER_API_KEY,
+	baseURL: "https://router.huggingface.co/together/v1",
+	apiKey: process.env.HF_TOKEN,
 });
 
 const chatCompletion = await client.chat.completions.create({