Merge branch 'main' into add-support-for-image-to-image-models-on-Replicate

Author: hanouticelina

README.md

@@ -57,7 +57,7 @@ This is a collection of JS libraries to interact with the Hugging Face API, with
 
 - [@huggingface/inference](packages/inference/README.md): Use all supported (serverless) Inference Providers or switch to Inference Endpoints (dedicated) to make calls to 100,000+ Machine Learning models
 - [@huggingface/hub](packages/hub/README.md): Interact with huggingface.co to create or delete repos and commit / download files
-- [@huggingface/agents](packages/agents/README.md): Interact with HF models through a natural language interface
+- [@huggingface/mcp-client](packages/mcp-client/README.md): A Model Context Protocol (MCP) client, and a tiny Agent library, built on top of InferenceClient.
 - [@huggingface/gguf](packages/gguf/README.md): A GGUF parser that works on remotely hosted files.
 - [@huggingface/dduf](packages/dduf/README.md): Similar package for DDUF (DDUF Diffusers Unified Format)
 - [@huggingface/tasks](packages/tasks/README.md): The definition files and source-of-truth for the Hub's main primitives like pipeline tasks, model libraries, etc.
@@ -79,15 +79,15 @@ To install via NPM, you can download the libraries as needed:
 ```bash
 npm install @huggingface/inference
 npm install @huggingface/hub
-npm install @huggingface/agents
+npm install @huggingface/mcp-client
 ```
 
 Then import the libraries in your code:
 
 ```ts
 import { InferenceClient } from "@huggingface/inference";
-import { HfAgent } from "@huggingface/agents";
 import { createRepo, commit, deleteRepo, listFiles } from "@huggingface/hub";
+import { McpClient } from "@huggingface/mcp-client";
 import type { RepoId } from "@huggingface/hub";
 ```
 
@@ -97,7 +97,7 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
 
 ```html
 <script type="module">
-    import { InferenceClient } from 'https://cdn.jsdelivr.net/npm/@huggingface/inference@3.12.1/+esm';
+    import { InferenceClient } from 'https://cdn.jsdelivr.net/npm/@huggingface/inference@3.13.0/+esm';
     import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/[email protected]/+esm";
 </script>
 ```
@@ -107,12 +107,10 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
 ```ts
 // esm.sh
 import { InferenceClient } from "https://esm.sh/@huggingface/inference"
-import { HfAgent } from "https://esm.sh/@huggingface/agents";
 
 import { createRepo, commit, deleteRepo, listFiles } from "https://esm.sh/@huggingface/hub"
 // or npm:
 import { InferenceClient } from "npm:@huggingface/inference"
-import { HfAgent } from "npm:@huggingface/agents";
 
 import { createRepo, commit, deleteRepo, listFiles } from "npm:@huggingface/hub"
 ```
@@ -223,29 +221,36 @@ await deleteFiles({
 });
 ```
 
-### @huggingface/agents example
+### @huggingface/mcp-client example
 
 ```ts
-import { HfAgent, LLMFromHub, defaultTools } from '@huggingface/agents';
+import { Agent } from '@huggingface/mcp-client';
 
 const HF_TOKEN = "hf_...";
 
-const agent = new HfAgent(
-  HF_TOKEN,
-  LLMFromHub(HF_TOKEN),
-  [...defaultTools]
-);
-
+const agent = new Agent({
+  provider: "auto",
+  model: "Qwen/Qwen2.5-72B-Instruct",
+  apiKey: HF_TOKEN,
+  servers: [
+    {
+      // Playwright MCP
+      command: "npx",
+      args: ["@playwright/mcp@latest"],
+    },
+  ],
+});
 
-// you can generate the code, inspect it and then run it
-const code = await agent.generateCode("Draw a picture of a cat wearing a top hat. Then caption the picture and read it out loud.");
-console.log(code);
-const messages = await agent.evaluateCode(code)
-console.log(messages); // contains the data
 
-// or you can run the code directly, however you can't check that the code is safe to execute this way, use at your own risk.
-const messages = await agent.run("Draw a picture of a cat wearing a top hat. Then caption the picture and read it out loud.")
-console.log(messages);
+await agent.loadTools();
+for await (const chunk of agent.run("What are the top 5 trending models on Hugging Face?")) {
+    if ("choices" in chunk) {
+        const delta = chunk.choices[0]?.delta;
+        if (delta.content) {
+            console.log(delta.content);
+        }
+    }
+}
 ```
 
 There are more features of course, check each library's README!

packages/agents/README.md

@@ -2,6 +2,10 @@
 
 A way to call Hugging Face models and Inference Endpoints from natural language, using an LLM.
 
+> [!WARNING]
+> `@huggingface/agents` is now deprecated, and a modern version, built on top of MCP, is [Tiny Agents](https://github.com/huggingface/huggingface.js/tree/main/packages/mcp-client).
+> Go checkout the `Tiny Agents` introduction blog [here](https://huggingface.co/blog/tiny-agents).
+
 ## Install
 
 ```console

packages/gguf/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/gguf",
 	"packageManager": "[email protected]",
-	"version": "0.1.14",
+	"version": "0.1.17",
 	"description": "a GGUF parser that works on remotely hosted files",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/inference/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@huggingface/inference",
-	"version": "3.12.1",
+	"version": "3.13.0",
 	"packageManager": "[email protected]",
 	"license": "MIT",
 	"author": "Hugging Face and Tim Mikeladze <[email protected]>",

packages/inference/src/lib/getProviderHelper.ts

@@ -115,6 +115,7 @@ export const PROVIDERS: Record<InferenceProvider, Partial<Record<InferenceTask,
 		"text-to-image": new Nebius.NebiusTextToImageTask(),
 		conversational: new Nebius.NebiusConversationalTask(),
 		"text-generation": new Nebius.NebiusTextGenerationTask(),
+		"feature-extraction": new Nebius.NebiusFeatureExtractionTask(),
 	},
 	novita: {
 		conversational: new Novita.NovitaConversationalTask(),

packages/inference/src/providers/fal-ai.ts

@@ -14,10 +14,12 @@
  *
  * Thanks!
  */
+import { base64FromBytes } from "../utils/base64FromBytes";
+
 import type { AutomaticSpeechRecognitionOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../lib/InferenceOutputError";
 import { isUrl } from "../lib/isUrl";
-import type { BodyParams, HeaderParams, ModelId, UrlParams } from "../types";
+import type { BodyParams, HeaderParams, ModelId, RequestArgs, UrlParams } from "../types";
 import { delay } from "../utils/delay";
 import { omit } from "../utils/omit";
 import {
@@ -27,6 +29,7 @@ import {
 	type TextToVideoTaskHelper,
 } from "./providerHelper";
 import { HF_HUB_URL } from "../config";
+import type { AutomaticSpeechRecognitionArgs } from "../tasks/audio/automaticSpeechRecognition";
 
 export interface FalAiQueueOutput {
 	request_id: string;
@@ -224,6 +227,28 @@ export class FalAIAutomaticSpeechRecognitionTask extends FalAITask implements Au
 		}
 		return { text: res.text };
 	}
+
+	async preparePayloadAsync(args: AutomaticSpeechRecognitionArgs): Promise<RequestArgs> {
+		const blob = "data" in args && args.data instanceof Blob ? args.data : "inputs" in args ? args.inputs : undefined;
+		const contentType = blob?.type;
+		if (!contentType) {
+			throw new Error(
+				`Unable to determine the input's content-type. Make sure your are passing a Blob when using provider fal-ai.`
+			);
+		}
+		if (!FAL_AI_SUPPORTED_BLOB_TYPES.includes(contentType)) {
+			throw new Error(
+				`Provider fal-ai does not support blob type ${contentType} - supported content types are: ${FAL_AI_SUPPORTED_BLOB_TYPES.join(
+					", "
+				)}`
+			);
+		}
+		const base64audio = base64FromBytes(new Uint8Array(await blob.arrayBuffer()));
+		return {
+			...("data" in args ? omit(args, "data") : omit(args, "inputs")),
+			audio_url: `data:${contentType};base64,${base64audio}`,
+		};
+	}
 }
 
 export class FalAITextToSpeechTask extends FalAITask {

packages/inference/src/providers/hf-inference.ts

@@ -36,7 +36,7 @@ import type {
 import { HF_ROUTER_URL } from "../config";
 import { InferenceOutputError } from "../lib/InferenceOutputError";
 import type { TabularClassificationOutput } from "../tasks/tabular/tabularClassification";
-import type { BodyParams, UrlParams } from "../types";
+import type { BodyParams, RequestArgs, UrlParams } from "../types";
 import { toArray } from "../utils/toArray";
 import type {
 	AudioClassificationTaskHelper,
@@ -70,7 +70,10 @@ import type {
 } from "./providerHelper";
 
 import { TaskProviderHelper } from "./providerHelper";
-
+import { base64FromBytes } from "../utils/base64FromBytes";
+import type { ImageToImageArgs } from "../tasks/cv/imageToImage";
+import type { AutomaticSpeechRecognitionArgs } from "../tasks/audio/automaticSpeechRecognition";
+import { omit } from "../utils/omit";
 interface Base64ImageGeneration {
 	data: Array<{
 		b64_json: string;
@@ -221,6 +224,15 @@ export class HFInferenceAutomaticSpeechRecognitionTask
 	override async getResponse(response: AutomaticSpeechRecognitionOutput): Promise<AutomaticSpeechRecognitionOutput> {
 		return response;
 	}
+
+	async preparePayloadAsync(args: AutomaticSpeechRecognitionArgs): Promise<RequestArgs> {
+		return "data" in args
+			? args
+			: {
+					...omit(args, "inputs"),
+					data: args.inputs,
+			  };
+	}
 }
 
 export class HFInferenceAudioToAudioTask extends HFInferenceTask implements AudioToAudioTaskHelper {
@@ -303,7 +315,12 @@ export class HFInferenceImageSegmentationTask extends HFInferenceTask implements
 	override async getResponse(response: ImageSegmentationOutput): Promise<ImageSegmentationOutput> {
 		if (
 			Array.isArray(response) &&
-			response.every((x) => typeof x.label === "string" && typeof x.mask === "string" && typeof x.score === "number")
+			response.every(
+				(x) =>
+					typeof x.label === "string" &&
+					typeof x.mask === "string" &&
+					(x.score === undefined || typeof x.score === "number")
+			)
 		) {
 			return response;
 		}
@@ -321,6 +338,23 @@ export class HFInferenceImageToTextTask extends HFInferenceTask implements Image
 }
 
 export class HFInferenceImageToImageTask extends HFInferenceTask implements ImageToImageTaskHelper {
+	async preparePayloadAsync(args: ImageToImageArgs): Promise<RequestArgs> {
+		if (!args.parameters) {
+			return {
+				...args,
+				model: args.model,
+				data: args.inputs,
+			};
+		} else {
+			return {
+				...args,
+				inputs: base64FromBytes(
+					new Uint8Array(args.inputs instanceof ArrayBuffer ? args.inputs : await (args.inputs as Blob).arrayBuffer())
+				),
+			};
+		}
+	}
+
 	override async getResponse(response: Blob): Promise<Blob> {
 		if (response instanceof Blob) {
 			return response;

packages/inference/src/providers/nebius.ts

@@ -14,13 +14,15 @@
  *
  * Thanks!
  */
+import type { FeatureExtractionOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../lib/InferenceOutputError";
-import type { BodyParams, UrlParams } from "../types";
+import type { BodyParams } from "../types";
 import { omit } from "../utils/omit";
 import {
 	BaseConversationalTask,
 	BaseTextGenerationTask,
 	TaskProviderHelper,
+	type FeatureExtractionTaskHelper,
 	type TextToImageTaskHelper,
 } from "./providerHelper";
 
@@ -32,6 +34,12 @@ interface NebiusBase64ImageGeneration {
 	}>;
 }
 
+interface NebiusEmbeddingsResponse {
+	data: Array<{
+		embedding: number[];
+	}>;
+}
+
 export class NebiusConversationalTask extends BaseConversationalTask {
 	constructor() {
 		super("nebius", NEBIUS_API_BASE_URL);
@@ -59,8 +67,7 @@ export class NebiusTextToImageTask extends TaskProviderHelper implements TextToI
 		};
 	}
 
-	makeRoute(params: UrlParams): string {
-		void params;
+	makeRoute(): string {
 		return "v1/images/generations";
 	}
 
@@ -88,3 +95,24 @@ export class NebiusTextToImageTask extends TaskProviderHelper implements TextToI
 		throw new InferenceOutputError("Expected Nebius text-to-image response format");
 	}
 }
+
+export class NebiusFeatureExtractionTask extends TaskProviderHelper implements FeatureExtractionTaskHelper {
+	constructor() {
+		super("nebius", NEBIUS_API_BASE_URL);
+	}
+
+	preparePayload(params: BodyParams): Record<string, unknown> {
+		return {
+			input: params.args.inputs,
+			model: params.model,
+		};
+	}
+
+	makeRoute(): string {
+		return "v1/embeddings";
+	}
+
+	async getResponse(response: NebiusEmbeddingsResponse): Promise<FeatureExtractionOutput> {
+		return response.data.map((item) => item.embedding);
+	}
+}

packages/inference/src/providers/providerHelper.ts

@@ -48,8 +48,10 @@ import type {
 import { HF_ROUTER_URL } from "../config";
 import { InferenceOutputError } from "../lib/InferenceOutputError";
 import type { AudioToAudioOutput } from "../tasks/audio/audioToAudio";
-import type { BaseArgs, BodyParams, HeaderParams, InferenceProvider, UrlParams } from "../types";
+import type { BaseArgs, BodyParams, HeaderParams, InferenceProvider, RequestArgs, UrlParams } from "../types";
 import { toArray } from "../utils/toArray";
+import type { ImageToImageArgs } from "../tasks/cv/imageToImage";
+import type { AutomaticSpeechRecognitionArgs } from "../tasks/audio/automaticSpeechRecognition";
 
 /**
  * Base class for task-specific provider helpers
@@ -142,6 +144,7 @@ export interface TextToVideoTaskHelper {
 export interface ImageToImageTaskHelper {
 	getResponse(response: unknown, url?: string, headers?: HeadersInit): Promise<Blob>;
 	preparePayload(params: BodyParams<ImageToImageInput & BaseArgs>): Record<string, unknown>;
+	preparePayloadAsync(args: ImageToImageArgs): Promise<RequestArgs>;
 }
 
 export interface ImageSegmentationTaskHelper {
@@ -245,6 +248,7 @@ export interface AudioToAudioTaskHelper {
 export interface AutomaticSpeechRecognitionTaskHelper {
 	getResponse(response: unknown, url?: string, headers?: HeadersInit): Promise<AutomaticSpeechRecognitionOutput>;
 	preparePayload(params: BodyParams<AutomaticSpeechRecognitionInput & BaseArgs>): Record<string, unknown> | BodyInit;
+	preparePayloadAsync(args: AutomaticSpeechRecognitionArgs): Promise<RequestArgs>;
 }
 
 export interface AudioClassificationTaskHelper {