Merge branch 'main' into provider-featherless-ai

Author: hanouticelina

packages/inference/README.md

@@ -60,6 +60,7 @@ Currently, we support the following providers:
 - [Blackforestlabs](https://blackforestlabs.ai)
 - [Cohere](https://cohere.com)
 - [Cerebras](https://cerebras.ai/)
+- [Groq](https://groq.com)
 
 To send requests to a third-party provider, you have to pass the `provider` parameter to the inference function. Make sure your request is authenticated with an access token.
 ```ts
@@ -88,6 +89,7 @@ Only a subset of models are supported when requesting third-party providers. You
 - [Together supported models](https://huggingface.co/api/partners/together/models)
 - [Cohere supported models](https://huggingface.co/api/partners/cohere/models)
 - [Cerebras supported models](https://huggingface.co/api/partners/cerebras/models)
+- [Groq supported models](https://console.groq.com/docs/models)
 - [HF Inference API (serverless)](https://huggingface.co/models?inference=warm&sort=trending)
 
 ❗**Important note:** To be compatible, the third-party API must adhere to the "standard" shape API we expect on HF model pages for each pipeline task type.

packages/inference/src/lib/getProviderHelper.ts

@@ -4,6 +4,7 @@ import * as Cohere from "../providers/cohere";
 import * as FalAI from "../providers/fal-ai";
 import * as FeatherlessAI from "../providers/featherless-ai";
 import * as Fireworks from "../providers/fireworks-ai";
+import * as Groq from "../providers/groq";
 import * as HFInference from "../providers/hf-inference";
 import * as Hyperbolic from "../providers/hyperbolic";
 import * as Nebius from "../providers/nebius";
@@ -100,6 +101,10 @@ export const PROVIDERS: Record<InferenceProvider, Partial<Record<InferenceTask,
 	"fireworks-ai": {
 		conversational: new Fireworks.FireworksConversationalTask(),
 	},
+	groq: {
+		conversational: new Groq.GroqConversationalTask(),
+		"text-generation": new Groq.GroqTextGenerationTask(),
+	},
 	hyperbolic: {
 		"text-to-image": new Hyperbolic.HyperbolicTextToImageTask(),
 		conversational: new Hyperbolic.HyperbolicConversationalTask(),

packages/inference/src/providers/consts.ts

@@ -25,6 +25,7 @@ export const HARDCODED_MODEL_INFERENCE_MAPPING: Record<
 	"fal-ai": {},
 	"featherless-ai": {},
 	"fireworks-ai": {},
+	groq: {},
 	"hf-inference": {},
 	hyperbolic: {},
 	nebius: {},

packages/inference/src/providers/groq.ts

@@ -0,0 +1,40 @@
+import { BaseConversationalTask, BaseTextGenerationTask } from "./providerHelper";
+
+/**
+ * See the registered mapping of HF model ID => Groq model ID here:
+ *
+ * https://huggingface.co/api/partners/groq/models
+ *
+ * This is a publicly available mapping.
+ *
+ * If you want to try to run inference for a new model locally before it's registered on huggingface.co,
+ * you can add it to the dictionary "HARDCODED_MODEL_ID_MAPPING" in consts.ts, for dev purposes.
+ *
+ * - If you work at Groq and want to update this mapping, please use the model mapping API we provide on huggingface.co
+ * - If you're a community member and want to add a new supported HF model to Groq, please open an issue on the present repo
+ * and we will tag Groq team members.
+ *
+ * Thanks!
+ */
+
+const GROQ_API_BASE_URL = "https://api.groq.com";
+
+export class GroqTextGenerationTask extends BaseTextGenerationTask {
+	constructor() {
+		super("groq", GROQ_API_BASE_URL);
+	}
+
+	override makeRoute(): string {
+		return "/openai/v1/chat/completions";
+	}
+}
+
+export class GroqConversationalTask extends BaseConversationalTask {
+	constructor() {
+		super("groq", GROQ_API_BASE_URL);
+	}
+
+	override makeRoute(): string {
+		return "/openai/v1/chat/completions";
+	}
+}

packages/inference/src/types.ts

@@ -44,6 +44,7 @@ export const INFERENCE_PROVIDERS = [
 	"fal-ai",
 	"featherless-ai",
 	"fireworks-ai",
+	"groq",
 	"hf-inference",
 	"hyperbolic",
 	"nebius",

packages/inference/test/InferenceClient.spec.ts

@@ -1824,4 +1824,55 @@ describe.skip("InferenceClient", () => {
 		},
 		TIMEOUT
 	);
+	describe.concurrent(
+		"Groq",
+		() => {
+			const client = new InferenceClient(env.HF_GROQ_KEY ?? "dummy");
+
+			HARDCODED_MODEL_INFERENCE_MAPPING["groq"] = {
+				"meta-llama/Llama-3.3-70B-Instruct": {
+					hfModelId: "meta-llama/Llama-3.3-70B-Instruct",
+					providerId: "llama-3.3-70b-versatile",
+					status: "live",
+					task: "conversational",
+				},
+			};
+
+			it("chatCompletion", async () => {
+				const res = await client.chatCompletion({
+					model: "meta-llama/Llama-3.3-70B-Instruct",
+					provider: "groq",
+					messages: [{ role: "user", content: "Complete this sentence with words, one plus one is equal " }],
+				});
+				if (res.choices && res.choices.length > 0) {
+					const completion = res.choices[0].message?.content;
+					expect(completion).toContain("two");
+				}
+			});
+
+			it("chatCompletion stream", async () => {
+				const stream = client.chatCompletionStream({
+					model: "meta-llama/Llama-3.3-70B-Instruct",
+					provider: "groq",
+					messages: [{ role: "user", content: "Say 'this is a test'" }],
+					stream: true,
+				}) as AsyncGenerator<ChatCompletionStreamOutput>;
+
+				let fullResponse = "";
+				for await (const chunk of stream) {
+					if (chunk.choices && chunk.choices.length > 0) {
+						const content = chunk.choices[0].delta?.content;
+						if (content) {
+							fullResponse += content;
+						}
+					}
+				}
+
+				// Verify we got a meaningful response
+				expect(fullResponse).toBeTruthy();
+				expect(fullResponse.length).toBeGreaterThan(0);
+			});
+		},
+		TIMEOUT
+	);
 });