Merge branch 'main' into kai/hyperbolic-integration

Author: SBrandeis

.github/workflows/test.yml

@@ -41,13 +41,16 @@ jobs:
         run: VCR_MODE=playback pnpm --filter ...[${{ steps.since.outputs.SINCE }}] test
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_BLACK_FOREST_LABS_KEY: dummy
           HF_FAL_KEY: dummy
           HF_FIREWORKS_KEY: dummy
           HF_HYPERBOLIC_KEY: dummy
           HF_NEBIUS_KEY: dummy
+          HF_NOVITA_KEY: dummy
           HF_REPLICATE_KEY: dummy
           HF_SAMBANOVA_KEY: dummy
           HF_TOGETHER_KEY: dummy
+
   browser:
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -83,13 +86,16 @@ jobs:
         run: VCR_MODE=playback pnpm --filter ...[${{ steps.since.outputs.SINCE }}] test:browser
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_BLACK_FOREST_LABS_KEY: dummy
           HF_FAL_KEY: dummy
           HF_FIREWORKS_KEY: dummy
           HF_HYPERBOLIC_KEY: dummy
           HF_NEBIUS_KEY: dummy
+          HF_NOVITA_KEY: dummy
           HF_REPLICATE_KEY: dummy
           HF_SAMBANOVA_KEY: dummy
           HF_TOGETHER_KEY: dummy
+
   e2e:
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -152,10 +158,12 @@ jobs:
         env:
           NPM_CONFIG_REGISTRY: http://localhost:4874/
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_BLACK_FOREST_LABS_KEY: dummy
           HF_FAL_KEY: dummy
           HF_FIREWORKS_KEY: dummy
           HF_HYPERBOLIC_KEY: dummy
           HF_NEBIUS_KEY: dummy
+          HF_NOVITA_KEY: dummy
           HF_REPLICATE_KEY: dummy
           HF_SAMBANOVA_KEY: dummy
           HF_TOGETHER_KEY: dummy

packages/inference/README.md

@@ -51,9 +51,11 @@ Currently, we support the following providers:
 - [Fireworks AI](https://fireworks.ai)
 - [Hyperbolic](https://hyperbolic.xyz)
 - [Nebius](https://studio.nebius.ai)
+- [Novita](https://novita.ai/?utm_source=github_huggingface&utm_medium=github_readme&utm_campaign=link)
 - [Replicate](https://replicate.com)
 - [Sambanova](https://sambanova.ai)
 - [Together](https://together.xyz)
+- [Blackforestlabs](https://blackforestlabs.ai)
 
 To send requests to a third-party provider, you have to pass the `provider` parameter to the inference function. Make sure your request is authenticated with an access token.
 ```ts

packages/inference/src/lib/makeRequestOptions.ts

@@ -4,8 +4,10 @@ import { NEBIUS_API_BASE_URL } from "../providers/nebius";
 import { REPLICATE_API_BASE_URL } from "../providers/replicate";
 import { SAMBANOVA_API_BASE_URL } from "../providers/sambanova";
 import { TOGETHER_API_BASE_URL } from "../providers/together";
+import { NOVITA_API_BASE_URL } from "../providers/novita";
 import { FIREWORKS_AI_API_BASE_URL } from "../providers/fireworks-ai";
 import { HYPERBOLIC_API_BASE_URL } from "../providers/hyperbolic";
+import { BLACKFORESTLABS_AI_API_BASE_URL } from "../providers/black-forest-labs";
 import type { InferenceProvider } from "../types";
 import type { InferenceTask, Options, RequestArgs } from "../types";
 import { isUrl } from "./isUrl";
@@ -29,8 +31,6 @@ export async function makeRequestOptions(
 		stream?: boolean;
 	},
 	options?: Options & {
-		/** When a model can be used for multiple tasks, and we want to run a non-default task */
-		forceTask?: string | InferenceTask;
 		/** To load default model if needed */
 		taskHint?: InferenceTask;
 		chatCompletion?: boolean;
@@ -40,14 +40,11 @@ export async function makeRequestOptions(
 	let otherArgs = remainingArgs;
 	const provider = maybeProvider ?? "hf-inference";
 
-	const { forceTask, includeCredentials, taskHint, chatCompletion } = options ?? {};
+	const { includeCredentials, taskHint, chatCompletion } = options ?? {};
 
 	if (endpointUrl && provider !== "hf-inference") {
 		throw new Error(`Cannot use endpointUrl with a third-party provider.`);
 	}
-	if (forceTask && provider !== "hf-inference") {
-		throw new Error(`Cannot use forceTask with a third-party provider.`);
-	}
 	if (maybeModel && isUrl(maybeModel)) {
 		throw new Error(`Model URLs are no longer supported. Use endpointUrl instead.`);
 	}
@@ -78,16 +75,20 @@ export async function makeRequestOptions(
 		: makeUrl({
 				authMethod,
 				chatCompletion: chatCompletion ?? false,
-				forceTask,
 				model,
 				provider: provider ?? "hf-inference",
 				taskHint,
 		  });
 
 	const headers: Record<string, string> = {};
 	if (accessToken) {
-		headers["Authorization"] =
-			provider === "fal-ai" && authMethod === "provider-key" ? `Key ${accessToken}` : `Bearer ${accessToken}`;
+		if (provider === "fal-ai" && authMethod === "provider-key") {
+			headers["Authorization"] = `Key ${accessToken}`;
+		} else if (provider === "black-forest-labs" && authMethod === "provider-key") {
+			headers["X-Key"] = accessToken;
+		} else {
+			headers["Authorization"] = `Bearer ${accessToken}`;
+		}
 	}
 
 	// e.g. @huggingface/inference/3.1.3
@@ -149,14 +150,19 @@ function makeUrl(params: {
 	model: string;
 	provider: InferenceProvider;
 	taskHint: InferenceTask | undefined;
-	forceTask?: string | InferenceTask;
 }): string {
 	if (params.authMethod === "none" && params.provider !== "hf-inference") {
 		throw new Error("Authentication is required when requesting a third-party provider. Please provide accessToken");
 	}
 
 	const shouldProxy = params.provider !== "hf-inference" && params.authMethod !== "provider-key";
 	switch (params.provider) {
+		case "black-forest-labs": {
+			const baseUrl = shouldProxy
+				? HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", params.provider)
+				: BLACKFORESTLABS_AI_API_BASE_URL;
+			return `${baseUrl}/${params.model}`;
+		}
 		case "fal-ai": {
 			const baseUrl = shouldProxy
 				? HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", params.provider)
@@ -216,6 +222,7 @@ function makeUrl(params: {
 			}
 			return baseUrl;
 		}
+
 		case "fireworks-ai": {
 			const baseUrl = shouldProxy
 				? HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", params.provider)
@@ -235,15 +242,28 @@ function makeUrl(params: {
 			}
 			return `${baseUrl}/v1/chat/completions`;
 		}
+		case "novita": {
+			const baseUrl = shouldProxy
+				? HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", params.provider)
+				: NOVITA_API_BASE_URL;
+			if (params.taskHint === "text-generation") {
+				if (params.chatCompletion) {
+					return `${baseUrl}/chat/completions`;
+				}
+				return `${baseUrl}/completions`;
+			}
+			return baseUrl;
+		}
 		default: {
 			const baseUrl = HF_HUB_INFERENCE_PROXY_TEMPLATE.replaceAll("{{PROVIDER}}", "hf-inference");
-			const url = params.forceTask
-				? `${baseUrl}/pipeline/${params.forceTask}/${params.model}`
-				: `${baseUrl}/models/${params.model}`;
+			if (params.taskHint && ["feature-extraction", "sentence-similarity"].includes(params.taskHint)) {
+				/// when deployed on hf-inference, those two tasks are automatically compatible with one another.
+				return `${baseUrl}/pipeline/${params.taskHint}/${params.model}`;
+			}
 			if (params.taskHint === "text-generation" && params.chatCompletion) {
-				return url + `/v1/chat/completions`;
+				return `${baseUrl}/models/${params.model}/v1/chat/completions`;
 			}
-			return url;
+			return `${baseUrl}/models/${params.model}`;
 		}
 	}
 }

packages/inference/src/providers/black-forest-labs.ts

@@ -0,0 +1,18 @@
+export const BLACKFORESTLABS_AI_API_BASE_URL = "https://api.us1.bfl.ai/v1";
+
+/**
+ * See the registered mapping of HF model ID => Black Forest Labs model ID here:
+ *
+ * https://huggingface.co/api/partners/blackforestlabs/models
+ *
+ * This is a publicly available mapping.
+ *
+ * If you want to try to run inference for a new model locally before it's registered on huggingface.co,
+ * you can add it to the dictionary "HARDCODED_MODEL_ID_MAPPING" in consts.ts, for dev purposes.
+ *
+ * - If you work at Black Forest Labs and want to update this mapping, please use the model mapping API we provide on huggingface.co
+ * - If you're a community member and want to add a new supported HF model to Black Forest Labs, please open an issue on the present repo
+ * and we will tag Black Forest Labs team members.
+ *
+ * Thanks!
+ */

packages/inference/src/providers/consts.ts

@@ -16,6 +16,7 @@ export const HARDCODED_MODEL_ID_MAPPING: Record<InferenceProvider, Record<ModelI
 	 * Example:
 	 * "Qwen/Qwen2.5-Coder-32B-Instruct": "Qwen2.5-Coder-32B-Instruct",
 	 */
+	"black-forest-labs": {},
 	"fal-ai": {},
 	"fireworks-ai": {},
 	"hf-inference": {},
@@ -24,4 +25,5 @@ export const HARDCODED_MODEL_ID_MAPPING: Record<InferenceProvider, Record<ModelI
 	replicate: {},
 	sambanova: {},
 	together: {},
+	novita: {},
 };

packages/inference/src/providers/novita.ts

@@ -0,0 +1,18 @@
+export const NOVITA_API_BASE_URL = "https://api.novita.ai/v3/openai";
+
+/**
+ * See the registered mapping of HF model ID => Novita model ID here:
+ *
+ * https://huggingface.co/api/partners/novita/models
+ *
+ * This is a publicly available mapping.
+ *
+ * If you want to try to run inference for a new model locally before it's registered on huggingface.co,
+ * you can add it to the dictionary "HARDCODED_MODEL_ID_MAPPING" in consts.ts, for dev purposes.
+ *
+ * - If you work at Novita and want to update this mapping, please use the model mapping API we provide on huggingface.co
+ * - If you're a community member and want to add a new supported HF model to Novita, please open an issue on the present repo
+ * and we will tag Novita team members.
+ *
+ * Thanks!
+ */

packages/inference/src/tasks/cv/textToImage.ts

@@ -3,6 +3,7 @@ import { InferenceOutputError } from "../../lib/InferenceOutputError";
 import type { BaseArgs, InferenceProvider, Options } from "../../types";
 import { omit } from "../../utils/omit";
 import { request } from "../custom/request";
+import { delay } from "../../utils/delay";
 
 export type TextToImageArgs = BaseArgs & TextToImageInput;
 
@@ -18,6 +19,11 @@ interface HyperbolicTextToImageOutput {
 	images: Array<{ image: string }>;
 }
 
+interface BlackForestLabsResponse {
+	id: string;
+	polling_url: string;
+}
+
 function getResponseFormatArg(provider: InferenceProvider) {
 	switch (provider) {
 		case "fal-ai":
@@ -48,13 +54,20 @@ export async function textToImage(args: TextToImageArgs, options?: Options): Pro
 					prompt: args.inputs,
 			  };
 	const res = await request<
-		TextToImageOutput | Base64ImageGeneration | OutputUrlImageGeneration | HyperbolicTextToImageOutput
+		| TextToImageOutput
+		| Base64ImageGeneration
+		| OutputUrlImageGeneration
+		| BlackForestLabsResponse
+		| HyperbolicTextToImageOutput
 	>(payload, {
 		...options,
 		taskHint: "text-to-image",
 	});
 
 	if (res && typeof res === "object") {
+		if (args.provider === "black-forest-labs" && "polling_url" in res && typeof res.polling_url === "string") {
+			return await pollBflResponse(res.polling_url);
+		}
 		if (args.provider === "fal-ai" && "images" in res && Array.isArray(res.images) && res.images[0].url) {
 			const image = await fetch(res.images[0].url);
 			return await image.blob();
@@ -88,3 +101,33 @@ export async function textToImage(args: TextToImageArgs, options?: Options): Pro
 	}
 	return res;
 }
+
+async function pollBflResponse(url: string): Promise<Blob> {
+	const urlObj = new URL(url);
+	for (let step = 0; step < 5; step++) {
+		await delay(1000);
+		console.debug(`Polling Black Forest Labs API for the result... ${step + 1}/5`);
+		urlObj.searchParams.set("attempt", step.toString(10));
+		const resp = await fetch(urlObj, { headers: { "Content-Type": "application/json" } });
+		if (!resp.ok) {
+			throw new InferenceOutputError("Failed to fetch result from black forest labs API");
+		}
+		const payload = await resp.json();
+		if (
+			typeof payload === "object" &&
+			payload &&
+			"status" in payload &&
+			typeof payload.status === "string" &&
+			payload.status === "Ready" &&
+			"result" in payload &&
+			typeof payload.result === "object" &&
+			payload.result &&
+			"sample" in payload.result &&
+			typeof payload.result.sample === "string"
+		) {
+			const image = await fetch(payload.result.sample);
+			return await image.blob();
+		}
+	}
+	throw new InferenceOutputError("Failed to fetch result from black forest labs API");
+}

packages/inference/src/tasks/nlp/featureExtraction.ts

@@ -1,5 +1,4 @@
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
-import { getDefaultTask } from "../../lib/getDefaultTask";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
 
@@ -25,12 +24,9 @@ export async function featureExtraction(
 	args: FeatureExtractionArgs,
 	options?: Options
 ): Promise<FeatureExtractionOutput> {
-	const defaultTask = args.model ? await getDefaultTask(args.model, args.accessToken, options) : undefined;
-
 	const res = await request<FeatureExtractionOutput>(args, {
 		...options,
 		taskHint: "feature-extraction",
-		...(defaultTask === "sentence-similarity" && { forceTask: "feature-extraction" }),
 	});
 	let isValidOutput = true;
 

packages/inference/src/tasks/nlp/sentenceSimilarity.ts

@@ -1,6 +1,5 @@
 import type { SentenceSimilarityInput, SentenceSimilarityOutput } from "@huggingface/tasks";
 import { InferenceOutputError } from "../../lib/InferenceOutputError";
-import { getDefaultTask } from "../../lib/getDefaultTask";
 import type { BaseArgs, Options } from "../../types";
 import { request } from "../custom/request";
 import { omit } from "../../utils/omit";
@@ -14,11 +13,9 @@ export async function sentenceSimilarity(
 	args: SentenceSimilarityArgs,
 	options?: Options
 ): Promise<SentenceSimilarityOutput> {
-	const defaultTask = args.model ? await getDefaultTask(args.model, args.accessToken, options) : undefined;
 	const res = await request<SentenceSimilarityOutput>(prepareInput(args), {
 		...options,
 		taskHint: "sentence-similarity",
-		...(defaultTask === "feature-extraction" && { forceTask: "sentence-similarity" }),
 	});
 
 	const isValidOutput = Array.isArray(res) && res.every((x) => typeof x === "number");

packages/inference/src/types.ts

@@ -29,15 +29,18 @@ export interface Options {
 export type InferenceTask = Exclude<PipelineType, "other">;
 
 export const INFERENCE_PROVIDERS = [
+	"black-forest-labs",
 	"fal-ai",
 	"fireworks-ai",
+	"hf-inference",
 	"hyperbolic",
 	"nebius",
-	"hf-inference",
+	"novita",
 	"replicate",
 	"sambanova",
 	"together",
 ] as const;
+
 export type InferenceProvider = (typeof INFERENCE_PROVIDERS)[number];
 
 export interface BaseArgs {