[tasks] in inference snippets, provider does not need to be strongly typed (#1184)

julien-c · SBrandeis · web-flow · commit d515d60b89f3 · 2025-02-05T16:49:29.000+01:00
will make things easier when adding new providers (one less moving part)

---------

Co-authored-by: SBrandeis &lt;simon@huggingface.co&gt;
diff --git a/packages/inference/test/tapes.json b/packages/inference/test/tapes.json
@@ -6789,5 +6789,45 @@
         "server": "UploadServer"
       }
     }
+  },
+  "efa2b5ab7171e43629fef33886a32583919f4dfe814ae07a44db19257ee123ae": {
+    "url": "https://fal.run/fal-ai/fast-sdxl",
+    "init": {
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "POST",
+      "body": "{\"response_format\":\"base64\",\"prompt\":\"Extreme close-up of a single tiger eye, direct frontal view. Detailed iris and pupil. Sharp focus on eye texture and color. Natural lighting to capture authentic eye shine and depth.\"}"
+    },
+    "response": {
+      "body": "{\"images\":[{\"url\":\"https://fal.media/files/monkey/t28MYvYK21vq9nIypBm0P.jpeg\",\"width\":1024,\"height\":1024,\"content_type\":\"image/jpeg\"}],\"timings\":{\"inference\":2.1236871778964996},\"seed\":15619174981588513000,\"has_nsfw_concepts\":[false],\"prompt\":\"Extreme close-up of a single tiger eye, direct frontal view. Detailed iris and pupil. Sharp focus on eye texture and color. Natural lighting to capture authentic eye shine and depth.\"}",
+      "status": 200,
+      "statusText": "OK",
+      "headers": {
+        "connection": "keep-alive",
+        "content-type": "application/json",
+        "strict-transport-security": "max-age=31536000; includeSubDomains"
+      }
+    }
+  },
+  "374890ec5b45788656310c21999957168f47242bd379c91da86d00eab7b9b218": {
+    "url": "https://fal.media/files/monkey/t28MYvYK21vq9nIypBm0P.jpeg",
+    "init": {},
+    "response": {
+      "body": "",
+      "status": 200,
+      "statusText": "OK",
+      "headers": {
+        "access-control-allow-headers": "*",
+        "access-control-allow-methods": "*",
+        "access-control-allow-origin": "*",
+        "access-control-max-age": "86400",
+        "cf-ray": "90d404087de9999f-CDG",
+        "connection": "keep-alive",
+        "content-type": "image/jpeg",
+        "server": "cloudflare",
+        "vary": "Accept-Encoding"
+      }
+    }
   }
 }
diff --git a/packages/inference/test/vcr.ts b/packages/inference/test/vcr.ts
@@ -117,7 +117,7 @@ async function vcr(
 
 	const { default: tapes } = await import(TAPES_FILE);
 
-	const cacheCandidate = !url.startsWith(HF_HUB_URL) || url.startsWith("https://huggingface.co/api/inference-proxy/");
+	const cacheCandidate = !url.startsWith(HF_HUB_URL);
 
 	if (VCR_MODE === MODE.PLAYBACK && cacheCandidate) {
 		if (!tapes[hash]) {
diff --git a/packages/tasks/src/inference-providers.ts b/packages/tasks/src/inference-providers.ts
@@ -1,16 +1,24 @@
-export const INFERENCE_PROVIDERS = ["hf-inference", "fal-ai", "replicate", "sambanova", "together"] as const;
+/// This list is for illustration purposes only.
+/// in the `tasks` sub-package, we do not need actual strong typing of the inference providers.
+const INFERENCE_PROVIDERS = [
+	"fal-ai",
+	"fireworks-ai",
+	"hf-inference",
+	"hyperbolic",
+	"replicate",
+	"sambanova",
+	"together",
+] as const;
 
-export type InferenceProvider = (typeof INFERENCE_PROVIDERS)[number];
+export type SnippetInferenceProvider = (typeof INFERENCE_PROVIDERS)[number] | string;
 
-export const HF_HUB_INFERENCE_PROXY_TEMPLATE = `https://huggingface.co/api/inference-proxy/{{PROVIDER}}`;
+export const HF_HUB_INFERENCE_PROXY_TEMPLATE = `https://router.huggingface.co/{{PROVIDER}}`;
 
 /**
  * URL to set as baseUrl in the OpenAI SDK.
  *
  * TODO(Expose this from HfInference in the future?)
  */
-export function openAIbaseUrl(provider: InferenceProvider): string {
-	return provider === "hf-inference"
-		? "https://api-inference.huggingface.co/v1/"
-		: HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", provider);
+export function openAIbaseUrl(provider: SnippetInferenceProvider): string {
+	return HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", provider);
 }
diff --git a/packages/tasks/src/snippets/curl.ts b/packages/tasks/src/snippets/curl.ts
@@ -1,4 +1,4 @@
-import { HF_HUB_INFERENCE_PROXY_TEMPLATE, type InferenceProvider } from "../inference-providers.js";
+import { HF_HUB_INFERENCE_PROXY_TEMPLATE, type SnippetInferenceProvider } from "../inference-providers.js";
 import type { PipelineType } from "../pipelines.js";
 import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js";
 import { stringifyGenerationConfig, stringifyMessages } from "./common.js";
@@ -8,7 +8,7 @@ import type { InferenceSnippet, ModelDataMinimal } from "./types.js";
 export const snippetBasic = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -29,7 +29,7 @@ curl https://api-inference.huggingface.co/models/${model.id} \\
 export const snippetTextGeneration = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: {
 		streaming?: boolean;
 		messages?: ChatCompletionInputMessage[];
@@ -84,7 +84,7 @@ export const snippetTextGeneration = (
 export const snippetZeroShotClassification = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -104,7 +104,7 @@ export const snippetZeroShotClassification = (
 export const snippetFile = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -126,7 +126,7 @@ export const curlSnippets: Partial<
 		(
 			model: ModelDataMinimal,
 			accessToken: string,
-			provider: InferenceProvider,
+			provider: SnippetInferenceProvider,
 			opts?: Record<string, unknown>
 		) => InferenceSnippet[]
 	>
@@ -160,7 +160,7 @@ export const curlSnippets: Partial<
 export function getCurlInferenceSnippet(
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: Record<string, unknown>
 ): InferenceSnippet[] {
 	return model.pipeline_tag && model.pipeline_tag in curlSnippets
diff --git a/packages/tasks/src/snippets/js.ts b/packages/tasks/src/snippets/js.ts
@@ -1,4 +1,4 @@
-import { openAIbaseUrl, type InferenceProvider } from "../inference-providers.js";
+import { openAIbaseUrl, type SnippetInferenceProvider } from "../inference-providers.js";
 import type { PipelineType } from "../pipelines.js";
 import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js";
 import { stringifyGenerationConfig, stringifyMessages } from "./common.js";
@@ -22,7 +22,7 @@ const HFJS_METHODS: Record<string, string> = {
 export const snippetBasic = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	return [
 		...(model.pipeline_tag && model.pipeline_tag in HFJS_METHODS
@@ -74,7 +74,7 @@ query({"inputs": ${getModelInputSnippet(model)}}).then((response) => {
 export const snippetTextGeneration = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: {
 		streaming?: boolean;
 		messages?: ChatCompletionInputMessage[];
@@ -225,7 +225,7 @@ export const snippetZeroShotClassification = (model: ModelDataMinimal, accessTok
 export const snippetTextToImage = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	return [
 		{
@@ -275,7 +275,7 @@ query({"inputs": ${getModelInputSnippet(model)}}).then((response) => {
 export const snippetTextToAudio = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -329,7 +329,7 @@ export const snippetTextToAudio = (
 export const snippetAutomaticSpeechRecognition = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	return [
 		{
@@ -357,7 +357,7 @@ console.log(output);
 export const snippetFile = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -395,7 +395,7 @@ export const jsSnippets: Partial<
 		(
 			model: ModelDataMinimal,
 			accessToken: string,
-			provider: InferenceProvider,
+			provider: SnippetInferenceProvider,
 			opts?: Record<string, unknown>
 		) => InferenceSnippet[]
 	>
@@ -429,7 +429,7 @@ export const jsSnippets: Partial<
 export function getJsInferenceSnippet(
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: Record<string, unknown>
 ): InferenceSnippet[] {
 	return model.pipeline_tag && model.pipeline_tag in jsSnippets
diff --git a/packages/tasks/src/snippets/python.ts b/packages/tasks/src/snippets/python.ts
@@ -1,11 +1,15 @@
-import { HF_HUB_INFERENCE_PROXY_TEMPLATE, openAIbaseUrl, type InferenceProvider } from "../inference-providers.js";
+import {
+	HF_HUB_INFERENCE_PROXY_TEMPLATE,
+	openAIbaseUrl,
+	type SnippetInferenceProvider,
+} from "../inference-providers.js";
 import type { PipelineType } from "../pipelines.js";
 import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js";
 import { stringifyGenerationConfig, stringifyMessages } from "./common.js";
 import { getModelInputSnippet } from "./inputs.js";
 import type { InferenceSnippet, ModelDataMinimal } from "./types.js";
 
-const snippetImportInferenceClient = (accessToken: string, provider: InferenceProvider): string =>
+const snippetImportInferenceClient = (accessToken: string, provider: SnippetInferenceProvider): string =>
 	`\
 from huggingface_hub import InferenceClient
 
@@ -17,7 +21,7 @@ client = InferenceClient(
 export const snippetConversational = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: {
 		streaming?: boolean;
 		messages?: ChatCompletionInputMessage[];
@@ -199,7 +203,7 @@ output = query(${getModelInputSnippet(model)})`,
 export const snippetTextToImage = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	return [
 		{
@@ -337,7 +341,7 @@ export const pythonSnippets: Partial<
 		(
 			model: ModelDataMinimal,
 			accessToken: string,
-			provider: InferenceProvider,
+			provider: SnippetInferenceProvider,
 			opts?: Record<string, unknown>
 		) => InferenceSnippet[]
 	>
@@ -375,7 +379,7 @@ export const pythonSnippets: Partial<
 export function getPythonInferenceSnippet(
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: Record<string, unknown>
 ): InferenceSnippet[] {
 	if (model.tags.includes("conversational")) {
