Merge branch 'main' into xsn/ollama_utils

ngxson · web-flow · commit 1724bfaf1b40 · 2025-02-05T23:13:41.000+01:00
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -14,5 +14,8 @@
 	},
 	"cSpell.words": ["huggingface"],
 	"deno.enablePaths": ["./e2e/deno"],
+	"search.exclude": {
+		"**/dist": true
+	},
 	"typescript.tsdk": "node_modules/typescript/lib"
 }
diff --git a/packages/inference/test/tapes.json b/packages/inference/test/tapes.json
@@ -6789,5 +6789,45 @@
         "server": "UploadServer"
       }
     }
+  },
+  "efa2b5ab7171e43629fef33886a32583919f4dfe814ae07a44db19257ee123ae": {
+    "url": "https://fal.run/fal-ai/fast-sdxl",
+    "init": {
+      "headers": {
+        "Content-Type": "application/json"
+      },
+      "method": "POST",
+      "body": "{\"response_format\":\"base64\",\"prompt\":\"Extreme close-up of a single tiger eye, direct frontal view. Detailed iris and pupil. Sharp focus on eye texture and color. Natural lighting to capture authentic eye shine and depth.\"}"
+    },
+    "response": {
+      "body": "{\"images\":[{\"url\":\"https://fal.media/files/monkey/t28MYvYK21vq9nIypBm0P.jpeg\",\"width\":1024,\"height\":1024,\"content_type\":\"image/jpeg\"}],\"timings\":{\"inference\":2.1236871778964996},\"seed\":15619174981588513000,\"has_nsfw_concepts\":[false],\"prompt\":\"Extreme close-up of a single tiger eye, direct frontal view. Detailed iris and pupil. Sharp focus on eye texture and color. Natural lighting to capture authentic eye shine and depth.\"}",
+      "status": 200,
+      "statusText": "OK",
+      "headers": {
+        "connection": "keep-alive",
+        "content-type": "application/json",
+        "strict-transport-security": "max-age=31536000; includeSubDomains"
+      }
+    }
+  },
+  "374890ec5b45788656310c21999957168f47242bd379c91da86d00eab7b9b218": {
+    "url": "https://fal.media/files/monkey/t28MYvYK21vq9nIypBm0P.jpeg",
+    "init": {},
+    "response": {
+      "body": "",
+      "status": 200,
+      "statusText": "OK",
+      "headers": {
+        "access-control-allow-headers": "*",
+        "access-control-allow-methods": "*",
+        "access-control-allow-origin": "*",
+        "access-control-max-age": "86400",
+        "cf-ray": "90d404087de9999f-CDG",
+        "connection": "keep-alive",
+        "content-type": "image/jpeg",
+        "server": "cloudflare",
+        "vary": "Accept-Encoding"
+      }
+    }
   }
 }
diff --git a/packages/inference/test/vcr.ts b/packages/inference/test/vcr.ts
@@ -117,7 +117,7 @@ async function vcr(
 
 	const { default: tapes } = await import(TAPES_FILE);
 
-	const cacheCandidate = !url.startsWith(HF_HUB_URL) || url.startsWith("https://huggingface.co/api/inference-proxy/");
+	const cacheCandidate = !url.startsWith(HF_HUB_URL);
 
 	if (VCR_MODE === MODE.PLAYBACK && cacheCandidate) {
 		if (!tapes[hash]) {
diff --git a/packages/tasks-gen/scripts/inference-tei-import.ts b/packages/tasks-gen/scripts/inference-tei-import.ts
@@ -63,8 +63,9 @@ async function _extractAndAdapt(task: string, mainComponentName: string, type: "
 					// but not Union[List[Union[List[int], int, str]], str]
 					// data.delete(key);
 					delete data[key];
-					data["type"] = "string";
-					data["description"] = "The text to embed.";
+					data["title"] = "FeatureExtractionInputs";
+					data["description"] = "The text or list of texts to embed.";
+					data["oneOf"] = [{ type: "string" }, { type: "array", items: { type: "string" } }];
 				} else if (key === "$ref" && typeof data[key] === "string") {
 					// Verify reference exists
 					const ref = (data[key] as string).split("/").pop() ?? "";
diff --git a/packages/tasks/src/inference-providers.ts b/packages/tasks/src/inference-providers.ts
@@ -1,16 +1,24 @@
-export const INFERENCE_PROVIDERS = ["hf-inference", "fal-ai", "replicate", "sambanova", "together"] as const;
+/// This list is for illustration purposes only.
+/// in the `tasks` sub-package, we do not need actual strong typing of the inference providers.
+const INFERENCE_PROVIDERS = [
+	"fal-ai",
+	"fireworks-ai",
+	"hf-inference",
+	"hyperbolic",
+	"replicate",
+	"sambanova",
+	"together",
+] as const;
 
-export type InferenceProvider = (typeof INFERENCE_PROVIDERS)[number];
+export type SnippetInferenceProvider = (typeof INFERENCE_PROVIDERS)[number] | string;
 
-export const HF_HUB_INFERENCE_PROXY_TEMPLATE = `https://huggingface.co/api/inference-proxy/{{PROVIDER}}`;
+export const HF_HUB_INFERENCE_PROXY_TEMPLATE = `https://router.huggingface.co/{{PROVIDER}}`;
 
 /**
  * URL to set as baseUrl in the OpenAI SDK.
  *
  * TODO(Expose this from HfInference in the future?)
  */
-export function openAIbaseUrl(provider: InferenceProvider): string {
-	return provider === "hf-inference"
-		? "https://api-inference.huggingface.co/v1/"
-		: HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", provider);
+export function openAIbaseUrl(provider: SnippetInferenceProvider): string {
+	return HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", provider);
 }
diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts
@@ -83,6 +83,20 @@ function get_prompt_from_diffusers_model(model: ModelData): string | undefined {
 	}
 }
 
+export const ben2 = (model: ModelData): string[] => [
+	`import requests
+from PIL import Image
+from ben2 import AutoModel
+
+url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/teapot.jpg"
+image = Image.open(requests.get(url, stream=True).raw)
+
+model = AutoModel.from_pretrained("${model.id}")
+model.to("cuda").eval()
+foreground = model.inference(image)
+`,
+];
+
 export const bertopic = (model: ModelData): string[] => [
 	`from bertopic import BERTopic
 
diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts
@@ -109,6 +109,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		countDownloads: `path_extension:"pth"`,
 		snippets: snippets.audioseal,
 	},
+	ben2: {
+		prettyLabel: "BEN2",
+		repoName: "BEN2",
+		repoUrl: "https://github.com/PramaLLC/BEN2",
+		snippets: snippets.ben2,
+		filter: false,
+	},
 	bertopic: {
 		prettyLabel: "BERTopic",
 		repoName: "BERTopic",
diff --git a/packages/tasks/src/pipelines.ts b/packages/tasks/src/pipelines.ts
@@ -676,6 +676,12 @@ export const PIPELINE_DATA = {
 		color: "red",
 		hideInDatasets: true,
 	},
+	"visual-document-retrieval": {
+		name: "Visual Document Retrieval",
+		modality: "multimodal",
+		color: "yellow",
+		hideInDatasets: true,
+	},
 	"any-to-any": {
 		name: "Any-to-Any",
 		modality: "multimodal",
diff --git a/packages/tasks/src/snippets/curl.ts b/packages/tasks/src/snippets/curl.ts
@@ -1,4 +1,4 @@
-import { HF_HUB_INFERENCE_PROXY_TEMPLATE, type InferenceProvider } from "../inference-providers.js";
+import { HF_HUB_INFERENCE_PROXY_TEMPLATE, type SnippetInferenceProvider } from "../inference-providers.js";
 import type { PipelineType } from "../pipelines.js";
 import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js";
 import { stringifyGenerationConfig, stringifyMessages } from "./common.js";
@@ -8,7 +8,7 @@ import type { InferenceSnippet, ModelDataMinimal } from "./types.js";
 export const snippetBasic = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -29,7 +29,7 @@ curl https://api-inference.huggingface.co/models/${model.id} \\
 export const snippetTextGeneration = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: {
 		streaming?: boolean;
 		messages?: ChatCompletionInputMessage[];
@@ -84,7 +84,7 @@ export const snippetTextGeneration = (
 export const snippetZeroShotClassification = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -104,7 +104,7 @@ export const snippetZeroShotClassification = (
 export const snippetFile = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -126,7 +126,7 @@ export const curlSnippets: Partial<
 		(
 			model: ModelDataMinimal,
 			accessToken: string,
-			provider: InferenceProvider,
+			provider: SnippetInferenceProvider,
 			opts?: Record<string, unknown>
 		) => InferenceSnippet[]
 	>
@@ -160,7 +160,7 @@ export const curlSnippets: Partial<
 export function getCurlInferenceSnippet(
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: Record<string, unknown>
 ): InferenceSnippet[] {
 	return model.pipeline_tag && model.pipeline_tag in curlSnippets
diff --git a/packages/tasks/src/snippets/js.ts b/packages/tasks/src/snippets/js.ts
@@ -1,4 +1,4 @@
-import { openAIbaseUrl, type InferenceProvider } from "../inference-providers.js";
+import { openAIbaseUrl, type SnippetInferenceProvider } from "../inference-providers.js";
 import type { PipelineType } from "../pipelines.js";
 import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js";
 import { stringifyGenerationConfig, stringifyMessages } from "./common.js";
@@ -22,7 +22,7 @@ const HFJS_METHODS: Record<string, string> = {
 export const snippetBasic = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	return [
 		...(model.pipeline_tag && model.pipeline_tag in HFJS_METHODS
@@ -74,7 +74,7 @@ query({"inputs": ${getModelInputSnippet(model)}}).then((response) => {
 export const snippetTextGeneration = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: {
 		streaming?: boolean;
 		messages?: ChatCompletionInputMessage[];
@@ -225,7 +225,7 @@ export const snippetZeroShotClassification = (model: ModelDataMinimal, accessTok
 export const snippetTextToImage = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	return [
 		{
@@ -275,7 +275,7 @@ query({"inputs": ${getModelInputSnippet(model)}}).then((response) => {
 export const snippetTextToAudio = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -329,7 +329,7 @@ export const snippetTextToAudio = (
 export const snippetAutomaticSpeechRecognition = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	return [
 		{
@@ -357,7 +357,7 @@ console.log(output);
 export const snippetFile = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	if (provider !== "hf-inference") {
 		return [];
@@ -395,7 +395,7 @@ export const jsSnippets: Partial<
 		(
 			model: ModelDataMinimal,
 			accessToken: string,
-			provider: InferenceProvider,
+			provider: SnippetInferenceProvider,
 			opts?: Record<string, unknown>
 		) => InferenceSnippet[]
 	>
@@ -429,7 +429,7 @@ export const jsSnippets: Partial<
 export function getJsInferenceSnippet(
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: Record<string, unknown>
 ): InferenceSnippet[] {
 	return model.pipeline_tag && model.pipeline_tag in jsSnippets
diff --git a/packages/tasks/src/snippets/python.ts b/packages/tasks/src/snippets/python.ts
@@ -1,11 +1,15 @@
-import { HF_HUB_INFERENCE_PROXY_TEMPLATE, openAIbaseUrl, type InferenceProvider } from "../inference-providers.js";
+import {
+	HF_HUB_INFERENCE_PROXY_TEMPLATE,
+	openAIbaseUrl,
+	type SnippetInferenceProvider,
+} from "../inference-providers.js";
 import type { PipelineType } from "../pipelines.js";
 import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js";
 import { stringifyGenerationConfig, stringifyMessages } from "./common.js";
 import { getModelInputSnippet } from "./inputs.js";
 import type { InferenceSnippet, ModelDataMinimal } from "./types.js";
 
-const snippetImportInferenceClient = (accessToken: string, provider: InferenceProvider): string =>
+const snippetImportInferenceClient = (accessToken: string, provider: SnippetInferenceProvider): string =>
 	`\
 from huggingface_hub import InferenceClient
 
@@ -17,7 +21,7 @@ client = InferenceClient(
 export const snippetConversational = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: {
 		streaming?: boolean;
 		messages?: ChatCompletionInputMessage[];
@@ -199,7 +203,7 @@ output = query(${getModelInputSnippet(model)})`,
 export const snippetTextToImage = (
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider
+	provider: SnippetInferenceProvider
 ): InferenceSnippet[] => {
 	return [
 		{
@@ -337,7 +341,7 @@ export const pythonSnippets: Partial<
 		(
 			model: ModelDataMinimal,
 			accessToken: string,
-			provider: InferenceProvider,
+			provider: SnippetInferenceProvider,
 			opts?: Record<string, unknown>
 		) => InferenceSnippet[]
 	>
@@ -375,7 +379,7 @@ export const pythonSnippets: Partial<
 export function getPythonInferenceSnippet(
 	model: ModelDataMinimal,
 	accessToken: string,
-	provider: InferenceProvider,
+	provider: SnippetInferenceProvider,
 	opts?: Record<string, unknown>
 ): InferenceSnippet[] {
 	if (model.tags.includes("conversational")) {
diff --git a/packages/tasks/src/tasks/feature-extraction/spec/input.json b/packages/tasks/src/tasks/feature-extraction/spec/input.json
@@ -8,11 +8,18 @@
 	"properties": {
 		"inputs": {
 			"title": "FeatureExtractionInputs",
+			"description": "The text or list of texts to embed.",
 			"oneOf": [
-				{ "type": "string" },
-				{ "type": "array", "items": { "type": "string" } }
-			],
-			"description": "The text or list of texts to embed."
+				{
+					"type": "string"
+				},
+				{
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			]
 		},
 		"normalize": {
 			"type": "boolean",
diff --git a/packages/tasks/src/tasks/index.ts b/packages/tasks/src/tasks/index.ts
@@ -174,6 +174,7 @@ export const TASKS_MODEL_LIBRARIES: Record<PipelineType, ModelLibraryKey[]> = {
 	"text-to-3d": ["diffusers"],
 	"image-to-3d": ["diffusers"],
 	"any-to-any": ["transformers"],
+	"visual-document-retrieval": ["transformers"],
 };
 
 /**
@@ -202,6 +203,7 @@ export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
 	"automatic-speech-recognition": getData("automatic-speech-recognition", automaticSpeechRecognition),
 	"depth-estimation": getData("depth-estimation", depthEstimation),
 	"document-question-answering": getData("document-question-answering", documentQuestionAnswering),
+	"visual-document-retrieval": getData("visual-document-retrieval", placeholder),
 	"feature-extraction": getData("feature-extraction", featureExtraction),
 	"fill-mask": getData("fill-mask", fillMask),
 	"graph-ml": undefined,
diff --git a/packages/tasks/src/tasks/text-generation/data.ts b/packages/tasks/src/tasks/text-generation/data.ts
@@ -61,9 +61,7 @@ const taskData: TaskDataCustom = {
 		},
 	],
 	models: [
-		{	description: "A text-generation model trained to follow instructions.",
-			id: "google/gemma-2-2b-it",
-		},
+		{ description: "A text-generation model trained to follow instructions.", id: "google/gemma-2-2b-it" },
 		{
 			description: "Smaller variant of one of the most powerful models.",
 			id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
diff --git a/packages/widgets/src/lib/components/Icons/IconVisualDocumentRetrieval.svelte b/packages/widgets/src/lib/components/Icons/IconVisualDocumentRetrieval.svelte
diff --git a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte

Original file line number	Diff line number	Diff line change
`@@ -14,5 +14,8 @@`
`14`	`14`	`},`
`15`	`15`	`"cSpell.words": ["huggingface"],`
`16`	`16`	`"deno.enablePaths": ["./e2e/deno"],`
	`17`	`+ "search.exclude": {`
	`18`	`+ "**/dist": true`
	`19`	`+ },`
`17`	`20`	`"typescript.tsdk": "node_modules/typescript/lib"`
`18`	`21`	`}`