Add code snippets for image-text-to-text

Author: Wauplin

packages/tasks/src/snippets/curl.ts

@@ -27,6 +27,32 @@ export const snippetTextGeneration = (model: ModelDataMinimal, accessToken: stri
 	}
 };
 
+export const snippetImageTextToTextGeneration = (model: ModelDataMinimal, accessToken: string): string => {
+	if (model.config?.tokenizer_config?.chat_template) {
+		// Conversational model detected, so we display a code snippet that features the Messages API
+		return `curl 'https://api-inference.huggingface.co/models/${model.id}/v1/chat/completions' \\
+-H "Authorization: Bearer ${accessToken || `{API_TOKEN}`}" \\
+-H 'Content-Type: application/json' \\
+-d '{
+	"model": "${model.id}",
+	"messages": [
+		{
+			"role": "user",
+			"content": [
+				{"type": "image_url", "image_url": {"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"}},
+				{"type": "text", "text": "Describe this image in one sentence."},
+			],
+		}
+	],
+	"max_tokens": 500,
+	"stream": false
+}'
+`;
+	} else {
+		return snippetBasic(model, accessToken);
+	}
+};
+
 export const snippetZeroShotClassification = (model: ModelDataMinimal, accessToken: string): string =>
 	`curl https://api-inference.huggingface.co/models/${model.id} \\
 	-X POST \\
@@ -51,6 +77,7 @@ export const curlSnippets: Partial<Record<PipelineType, (model: ModelDataMinimal
 	summarization: snippetBasic,
 	"feature-extraction": snippetBasic,
 	"text-generation": snippetTextGeneration,
+	"image-text-to-text": snippetImageTextToTextGeneration,
 	"text2text-generation": snippetBasic,
 	"fill-mask": snippetBasic,
 	"sentence-similarity": snippetBasic,

packages/tasks/src/snippets/js.ts

@@ -41,6 +41,35 @@ for await (const chunk of inference.chatCompletionStream({
 		return snippetBasic(model, accessToken);
 	}
 };
+
+export const snippetImageTextToTextGeneration = (model: ModelDataMinimal, accessToken: string): string => {
+	if (model.config?.tokenizer_config?.chat_template) {
+		// Conversational model detected, so we display a code snippet that features the Messages API
+		return `import { HfInference } from "@huggingface/inference";
+
+const inference = new HfInference("${accessToken || `{API_TOKEN}`}");
+const image_url = "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
+
+for await (const chunk of inference.chatCompletionStream({
+	model: "${model.id}",
+	messages: [
+		{
+			"role": "user",
+			"content": [
+				{"type": "image_url", "image_url": {"url": image_url}},
+				{"type": "text", "text": "Describe this image in one sentence."},
+			],
+		}
+	],
+	max_tokens: 500,
+})) {
+	process.stdout.write(chunk.choices[0]?.delta?.content || "");
+}`;
+	} else {
+		return snippetBasic(model, accessToken);
+	}
+};
+
 export const snippetZeroShotClassification = (model: ModelDataMinimal, accessToken: string): string =>
 	`async function query(data) {
 	const response = await fetch(
@@ -156,6 +185,7 @@ export const jsSnippets: Partial<Record<PipelineType, (model: ModelDataMinimal,
 	summarization: snippetBasic,
 	"feature-extraction": snippetBasic,
 	"text-generation": snippetTextGeneration,
+	"image-text-to-text": snippetImageTextToTextGeneration,
 	"text2text-generation": snippetBasic,
 	"fill-mask": snippetBasic,
 	"sentence-similarity": snippetBasic,

packages/tasks/src/snippets/python.ts

@@ -17,6 +17,30 @@ for message in client.chat_completion(
 ):
     print(message.choices[0].delta.content, end="")`;
 
+export const snippetConversationalWithImage = (model: ModelDataMinimal, accessToken: string): string =>
+	`from huggingface_hub import InferenceClient
+
+client = InferenceClient(
+	"${model.id}",
+	token="${accessToken || "{API_TOKEN}"}",
+)
+image_url = "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
+
+for message in client.chat_completion(
+	messages=[
+		{
+			"role": "user",
+			"content": [
+				{"type": "image_url", "image_url": {"url": image_url}},
+				{"type": "text", "text": "Describe this image in one sentence."},
+			],
+		}
+	],
+	max_tokens=500,
+	stream=True,
+):
+	print(message.choices[0].delta.content, end="")`;
+
 export const snippetZeroShotClassification = (model: ModelDataMinimal): string =>
 	`def query(payload):
 	response = requests.post(API_URL, headers=headers, json=payload)
@@ -156,6 +180,9 @@ export function getPythonInferenceSnippet(model: ModelDataMinimal, accessToken:
 	if (model.pipeline_tag === "text-generation" && model.config?.tokenizer_config?.chat_template) {
 		// Conversational model detected, so we display a code snippet that features the Messages API
 		return snippetConversational(model, accessToken);
+	} else if (model.pipeline_tag === "image-text-to-text" && model.config?.tokenizer_config?.chat_template) {
+		// Example sending an image to the Message API
+		return snippetConversationalWithImage(model, accessToken);
 	} else {
 		const body =
 			model.pipeline_tag && model.pipeline_tag in pythonSnippets