[inference] add snippets for image-to-video (#1678)

Add inference snippets for the text-to-video task

---------

Co-authored-by: Celina Hanouti <[email protected]>

Author: SBrandeis

packages/inference/src/snippets/getInferenceSnippets.ts

@@ -389,6 +389,7 @@ const snippets: Partial<
 	"image-text-to-text": snippetGenerator("conversational"),
 	"image-to-image": snippetGenerator("imageToImage", prepareImageToImageInput),
 	"image-to-text": snippetGenerator("basicImage"),
+	"image-to-video": snippetGenerator("imageToVideo", prepareImageToImageInput),
 	"object-detection": snippetGenerator("basicImage"),
 	"question-answering": snippetGenerator("questionAnswering", prepareQuestionAnsweringInput),
 	"sentence-similarity": snippetGenerator("basic"),

packages/inference/src/snippets/templates/js/fetch/imageToVideo.jinja

@@ -0,0 +1,29 @@
+const image = fs.readFileSync("{{inputs.asObj.inputs}}");
+
+async function query(data) {
+	const response = await fetch(
+		"{{ fullUrl }}",
+		{
+			headers: {
+				Authorization: "{{ authorizationHeader }}",
+				"Content-Type": "image/jpeg",
+{% if billTo %}
+				"X-HF-Bill-To": "{{ billTo }}",
+{% endif %}			},
+			method: "POST",
+			body: {
+				"image_url": `data:image/png;base64,${data.image.encode("base64")}`,
+				"prompt": data.prompt,
+			}
+		}
+	);
+	const result = await response.json();
+	return result;
+}
+
+query({
+	"image": image,
+	"prompt": "{{inputs.asObj.parameters.prompt}}",
+}).then((response) => {
+    // Use video
+});

packages/inference/src/snippets/templates/js/huggingface.js/imageToVideo.jinja

@@ -0,0 +1,20 @@
+import { InferenceClient } from "@huggingface/inference";
+
+const client = new InferenceClient("{{ accessToken }}");
+
+const data = fs.readFileSync("{{inputs.asObj.inputs}}");
+
+const video = await client.imageToVideo({
+{% if endpointUrl %}
+	endpointUrl: "{{ endpointUrl }}",
+{% endif %}
+	provider: "{{provider}}",
+	model: "{{model.id}}",
+	inputs: data,
+	parameters: { prompt: "{{inputs.asObj.parameters.prompt}}", },
+}{% if billTo %}, {
+	billTo: "{{ billTo }}",
+}{% endif %});
+
+/// Use the generated video (it's a Blob)
+// For example, you can save it to a file or display it in a video element

packages/inference/src/snippets/templates/python/fal_client/imageToVideo.jinja

@@ -0,0 +1,23 @@
+{%if provider == "fal-ai" %}
+import fal_client
+import base64
+
+def on_queue_update(update):
+    if isinstance(update, fal_client.InProgress):
+        for log in update.logs:
+           print(log["message"])
+
+with open("{{inputs.asObj.inputs}}", "rb") as image_file:
+    image_base_64 = base64.b64encode(image_file.read()).decode('utf-8')
+
+result = fal_client.subscribe(
+    "{{model.id}}",
+    arguments={
+        "image_url": f"data:image/png;base64,{image_base_64}",
+        "prompt": "{{inputs.asObj.parameters.prompt}}",
+    },
+    with_logs=True,
+    on_queue_update=on_queue_update,
+)
+print(result)
+{%endif%}

packages/inference/src/snippets/templates/python/huggingface_hub/imageToImage.jinja

@@ -6,4 +6,4 @@ image = client.image_to_image(
     input_image,
     prompt="{{ inputs.asObj.parameters.prompt }}",
     model="{{ model.id }}",
-) 
+)

packages/inference/src/snippets/templates/python/huggingface_hub/imageToVideo.jinja

@@ -0,0 +1,8 @@
+with open("{{ inputs.asObj.inputs }}", "rb") as image_file:
+   input_image = image_file.read()
+
+video = client.image_to_video(
+    input_image,
+    prompt="{{ inputs.asObj.parameters.prompt }}",
+    model="{{ model.id }}",
+) 

packages/inference/src/snippets/templates/python/requests/imageToImage.jinja

@@ -1,5 +1,3 @@
-with open("{{inputs.asObj.inputs}}", "rb") as image_file:
-    image_base_64 = base64.b64encode(image_file.read()).decode('utf-8')
 
 def query(payload):
     with open(payload["inputs"], "rb") as f:

packages/inference/src/snippets/templates/python/requests/imageToVideo.jinja

@@ -0,0 +1,11 @@
+
+def query(payload):
+    with open(payload["inputs"], "rb") as f:
+        img = f.read()
+        payload["inputs"] = base64.b64encode(img).decode("utf-8")
+    response = requests.post(API_URL, headers=headers, json=payload)
+    return response.content
+
+video_bytes = query({
+{{ inputs.asJsonString }}
+})

packages/tasks-gen/scripts/generate-snippets-fixtures.ts

@@ -140,6 +140,17 @@ const TEST_CASES: {
 		},
 		providers: ["fal-ai", "replicate", "hf-inference"],
 	},
+	{
+		testName: "image-to-video",
+		task: "image-to-video",
+		model: {
+			id: "Wan-AI/Wan2.2-I2V-A14B",
+			pipeline_tag: "image-to-video",
+			tags: [],
+			inference: "",
+		},
+		providers: ["fal-ai"],
+	},
 	{
 		testName: "tabular",
 		task: "tabular-classification",

packages/tasks-gen/snippets-fixtures/image-to-image/python/requests/0.fal-ai.py

@@ -7,9 +7,6 @@
     "Authorization": f"Bearer {os.environ['HF_TOKEN']}",
 }
 
-with open("cat.png", "rb") as image_file:
-    image_base_64 = base64.b64encode(image_file.read()).decode('utf-8')
-
 def query(payload):
     with open(payload["inputs"], "rb") as f:
         img = f.read()