[Inference] Add imageToVideo inference task (#1578)

Add general support for the `image-to-video` task in
`@huggingface/inference`

cc @zeke (related to #1575 )

---------

Co-authored-by: apolinário <[email protected]>

Author: SBrandeis

packages/inference/src/lib/getProviderHelper.ts

@@ -24,6 +24,7 @@ import type {
 	ImageSegmentationTaskHelper,
 	ImageToImageTaskHelper,
 	ImageToTextTaskHelper,
+	ImageToVideoTaskHelper,
 	ObjectDetectionTaskHelper,
 	QuestionAnsweringTaskHelper,
 	SentenceSimilarityTaskHelper,
@@ -239,6 +240,10 @@ export function getProviderHelper(
 	provider: InferenceProviderOrPolicy,
 	task: "image-to-image"
 ): ImageToImageTaskHelper & TaskProviderHelper;
+export function getProviderHelper(
+	provider: InferenceProviderOrPolicy,
+	task: "image-to-video"
+): ImageToVideoTaskHelper & TaskProviderHelper;
 export function getProviderHelper(
 	provider: InferenceProviderOrPolicy,
 	task: "sentence-similarity"
@@ -275,7 +280,6 @@ export function getProviderHelper(
 	provider: InferenceProviderOrPolicy,
 	task: InferenceTask | undefined
 ): TaskProviderHelper;
-
 export function getProviderHelper(
 	provider: InferenceProviderOrPolicy,
 	task: InferenceTask | undefined

packages/inference/src/providers/providerHelper.ts

@@ -18,6 +18,7 @@ import type {
 	ImageToImageInput,
 	ImageToTextInput,
 	ImageToTextOutput,
+	ImageToVideoInput,
 	ObjectDetectionInput,
 	ObjectDetectionOutput,
 	QuestionAnsweringInput,
@@ -52,6 +53,7 @@ import type { BaseArgs, BodyParams, HeaderParams, InferenceProvider, RequestArgs
 import { toArray } from "../utils/toArray.js";
 import type { ImageToImageArgs } from "../tasks/cv/imageToImage.js";
 import type { AutomaticSpeechRecognitionArgs } from "../tasks/audio/automaticSpeechRecognition.js";
+import type { ImageToVideoArgs } from "../tasks/cv/imageToVideo.js";
 
 /**
  * Base class for task-specific provider helpers
@@ -150,6 +152,12 @@ export interface ImageToImageTaskHelper {
 	preparePayloadAsync(args: ImageToImageArgs): Promise<RequestArgs>;
 }
 
+export interface ImageToVideoTaskHelper {
+	getResponse(response: unknown, url?: string, headers?: HeadersInit): Promise<Blob>;
+	preparePayload(params: BodyParams<ImageToVideoInput & BaseArgs>): Record<string, unknown>;
+	preparePayloadAsync(args: ImageToVideoArgs): Promise<RequestArgs>;
+}
+
 export interface ImageSegmentationTaskHelper {
 	getResponse(response: unknown, url?: string, headers?: HeadersInit): Promise<ImageSegmentationOutput>;
 	preparePayload(params: BodyParams<ImageSegmentationInput & BaseArgs>): Record<string, unknown> | BodyInit;

packages/inference/src/tasks/cv/imageToVideo.ts

@@ -0,0 +1,24 @@
+import type { ImageToVideoInput } from "@huggingface/tasks";
+import { resolveProvider } from "../../lib/getInferenceProviderMapping.js";
+import { getProviderHelper } from "../../lib/getProviderHelper.js";
+import type { BaseArgs, Options } from "../../types.js";
+import { innerRequest } from "../../utils/request.js";
+import { makeRequestOptions } from "../../lib/makeRequestOptions.js";
+
+export type ImageToVideoArgs = BaseArgs & ImageToVideoInput;
+
+/**
+ * This task reads some text input and outputs an image.
+ * Recommended model: Wan-AI/Wan2.1-I2V-14B-720P
+ */
+export async function imageToVideo(args: ImageToVideoArgs, options?: Options): Promise<Blob> {
+	const provider = await resolveProvider(args.provider, args.model, args.endpointUrl);
+	const providerHelper = getProviderHelper(provider, "image-to-video");
+	const payload = await providerHelper.preparePayloadAsync(args);
+	const { data: res } = await innerRequest<Blob>(payload, providerHelper, {
+		...options,
+		task: "image-to-video",
+	});
+	const { url, info } = await makeRequestOptions(args, providerHelper, { ...options, task: "image-to-video" });
+	return providerHelper.getResponse(res, url, info.headers as Record<string, string>);
+}

packages/inference/src/tasks/index.ts

@@ -13,6 +13,7 @@ export * from "./cv/imageClassification.js";
 export * from "./cv/imageSegmentation.js";
 export * from "./cv/imageToImage.js";
 export * from "./cv/imageToText.js";
+export * from "./cv/imageToVideo.js";
 export * from "./cv/objectDetection.js";
 export * from "./cv/textToImage.js";
 export * from "./cv/textToVideo.js";