feat. Add text-to-video - Wan2.1-T2V-14B from Novita (#1263)

Co-authored-by: Lucain <[email protected]>
Co-authored-by: Lucain <[email protected]>

Author: viktor2077

packages/inference/src/providers/novita.ts

@@ -16,7 +16,7 @@
  */
 import type { ProviderConfig, UrlParams, HeaderParams, BodyParams } from "../types";
 
-const NOVITA_API_BASE_URL = "https://api.novita.ai/v3/openai";
+const NOVITA_API_BASE_URL = "https://api.novita.ai";
 
 const makeBody = (params: BodyParams): Record<string, unknown> => {
 	return {
@@ -32,9 +32,11 @@ const makeHeaders = (params: HeaderParams): Record<string, string> => {
 const makeUrl = (params: UrlParams): string => {
 	if (params.task === "text-generation") {
 		if (params.chatCompletion) {
-			return `${params.baseUrl}/chat/completions`;
+			return `${params.baseUrl}/v3/openai/chat/completions`;
 		}
-		return `${params.baseUrl}/completions`;
+		return `${params.baseUrl}/v3/openai/completions`;
+	} else if (params.task === "text-to-video") {
+		return `${params.baseUrl}/v3/hf/${params.model}`;
 	}
 	return params.baseUrl;
 };

packages/inference/src/tasks/cv/textToVideo.ts

@@ -20,7 +20,13 @@ interface ReplicateOutput {
 	output: string;
 }
 
-const SUPPORTED_PROVIDERS = ["fal-ai", "replicate"] as const satisfies readonly InferenceProvider[];
+interface NovitaOutput {
+	video: {
+		video_url: string;
+	};
+}
+
+const SUPPORTED_PROVIDERS = ["fal-ai", "novita", "replicate"] as const satisfies readonly InferenceProvider[];
 
 export async function textToVideo(args: TextToVideoArgs, options?: Options): Promise<TextToVideoOutput> {
 	if (!args.provider || !typedInclude(SUPPORTED_PROVIDERS, args.provider)) {
@@ -30,14 +36,13 @@ export async function textToVideo(args: TextToVideoArgs, options?: Options): Pro
 	}
 
 	const payload =
-		args.provider === "fal-ai" || args.provider === "replicate"
+		args.provider === "fal-ai" || args.provider === "replicate" || args.provider === "novita"
 			? { ...omit(args, ["inputs", "parameters"]), ...args.parameters, prompt: args.inputs }
 			: args;
-	const res = await request<FalAiOutput | ReplicateOutput>(payload, {
+	const res = await request<FalAiOutput | ReplicateOutput | NovitaOutput>(payload, {
 		...options,
 		task: "text-to-video",
 	});
-
 	if (args.provider === "fal-ai") {
 		const isValidOutput =
 			typeof res === "object" &&
@@ -51,7 +56,22 @@ export async function textToVideo(args: TextToVideoArgs, options?: Options): Pro
 		if (!isValidOutput) {
 			throw new InferenceOutputError("Expected { video: { url: string } }");
 		}
-		const urlResponse = await fetch(res.video.url);
+		const urlResponse = await fetch((res as FalAiOutput).video.url);
+		return await urlResponse.blob();
+	} else if (args.provider === "novita") {
+		const isValidOutput =
+			typeof res === "object" &&
+			!!res &&
+			"video" in res &&
+			typeof res.video === "object" &&
+			!!res.video &&
+			"video_url" in res.video &&
+			typeof res.video.video_url === "string" &&
+			isUrl(res.video.video_url);
+		if (!isValidOutput) {
+			throw new InferenceOutputError("Expected { video: { video_url: string } }");
+		}
+		const urlResponse = await fetch((res as NovitaOutput).video.video_url);
 		return await urlResponse.blob();
 	} else {
 		/// TODO: Replicate: handle the case where the generation request "times out" / is async (ie output is null)