image-to-iùage fal-ai

Author: SBrandeis

packages/inference/src/lib/getProviderHelper.ts

@@ -64,6 +64,7 @@ export const PROVIDERS: Record<InferenceProvider, Partial<Record<InferenceTask,
 		"text-to-image": new FalAI.FalAITextToImageTask(),
 		"text-to-speech": new FalAI.FalAITextToSpeechTask(),
 		"text-to-video": new FalAI.FalAITextToVideoTask(),
+		"image-to-image": new FalAI.FalAIImageToImageTask(),
 		"automatic-speech-recognition": new FalAI.FalAIAutomaticSpeechRecognitionTask(),
 	},
 	"featherless-ai": {

packages/inference/src/providers/fal-ai.ts

@@ -21,6 +21,9 @@ import { isUrl } from "../lib/isUrl.js";
 import type { BodyParams, HeaderParams, ModelId, RequestArgs, UrlParams } from "../types.js";
 import { delay } from "../utils/delay.js";
 import { omit } from "../utils/omit.js";
+import type {
+	ImageToImageTaskHelper
+} from "./providerHelper.js";
 import {
 	type AutomaticSpeechRecognitionTaskHelper,
 	TaskProviderHelper,
@@ -34,6 +37,7 @@ import {
 	InferenceClientProviderApiError,
 	InferenceClientProviderOutputError,
 } from "../errors.js";
+import type { ImageToImageArgs } from "../tasks/index.js";
 
 export interface FalAiQueueOutput {
 	request_id: string;
@@ -130,6 +134,118 @@ export class FalAITextToImageTask extends FalAITask implements TextToImageTaskHe
 	}
 }
 
+export class FalAIImageToImageTask extends FalAITask implements ImageToImageTaskHelper {
+	constructor() {
+		super("https://queue.fal.run");
+	}
+
+
+	override makeRoute(params: UrlParams): string {
+		if (params.authMethod !== "provider-key") {
+			return `/${params.model}?_subdomain=queue`;
+		}
+		return `/${params.model}`;
+	}
+
+	async preparePayloadAsync(args: ImageToImageArgs): Promise<RequestArgs> {
+		const mimeType = args.inputs instanceof Blob ? args.inputs.type : "image/png";
+		return {
+			...omit(args, ["inputs", "parameters"]),
+			image_url: `data:${mimeType};base64,${base64FromBytes(
+				new Uint8Array(args.inputs instanceof ArrayBuffer ? args.inputs : await (args.inputs as Blob).arrayBuffer())
+			)}`,
+			...args.parameters,
+			...args,
+		};
+	}
+
+	override async getResponse(
+		response: FalAiQueueOutput,
+		url?: string,
+		headers?: Record<string, string>
+	): Promise<Blob> {
+		if (!url || !headers) {
+			throw new InferenceClientInputError("URL and headers are required for image-to-image task");
+		}
+		const requestId = response.request_id;
+		if (!requestId) {
+			throw new InferenceClientProviderOutputError(
+				"Received malformed response from Fal.ai text-to-video API: no request ID found in the response"
+			);
+		}
+		let status = response.status;
+
+		const parsedUrl = new URL(url);
+		const baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}${parsedUrl.host === "router.huggingface.co" ? "/fal-ai" : ""
+			}`;
+
+		// extracting the provider model id for status and result urls
+		// from the response as it might be different from the mapped model in `url`
+		const modelId = new URL(response.response_url).pathname;
+		const queryParams = parsedUrl.search;
+
+		const statusUrl = `${baseUrl}${modelId}/status${queryParams}`;
+		const resultUrl = `${baseUrl}${modelId}${queryParams}`;
+
+		while (status !== "COMPLETED") {
+			await delay(500);
+			const statusResponse = await fetch(statusUrl, { headers });
+
+			if (!statusResponse.ok) {
+				console
+				throw new InferenceClientProviderApiError(
+					"Failed to fetch response status from fal-ai API",
+					{ url: statusUrl, method: "GET" },
+					{
+						requestId: statusResponse.headers.get("x-request-id") ?? "",
+						status: statusResponse.status,
+						body: await statusResponse.text(),
+					}
+				);
+			}
+			try {
+				status = (await statusResponse.json()).status;
+			} catch (error) {
+				throw new InferenceClientProviderOutputError(
+					"Failed to parse status response from fal-ai API: received malformed response"
+				);
+			}
+		}
+
+		const resultResponse = await fetch(resultUrl, { headers });
+		let result: unknown;
+		try {
+			result = await resultResponse.json();
+		} catch (error) {
+			throw new InferenceClientProviderOutputError(
+				"Failed to parse result response from fal-ai API: received malformed response"
+			);
+		}
+		console.log("result", result);
+		if (
+			typeof result === "object" &&
+			!!result &&
+			"images" in result &&
+			Array.isArray(result.images) &&
+			result.images.length > 0 &&
+			typeof result.images[0] === "object" &&
+			!!result.images[0] &&
+			"url" in result.images[0] &&
+			typeof result.images[0].url === "string" &&
+			isUrl(result.images[0].url)
+		) {
+			const urlResponse = await fetch(result.images[0].url);
+			return await urlResponse.blob();
+		} else {
+			throw new InferenceClientProviderOutputError(
+				`Received malformed response from Fal.ai image-to-image API: expected { images: Array<{ url: string }> } result format, got instead: ${JSON.stringify(
+					result
+				)}`
+			);
+		}
+	}
+}
+
 export class FalAITextToVideoTask extends FalAITask implements TextToVideoTaskHelper {
 	constructor() {
 		super("https://queue.fal.run");
@@ -165,9 +281,8 @@ export class FalAITextToVideoTask extends FalAITask implements TextToVideoTaskHe
 		let status = response.status;
 
 		const parsedUrl = new URL(url);
-		const baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}${
-			parsedUrl.host === "router.huggingface.co" ? "/fal-ai" : ""
-		}`;
+		const baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}${parsedUrl.host === "router.huggingface.co" ? "/fal-ai" : ""
+			}`;
 
 		// extracting the provider model id for status and result urls
 		// from the response as it might be different from the mapped model in `url`

packages/inference/src/tasks/cv/imageToImage.ts

@@ -3,6 +3,7 @@ import { resolveProvider } from "../../lib/getInferenceProviderMapping.js";
 import { getProviderHelper } from "../../lib/getProviderHelper.js";
 import type { BaseArgs, Options } from "../../types.js";
 import { innerRequest } from "../../utils/request.js";
+import { makeRequestOptions } from "../../lib/makeRequestOptions.js";
 
 export type ImageToImageArgs = BaseArgs & ImageToImageInput;
 
@@ -18,5 +19,6 @@ export async function imageToImage(args: ImageToImageArgs, options?: Options): P
 		...options,
 		task: "image-to-image",
 	});
-	return providerHelper.getResponse(res);
+	const { url, info } = await makeRequestOptions(args, providerHelper, { ...options, task: "image-to-image" });
+	return providerHelper.getResponse(res, url, info.headers as Record<string, string>);
 }