textToImage should work too

Author: julien-c

packages/inference/src/lib/makeRequestOptions.ts

@@ -118,6 +118,9 @@ export async function makeRequestOptions(
 					case "sambanova":
 						return SAMBANOVA_API_BASE_URL;
 					case "together":
+						if (taskHint === "text-to-image") {
+							return `${TOGETHER_API_BASE_URL}/v1/images/generations`;
+						}
 						return TOGETHER_API_BASE_URL;
 					default:
 						break;

packages/inference/src/tasks/cv/textToImage.ts

@@ -8,6 +8,12 @@ export type TextToImageArgs = BaseArgs & {
 	 */
 	inputs: string;
 
+	/**
+	 * Same param but for external providers like Together
+	 */
+	prompt?: string;
+	response_format?: "base64";
+
 	parameters?: {
 		/**
 		 * An optional negative prompt for the image generation
@@ -32,17 +38,35 @@ export type TextToImageArgs = BaseArgs & {
 	};
 };
 
+interface Base64ImageGeneration {
+	id: string;
+	model: string;
+	data: Array<{
+		b64_json: string;
+	}>;
+}
 export type TextToImageOutput = Blob;
 
 /**
  * This task reads some text input and outputs an image.
  * Recommended model: stabilityai/stable-diffusion-2
  */
 export async function textToImage(args: TextToImageArgs, options?: Options): Promise<TextToImageOutput> {
-	const res = await request<TextToImageOutput>(args, {
+	if (args.provider === "together") {
+		args.prompt = args.inputs;
+		args.inputs = "";
+		args.response_format = "base64";
+	}
+	const res = await request<TextToImageOutput | Base64ImageGeneration>(args, {
 		...options,
 		taskHint: "text-to-image",
 	});
+	if (res && typeof res === "object" && Array.isArray(res.data) && res.data[0].b64_json) {
+		const base64Data = res.data[0].b64_json;
+		const base64Response = await fetch(`data:image/jpeg;base64,${base64Data}`);
+		const blob = await base64Response.blob();
+		return blob;
+	}
 	const isValidOutput = res && res instanceof Blob;
 	if (!isValidOutput) {
 		throw new InferenceOutputError("Expected Blob");

packages/inference/test/HfInference.spec.ts

@@ -816,6 +816,16 @@ describe.concurrent(
 			}
 			expect(out).toContain("2");
 		});
+
+		it("textToImage together", async () => {
+			const hf = new HfInference(env.TOGETHER_KEY);
+			const res = await hf.textToImage({
+				model: "stabilityai/stable-diffusion-xl-base-1.0",
+				provider: "together",
+				inputs: "award winning high resolution photo of a giant tortoise",
+			});
+			expect(res).toBeInstanceOf(Blob);
+		});
 	},
 	TIMEOUT
 );