text generation working

Author: connorch

packages/inference/src/lib/makeRequestOptions.ts

@@ -145,7 +145,9 @@ export async function makeRequestOptions(
 			? args.data
 			: JSON.stringify({
 					...otherArgs,
-					...(chatCompletion || provider === "together" || provider === "nebius" ? { model } : undefined),
+					...(chatCompletion || provider === "together" || provider === "nebius" || provider === "hyperbolic"
+						? { model }
+						: undefined),
 			  }),
 		...(credentials ? { credentials } : undefined),
 		signal: options?.signal,

packages/inference/src/tasks/nlp/textGeneration.ts

@@ -21,6 +21,12 @@ interface TogeteherTextCompletionOutput extends Omit<ChatCompletionOutput, "choi
 	}>;
 }
 
+interface HyperbolicTextCompletionOutput extends Omit<ChatCompletionOutput, "choices"> {
+	choices: Array<{
+		message: { content: string };
+	}>;
+}
+
 /**
  * Use to continue text from a prompt. This is a very generic task. Recommended model: gpt2 (it’s a simple model, but fun to play with).
  */
@@ -43,6 +49,21 @@ export async function textGeneration(
 		return {
 			generated_text: completion.text,
 		};
+	} else if (args.provider === "hyperbolic") {
+		args.prompt = args.inputs;
+		const raw = await request<HyperbolicTextCompletionOutput>(args, {
+			...options,
+			taskHint: "text-generation",
+		});
+		const isValidOutput =
+			typeof raw === "object" && "choices" in raw && Array.isArray(raw?.choices) && typeof raw?.model === "string";
+		if (!isValidOutput) {
+			throw new InferenceOutputError("Expected ChatCompletionOutput");
+		}
+		const completion = raw.choices[0];
+		return {
+			generated_text: completion.message.content,
+		};
 	} else {
 		const res = toArray(
 			await request<TextGenerationOutput | TextGenerationOutput[]>(args, {

packages/inference/test/HfInference.spec.ts

@@ -1186,7 +1186,7 @@ describe.concurrent("HfInference", () => {
 				"meta-llama/Llama-3.2-3B-Instruct": "meta-llama/Llama-3.2-3B-Instruct",
 				"meta-llama/Llama-3.3-70B-Instruct": "meta-llama/Llama-3.3-70B-Instruct",
 				"stabilityai/stable-diffusion-2": "stabilityai/stable-diffusion-2",
-				"meta-llama/Llama-3.1-405B": "meta-llama/Llama-3.1-405B",
+				"meta-llama/Llama-3.1-405B": "meta-llama/Meta-Llama-3.1-405B-Instruct",
 			};
 
 			it("chatCompletion - hyperbolic", async () => {
@@ -1244,9 +1244,10 @@ describe.concurrent("HfInference", () => {
 					provider: "hyperbolic",
 					messages: [{ role: "user", content: "Paris is" }],
 					temperature: 0,
+					top_p: 0.01,
 					max_tokens: 10,
 				});
-				expect(res).toMatchObject({ generated_text: " city of love" });
+				expect(res).toMatchObject({ generated_text: "...the capital and most populous city of France," });
 			});
 		},
 		TIMEOUT