From 8d0ba035b29952edeb64314f55307e0d793ee1ff Mon Sep 17 00:00:00 2001 From: Guillaume Noale Date: Tue, 5 Aug 2025 18:01:13 +0200 Subject: [PATCH 1/5] feat: add scaleway inference provider --- packages/inference/README.md | 2 + .../inference/src/lib/getProviderHelper.ts | 7 ++ packages/inference/src/providers/consts.ts | 1 + packages/inference/src/providers/scaleway.ts | 100 ++++++++++++++++ packages/inference/src/types.ts | 1 + .../inference/test/InferenceClient.spec.ts | 108 ++++++++++++++++++ 6 files changed, 219 insertions(+) create mode 100644 packages/inference/src/providers/scaleway.ts diff --git a/packages/inference/README.md b/packages/inference/README.md index 0ea60b2be7..ed43e0644f 100644 --- a/packages/inference/README.md +++ b/packages/inference/README.md @@ -58,6 +58,7 @@ Currently, we support the following providers: - [OVHcloud](https://endpoints.ai.cloud.ovh.net/) - [Replicate](https://replicate.com) - [Sambanova](https://sambanova.ai) +- [Scaleway](https://www.scaleway.com/en/generative-apis/) - [Together](https://together.xyz) - [Blackforestlabs](https://blackforestlabs.ai) - [Cohere](https://cohere.com) @@ -92,6 +93,7 @@ Only a subset of models are supported when requesting third-party providers. You - [OVHcloud supported models](https://huggingface.co/api/partners/ovhcloud/models) - [Replicate supported models](https://huggingface.co/api/partners/replicate/models) - [Sambanova supported models](https://huggingface.co/api/partners/sambanova/models) +- [Scaleway supported models](https://huggingface.co/api/partners/scaleway/models) - [Together supported models](https://huggingface.co/api/partners/together/models) - [Cohere supported models](https://huggingface.co/api/partners/cohere/models) - [Cerebras supported models](https://huggingface.co/api/partners/cerebras/models) diff --git a/packages/inference/src/lib/getProviderHelper.ts b/packages/inference/src/lib/getProviderHelper.ts index 72cc35bf62..d48f6685d7 100644 --- a/packages/inference/src/lib/getProviderHelper.ts +++ b/packages/inference/src/lib/getProviderHelper.ts @@ -47,6 +47,7 @@ import type { } from "../providers/providerHelper.js"; import * as Replicate from "../providers/replicate.js"; import * as Sambanova from "../providers/sambanova.js"; +import * as Scaleway from "../providers/scaleway.js"; import * as Together from "../providers/together.js"; import type { InferenceProvider, InferenceProviderOrPolicy, InferenceTask } from "../types.js"; import { InferenceClientInputError } from "../errors.js"; @@ -148,6 +149,12 @@ export const PROVIDERS: Record Scaleway model ID here: + * + * https://huggingface.co/api/partners/scaleway/models + * + * This is a publicly available mapping. + * + * If you want to try to run inference for a new model locally before it's registered on huggingface.co, + * you can add it to the dictionary "HARDCODED_MODEL_ID_MAPPING" in consts.ts, for dev purposes. + * + * - If you work at Scaleway and want to update this mapping, please use the model mapping API we provide on huggingface.co + * - If you're a community member and want to add a new supported HF model to Scaleway, please open an issue on the present repo + * and we will tag Scaleway team members. + * + * Thanks! + */ +import type { + FeatureExtractionOutput, + ImageToTextInput, + ImageToTextOutput, + TextGenerationOutput, +} from "@huggingface/tasks"; +import type { BodyParams, RequestArgs } from "../types.js"; +import { InferenceClientProviderOutputError } from "../errors.js"; +import { base64FromBytes } from "../utils/base64FromBytes.js"; + +import { + BaseConversationalTask, + TaskProviderHelper, + FeatureExtractionTaskHelper, + BaseTextGenerationTask, + ImageToTextTaskHelper, +} from "./providerHelper.js"; + +const SCALEWAY_API_BASE_URL = "https://api.scaleway.ai"; + +interface ScalewayEmbeddingsResponse { + data: Array<{ + embedding: number[]; + }>; +} + +export class ScalewayConversationalTask extends BaseConversationalTask { + constructor() { + super("scaleway", SCALEWAY_API_BASE_URL); + } +} + +export class ScalewayTextGenerationTask extends BaseTextGenerationTask { + constructor() { + super("scaleway", SCALEWAY_API_BASE_URL); + } + + override preparePayload(params: BodyParams): Record { + return { + model: params.model, + ...params.args, + prompt: params.args.inputs, + }; + } + + override async getResponse(response: unknown): Promise { + if ( + typeof response === "object" && + response !== null && + "choices" in response && + Array.isArray((response as any).choices) && + (response as any).choices.length > 0 + ) { + const completion = (response as any).choices[0]; + if (completion.text) { + return { + generated_text: completion.text, + }; + } + } + throw new InferenceClientProviderOutputError("Received malformed response from Scaleway text generation API"); + } +} + +export class ScalewayFeatureExtractionTask extends TaskProviderHelper implements FeatureExtractionTaskHelper { + constructor() { + super("scaleway", SCALEWAY_API_BASE_URL); + } + + preparePayload(params: BodyParams): Record { + return { + input: params.args.inputs, + model: params.model, + }; + } + + makeRoute(): string { + return "v1/embeddings"; + } + + async getResponse(response: ScalewayEmbeddingsResponse): Promise { + return response.data.map((item) => item.embedding); + } +} diff --git a/packages/inference/src/types.ts b/packages/inference/src/types.ts index 5d6be233d8..b31843b99b 100644 --- a/packages/inference/src/types.ts +++ b/packages/inference/src/types.ts @@ -61,6 +61,7 @@ export const INFERENCE_PROVIDERS = [ "ovhcloud", "replicate", "sambanova", + "scaleway", "together", ] as const; diff --git a/packages/inference/test/InferenceClient.spec.ts b/packages/inference/test/InferenceClient.spec.ts index d5cefcc60e..f11f9b5bb2 100644 --- a/packages/inference/test/InferenceClient.spec.ts +++ b/packages/inference/test/InferenceClient.spec.ts @@ -1516,6 +1516,114 @@ describe.skip("InferenceClient", () => { TIMEOUT ); + describe.concurrent( + "Scaleway", + () => { + const client = new InferenceClient(env.HF_SCALEWAY_KEY ?? "dummy"); + + HARDCODED_MODEL_INFERENCE_MAPPING.scaleway = { + "meta-llama/Llama-3.1-8B-Instruct": { + provider: "scaleway", + hfModelId: "meta-llama/Llama-3.1-8B-Instruct", + providerId: "llama-3.1-8b-instruct", + status: "live", + task: "conversational", + }, + "BAAI/bge-multilingual-gemma2": { + provider: "scaleway", + hfModelId: "BAAI/bge-multilingual-gemma2", + providerId: "bge-multilingual-gemma2", + task: "feature-extraction", + status: "live", + }, + "google/gemma-3-27b-it": { + provider: "scaleway", + hfModelId: "google/gemma-3-27b-it", + providerId: "gemma-3-27b-it", + task: "conversational", + status: "live", + }, + }; + + it("chatCompletion", async () => { + const res = await client.chatCompletion({ + model: "meta-llama/Llama-3.1-8B-Instruct", + provider: "scaleway", + messages: [{ role: "user", content: "Complete this sentence with words, one plus one is equal " }], + }); + if (res.choices && res.choices.length > 0) { + const completion = res.choices[0].message?.content; + expect(completion).toMatch(/(to )?(two|2)/i); + } + }); + + it("chatCompletion stream", async () => { + const stream = client.chatCompletionStream({ + model: "meta-llama/Llama-3.1-8B-Instruct", + provider: "scaleway", + messages: [{ role: "system", content: "Complete the equation 1 + 1 = , just the answer" }], + }) as AsyncGenerator; + let out = ""; + for await (const chunk of stream) { + if (chunk.choices && chunk.choices.length > 0) { + out += chunk.choices[0].delta.content; + } + } + expect(out).toMatch(/(two|2)/i); + }); + + it("imageToText", async () => { + const res = await client.chatCompletion({ + model: "google/gemma-3-27b-it", + provider: "scaleway", + messages: [ + { + role: "user", + content: [ + { + type: "image_url", + image_url: { + url: "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg", + }, + }, + ], + }, + ], + }); + expect(res.choices).toBeDefined(); + expect(res.choices?.length).toBeGreaterThan(0); + expect(res.choices?.[0].message?.content).toContain("Statue of Liberty"); + }); + + it("textGeneration", async () => { + const res = await client.textGeneration({ + model: "meta-llama/Llama-3.1-8B-Instruct", + provider: "scaleway", + inputs: "Once upon a time,", + temperature: 0, + max_tokens: 19, + }); + + expect(res).toMatchObject({ + generated_text: + " in a small village nestled in the rolling hills of the countryside, there lived a young girl named", + }); + }); + + it("featureExtraction", async () => { + const res = await client.featureExtraction({ + model: "BAAI/bge-multilingual-gemma2", + provider: "scaleway", + inputs: "That is a happy person", + }); + + expect(res).toBeInstanceOf(Array); + expect(res[0]).toEqual(expect.arrayContaining([expect.any(Number)])); + }); + }, + TIMEOUT + ); + describe.concurrent("3rd party providers", () => { it("chatCompletion - fails with unsupported model", async () => { expect( From 41a1e42da6ba0acf842977262aca39f11f3e746f Mon Sep 17 00:00:00 2001 From: SBrandeis Date: Fri, 8 Aug 2025 14:51:06 +0200 Subject: [PATCH 2/5] code style / type system --- packages/inference/src/providers/scaleway.ts | 22 +++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/inference/src/providers/scaleway.ts b/packages/inference/src/providers/scaleway.ts index 530f1574f1..012f546748 100644 --- a/packages/inference/src/providers/scaleway.ts +++ b/packages/inference/src/providers/scaleway.ts @@ -16,20 +16,16 @@ */ import type { FeatureExtractionOutput, - ImageToTextInput, - ImageToTextOutput, TextGenerationOutput, } from "@huggingface/tasks"; -import type { BodyParams, RequestArgs } from "../types.js"; +import type { BodyParams } from "../types.js"; import { InferenceClientProviderOutputError } from "../errors.js"; -import { base64FromBytes } from "../utils/base64FromBytes.js"; +import type { FeatureExtractionTaskHelper } from "./providerHelper.js"; import { BaseConversationalTask, TaskProviderHelper, - FeatureExtractionTaskHelper, BaseTextGenerationTask, - ImageToTextTaskHelper, } from "./providerHelper.js"; const SCALEWAY_API_BASE_URL = "https://api.scaleway.ai"; @@ -64,11 +60,17 @@ export class ScalewayTextGenerationTask extends BaseTextGenerationTask { typeof response === "object" && response !== null && "choices" in response && - Array.isArray((response as any).choices) && - (response as any).choices.length > 0 + Array.isArray(response.choices) && + response.choices.length > 0 ) { - const completion = (response as any).choices[0]; - if (completion.text) { + const completion: unknown = response.choices[0]; + if ( + typeof completion === "object" && + !!completion && + "text" in completion && + completion.text && + typeof completion.text === "string" + ) { return { generated_text: completion.text, }; From 1f7b1921b108abab04fbcac3e7a15c7ed6196460 Mon Sep 17 00:00:00 2001 From: SBrandeis Date: Fri, 8 Aug 2025 14:51:20 +0200 Subject: [PATCH 3/5] tests tweaks --- packages/inference/test/InferenceClient.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/inference/test/InferenceClient.spec.ts b/packages/inference/test/InferenceClient.spec.ts index f11f9b5bb2..ca5427de4c 100644 --- a/packages/inference/test/InferenceClient.spec.ts +++ b/packages/inference/test/InferenceClient.spec.ts @@ -22,7 +22,7 @@ if (!env.HF_TOKEN) { console.warn("Set HF_TOKEN in the env to run the tests for better rate limits"); } -describe.skip("InferenceClient", () => { +describe("InferenceClient", () => { // Individual tests can be ran without providing an api key, however running all tests without an api key will result in rate limiting error. describe("backward compatibility", () => { @@ -1550,6 +1550,7 @@ describe.skip("InferenceClient", () => { model: "meta-llama/Llama-3.1-8B-Instruct", provider: "scaleway", messages: [{ role: "user", content: "Complete this sentence with words, one plus one is equal " }], + tool_choice: "none", }); if (res.choices && res.choices.length > 0) { const completion = res.choices[0].message?.content; @@ -1586,6 +1587,7 @@ describe.skip("InferenceClient", () => { url: "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg", }, }, + { type: "text", text: "What is this?" }, ], }, ], From d09795de166abf5365bf6f90ce82f68deccc0653 Mon Sep 17 00:00:00 2001 From: SBrandeis Date: Fri, 8 Aug 2025 14:54:50 +0200 Subject: [PATCH 4/5] last tweaks --- packages/inference/src/lib/getProviderHelper.ts | 1 - packages/inference/test/InferenceClient.spec.ts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/inference/src/lib/getProviderHelper.ts b/packages/inference/src/lib/getProviderHelper.ts index d48f6685d7..5836fb9f73 100644 --- a/packages/inference/src/lib/getProviderHelper.ts +++ b/packages/inference/src/lib/getProviderHelper.ts @@ -151,7 +151,6 @@ export const PROVIDERS: Record { +describe.skip("InferenceClient", () => { // Individual tests can be ran without providing an api key, however running all tests without an api key will result in rate limiting error. describe("backward compatibility", () => { @@ -1573,7 +1573,7 @@ describe("InferenceClient", () => { expect(out).toMatch(/(two|2)/i); }); - it("imageToText", async () => { + it("chatCompletion multimodal", async () => { const res = await client.chatCompletion({ model: "google/gemma-3-27b-it", provider: "scaleway", From fe2d1840019180f6e8da7b4dd93f0be7238b3aae Mon Sep 17 00:00:00 2001 From: SBrandeis Date: Fri, 8 Aug 2025 14:55:26 +0200 Subject: [PATCH 5/5] lint+format --- packages/inference/src/providers/scaleway.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/inference/src/providers/scaleway.ts b/packages/inference/src/providers/scaleway.ts index 012f546748..007a496939 100644 --- a/packages/inference/src/providers/scaleway.ts +++ b/packages/inference/src/providers/scaleway.ts @@ -14,19 +14,12 @@ * * Thanks! */ -import type { - FeatureExtractionOutput, - TextGenerationOutput, -} from "@huggingface/tasks"; +import type { FeatureExtractionOutput, TextGenerationOutput } from "@huggingface/tasks"; import type { BodyParams } from "../types.js"; import { InferenceClientProviderOutputError } from "../errors.js"; import type { FeatureExtractionTaskHelper } from "./providerHelper.js"; -import { - BaseConversationalTask, - TaskProviderHelper, - BaseTextGenerationTask, -} from "./providerHelper.js"; +import { BaseConversationalTask, TaskProviderHelper, BaseTextGenerationTask } from "./providerHelper.js"; const SCALEWAY_API_BASE_URL = "https://api.scaleway.ai";