diff --git a/menu/navigation.json b/menu/navigation.json index 8c65f19497..ef374612b9 100644 --- a/menu/navigation.json +++ b/menu/navigation.json @@ -872,6 +872,10 @@ "label": "Support for function calling in Scaleway Managed Inference", "slug": "function-calling-support" }, + { + "label": "Managed Inference model catalog", + "slug": "model-catalog" + }, { "label": "BGE-Multilingual-Gemma2 model", "slug": "bge-multilingual-gemma2" @@ -3300,7 +3304,7 @@ "slug": "faq" }, { - "items": [ + "items": [ { "label": "Order an InterLink", "slug": "order-interlink" @@ -3310,61 +3314,61 @@ "slug": "complete-provisioning" }, { - "label": "Configure an InterLink", - "slug": "configure-interlink" - }, - { - "label": "Create a routing policy", - "slug": "create-routing-policy" - }, - { - "label": "Delete an InterLink", - "slug": "delete-interlink" - } - ], - "label": "How to", - "slug": "how-to" - }, - { - "items": [ - { - "label": "InterLink API Reference", - "slug": "https://www.scaleway.com/en/developers/api/interlink/" - } - ], - "label": "API/CLI", - "slug": "api-cli" - }, - { - "items": [ - { - "label": "InterLink overview", - "slug": "overview" - }, - { - "label": "InterLink provisioning", - "slug": "provisioning" - }, - { - "label": "Configuring an InterLink", - "slug": "configuring" + "label": "Configure an InterLink", + "slug": "configure-interlink" + }, + { + "label": "Create a routing policy", + "slug": "create-routing-policy" + }, + { + "label": "Delete an InterLink", + "slug": "delete-interlink" + } + ], + "label": "How to", + "slug": "how-to" }, { - "label": "InterLink statuses", - "slug": "statuses" + "items": [ + { + "label": "InterLink API Reference", + "slug": "https://www.scaleway.com/en/developers/api/interlink/" + } + ], + "label": "API/CLI", + "slug": "api-cli" }, { - "label": "Using BGP communities", - "slug": "bgp-communities" + "items": [ + { + "label": "InterLink overview", + "slug": "overview" + }, + { + "label": "InterLink provisioning", + "slug": "provisioning" + }, + { + "label": "Configuring an InterLink", + "slug": "configuring" + }, + { + "label": "InterLink statuses", + "slug": "statuses" + }, + { + "label": "Using BGP communities", + "slug": "bgp-communities" + } + ], + "label": "Additional Content", + "slug": "reference-content" } ], - "label": "Additional Content", - "slug": "reference-content" - } - ], - "label": "InterLink", - "slug": "interlink" - }, + "label": "InterLink", + "slug": "interlink" + }, { "items": [ { diff --git a/pages/managed-inference/reference-content/model-catalog.mdx b/pages/managed-inference/reference-content/model-catalog.mdx new file mode 100644 index 0000000000..3ea11e5d0e --- /dev/null +++ b/pages/managed-inference/reference-content/model-catalog.mdx @@ -0,0 +1,307 @@ +--- +meta: + title: Managed Inference model catalog + description: Deploy your own model with Scaleway Managed Inference. Privacy-focused, fully managed. +content: + h1: Managed Inference model catalog + paragraph: This page provides information on the Scaleway Managed Inference product catalog +tags: +dates: + validation: 2025-04-18 + posted: 2024-04-18 +categories: + - ai-data +--- +A quick overview of available models in Scaleway's catalog and their core attributes. Expand any model below to see usage examples, curl commands, and detailed capabilities. + +## Models technical summary + +| Model name | Provider | Maximum Context length (tokens) | Modalities | Instances | License | +|------------|----------|--------------|------------|-----------|---------| +| [`gemma-3-27b-it`](#gemma-3-27b-it) | Google | 40k | Text, Vision | H100, H100-2 | [Gemma](https://ai.google.dev/gemma/terms) | +| [`llama-3.3-70b-instruct`](#llama-33-70b-instruct) | Meta | 128k | Text | H100, H100-2 | [Llama 3.3 community](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | +| [`llama-3.1-70b-instruct`](#llama-31-70b-instruct) | Meta | 128k | Text | H100, H100-2 | [Llama 3.1 community](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct/blob/main/LICENSE) | +| [`llama-3.1-8b-instruct`](#llama-31-8b-instruct) | Meta | 128k | Text | L4, L40S, H100, H100-2 | [Llama 3.1 community](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/blob/main/LICENSE) | +| [`llama-3-70b-instruct`](#llama-3-70b-instruct) | Meta | 8k | Text | H100, H100-2 | [Llama 3 community](https://huggingface.co/meta-llama/Meta-Llama-3-8B/blob/main/LICENSE) | +| [`llama-3.1-nemotron-70b-instruct`](#llama-31-nemotron-70b-instruct) | Nvidia | 128k | Text | H100, H100-2 | [Llama 3.1 community](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct/blob/main/LICENSE) | +| [`deepseek-r1-distill-70b`](#deepseek-r1-distill-llama-70b) | Deepseek | 128k | Text | H100, H100-2 | [MIT](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B/blob/main/LICENSE) and [Llama 3.3 Community](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct/blob/main/LICENSE) | +| [`deepseek-r1-distill-8b`](#deepseek-r1-distill-llama-8b) | Deepseek | 128k | Text | L4, L40S, H100, H100-2 | [MIT](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B/blob/main/LICENSE) and [Llama 3.1 Community](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/blob/main/LICENSE) | +| [`mistral-7b-instruct-v0.3`](#mistral-7b-instruct-v03) | Mistral | 32k | Text | L4, L40S, H100, H100-2 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| [`mistral-small-3.1-24b-instruct-2503`](#mistral-small-31-24b-instruct-2503) | Mistral | 128k | Text, Vision | H100, H100-2 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| [`mistral-small-24b-instruct-2501`](#mistral-small-24b-instruct-2501) | Mistral | 32k | Text | L40S, H100, H100-2 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| [`mistral-nemo-instruct-2407`](#mistral-nemo-instruct-2407) | Mistral | 128k | Text | L40S, H100, H100-2 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| [`mixtral-8x7b-instruct-v0.1`](#mixtral-8x7b-instruct-v01) | Mistral | 32k | Text | H100-2 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| [`moshiko-0.1-8b`](#moshiko-01-8b) | Kyutai | 4k | Audio to Audio | L4, H100 | [CC-BY-4.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/cc-by-4.0.md) | +| [`moshika-0.1-8b`](#moshika-01-8b) | Kyutai | 4k | Audio to Audio| L4, H100 | [CC-BY-4.0](https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/cc-by-4.0.md) | +| [`wizardlm-70b-v1.0`](#wizardlm-70b-v10) | WizardLM | 4k | Text | H100, H100-2 | [Llama 2 community](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/blob/main/LICENSE.txt) | +| [`pixtral-12b-2409`](#pixtral-12b-2409) | Mistral | 128k | Text, Vision | L40S, H100, H100-2 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| [`molmo-72b-0924`](#molmo-72b-0924) | Allen AI | 50k | Text, Vision | H100-2 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) and [Twonyi Qianwen license](https://huggingface.co/Qwen/Qwen2-72B/blob/main/LICENSE)| +| [`qwen2.5-coder-32b-instruct`](#qwen25-coder-32b-instruct) | Qwen | 32k | Code | H100, H100-2 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) | +| [`bge-multilingual-gemma2`](#bge-multilingual-gemma2) | BAAI | 4k | Embeddings | L4, L40S, H100, H100-2 | [Gemma](https://ai.google.dev/gemma/terms) | +| [`sentence-t5-xxl`](#sentence-t5-xxl) | Sentence transformers | 512 | Embeddings | L4 | [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) | + +## Models feature summary +| Model name | Structured output supported | Function calling | Supported languages | +| --- | --- | --- | --- | +| `gemma-3-27b-it` | Yes | Partial | English, Chinese, Japanese, Korean and 31 additional languages | +| `llama-3.3-70b-instruct` | Yes | Yes | English, German, French, Italian, Portuguese, Hindi, Spanish, and Thai | +| `llama-3.1-70b-instruct` | Yes | Yes | English, German, French, Italian, Portuguese, Hindi, Spanish, and Thai | +| `llama-3.1-8b-instruct` | Yes | Yes | English, German, French, Italian, Portuguese, Hindi, Spanish, and Thai | +| `llama-3-70b-instruct` | Yes | Yes | English | +| `llama-3.1-nemotron-70b-instruct` | Yes | Yes | English | +| `deepseek-r1-distill-llama-70B` | Yes | Yes | English, Chinese | +| `deepseek-r1-distill-llama-8B` | Yes | Yes | English, Chinese | +| `mistral-7b-instruct-v0.3` | Yes | Yes | English | +| `mistral-small-3.1-24b-instruct-2503` | Yes | Yes | English, French, German, Greek, Hindi, Indonesian, Italian, Japanese, Korean, Malay, Nepali, Polish, Portuguese, Romanian, Russian, Serbian, Spanish, Swedish, Turkish, Ukrainian, Vietnamese, Arabic, Bengali, Chinese, Farsi | +| `mistral-small-24b-instruct-2501` | Yes | Yes | Text | English, French, German, Dutch, Spanish, Italian, Polish, Portuguese, Chinese, Japanese, Korean | +| `mistral-nemo-instruct-2407` | Yes | Yes | English, French, German, Spanish, Italian, Portuguese, Russian, Chinese, Japanese | +| `mixtral-8x7b-instruct-v0.1` | Yes | Yes | English, French, German, Italian, Spanish | +| `moshiko-0.1-8b` | No | No | English | +| `moshika-0.1-8b` | No | No | English | +| `wizardLM-70b-v1.0` | Yes | No | English | +| `pixtral-12b-2409` | Yes | Yes | English | +| `molmo-72b-0924` | Yes | No | English | +| `qwen2.5-coder-32b-instruct` | Yes | Yes | English, French, Spanish, Portuguese, German, Italian, Russian, Chinese, Japanese, Korean, Vietnamese, Thai, Arabic and 16 additional languages. | +| `bge-multilingual-gemma2` | No | No | English, French, Chinese, Japanese, Korean | +| `sentence-t5-xxl` | No | No | English | + + +## Model details + + Despite efforts for accuracy, the possibility of generated text containing inaccuracies or [hallucinations](/managed-inference/concepts/#hallucinations) exists. Always verify the content generated independently. + + +## Multimodal models (Text and Vision) + + + Vision models can understand and analyze images, not generate them. You will use it through the /v1/chat/completions endpoint. + + +### Gemma-3-27b-it +Gemma-3-27b-it is a model developed by Google to perform text processing and image analysis on many languages. +The model was not trained specifically to output function / tool call tokens. Hence function calling is currently supported, but reliability remains limited. + +#### Model names +``` +google/gemma-3-27b-it:bf16 +``` + +### Mistral-small-3.1-24b-instruct-2503 +Mistral-small-3.1-24b-instruct-2503 is a model developed by Mistral to perform text processing and image analysis on many languages. +This model was optimized to have a dense knowledge and faster tokens throughput compared to its size. + +#### Model names +``` +mistral/mistral-small-3.1-24b-instruct-2503:bf16 +``` + +### Pixtral-12b-2409 +Pixtral is a vision language model introducing a novel architecture: 12B parameter multimodal decoder plus 400M parameter vision encoder. +It can analyze images and offer insights from visual content alongside text. +This multimodal functionality creates new opportunities for applications that need both visual and textual comprehension. +Pixtral is open-weight and distributed under the Apache 2.0 license. + +#### Model name +``` +mistral/pixtral-12b-2409:bf16 +``` + +### Molmo-72b-0924 +Molmo 72B is the powerhouse of the Molmo family, multimodal models developed by the renowned research lab Allen Institute for AI. +Vision-language models like Molmo can analyze an image and offer insights from visual content alongside text. This multimodal functionality creates new opportunities for applications that need both visual and textual comprehension. + +#### Model name +``` +allenai/molmo-72b-0924:fp8 +``` + +## Text models + +### Llama-3.3-70b-instruct +Released December 6, 2024, Meta’s Llama 3.3 70b is a fine-tune of the [Llama 3.1 70b](/managed-inference/reference-content/llama-3.1-70b-instruct/) model. +This model is still text-only (text in/text out). However, Llama 3.3 was designed to approach the performance of Llama 3.1 405B on some applications. + +#### Model name +``` +meta/llama-3.3-70b-instruct:fp8 +meta/llama-3.3-70b-instruct:bf16 +``` + +### Llama-3.1-70b-instruct +Released July 23, 2024, Meta’s Llama 3.1 is an iteration of the open-access Llama family. +Llama 3.1 was designed to match the best proprietary models and outperform many of the available open source on common industry benchmarks. + +#### Model names +``` +meta/llama-3.1-70b-instruct:fp8 +meta/llama-3.1-70b-instruct:bf16 +``` + +### Llama-3.1-8b-instruct +Released July 23, 2024, Meta’s Llama 3.1 is an iteration of the open-access Llama family. +Llama 3.1 was designed to match the best proprietary models and outperform many of the available open source on common industry benchmarks. + +#### Model names +``` +meta/llama-3.1-8b-instruct:fp8 +meta/llama-3.1-8b-instruct:bf16 +``` + +### Llama-3-70b-instruct +Meta’s Llama 3 is an iteration of the open-access Llama family. +Llama 3 was designed to match the best proprietary models, enhanced by community feedback for greater utility and responsibly spearheading the deployment of LLMs. +With a commitment to open-source principles, this release marks the beginning of a multilingual, multimodal future for Llama 3, pushing the boundaries in reasoning and coding capabilities. + +#### Model name +``` +meta/llama-3-70b-instruct:fp8 +``` + +### Llama-3.1-Nemotron-70b-instruct +Introduced October 14, 2024, NVIDIA's Nemotron 70B Instruct is a specialized version of the Llama 3.1 model designed to follow complex instructions. +NVIDIA employed Reinforcement Learning from Human Feedback (RLHF) to fine-tune the model’s ability to generate relevant and informative responses. + +#### Model name +``` +nvidia/llama-3.1-nemotron-70b-instruct:fp8 +``` + +### DeepSeek-R1-Distill-Llama-70B +Released January 21, 2025, Deepseek’s R1 Distilled Llama 70B is a distilled version of the Llama model family based on Deepseek R1. +DeepSeek R1 Distill Llama 70B is designed to improve the performance of Llama models on reasoning use cases such as mathematics and coding tasks. + +#### Model name +``` +deepseek/deepseek-r1-distill-llama-70b:fp8 +deepseek/deepseek-r1-distill-llama-70b:bf16 +``` + +### DeepSeek-R1-Distill-Llama-8B +Released January 21, 2025, Deepseek’s R1 Distilled Llama 8B is a distilled version of the Llama model family based on Deepseek R1. +DeepSeek R1 Distill Llama 8B is designed to improve the performance of Llama models on reasoning use cases such as mathematics and coding tasks. + +#### Model names +``` +deepseek/deepseek-r1-distill-llama-8b:fp8 +deepseek/deepseek-r1-distill-llama-8b:bf16 +``` + +### Mixtral-8x7b-instruct-v0.1 +Mixtral-8x7b-instruct-v0.1, developed by Mistral, is tailored for instructional platforms and virtual assistants. +Trained on vast instructional datasets, it provides clear and concise instructions across various domains, enhancing user learning experiences. + +#### Model names +``` +mistral/mixtral-8x7b-instruct-v0.1:fp8 +mistral/mixtral-8x7b-instruct-v0.1:bf16 +``` + +### Mistral-7b-instruct-v0.3 +The first dense model released by Mistral AI, perfect for experimentation, customization, and quick iteration. At the time of the release, it matched the capabilities of models up to 30B parameters. +This model is open-weight and distributed under the Apache 2.0 license. + +#### Model name +``` +mistral/mistral-7b-instruct-v0.3:bf16 +``` + +### Mistral-small-24b-instruct-2501 +Mistral Small 24B Instruct is a state-of-the-art transformer model of 24B parameters, built by Mistral. +This model is open-weight and distributed under the Apache 2.0 license. + +#### Model name +``` +mistral/mistral-small-24b-instruct-2501:fp8 +mistral/mistral-small-24b-instruct-2501:bf16 +``` + +### Mistral-nemo-instruct-2407 +Mistral Nemo is a state-of-the-art transformer model of 12B parameters, built by Mistral in collaboration with NVIDIA. +This model is open-weight and distributed under the Apache 2.0 license. +It was trained on a large proportion of multilingual and code data. + +#### Model name +``` +mistral/mistral-nemo-instruct-2407:fp8 +``` + +### Moshiko-0.1-8b +Kyutai's Moshi is a speech-text foundation model for real-time dialogue. +Moshi is an experimental next-generation conversational model, designed to understand and respond fluidly and naturally to complex conversations, while providing unprecedented expressiveness and spontaneity. +While current systems for spoken dialogue rely on a pipeline of separate components, Moshi is the first real-time full-duplex spoken large language model. +Moshiko is the variant of Moshi with a male voice in English. + +#### Model names +``` +kyutai/moshiko-0.1-8b:bf16 +kyutai/moshiko-0.1-8b:fp8 +``` + +### Moshika-0.1-8b +Kyutai's Moshi is a speech-text foundation model for real-time dialogue. +Moshi is an experimental next-generation conversational model, designed to understand and respond fluidly and naturally to complex conversations, while providing unprecedented expressiveness and spontaneity. +While current systems for spoken dialogue rely on a pipeline of separate components, Moshi is the first real-time full-duplex spoken large language model. +Moshika is the variant of Moshi with a female voice in English. + +#### Model names +``` +kyutai/moshika-0.1-8b:bf16 +kyutai/moshika-0.1-8b:fp8 +``` + +### WizardLM-70B-V1.0 +WizardLM-70B-V1.0, developed by WizardLM, is specifically designed for content creation platforms and writing assistants. +With its extensive training in diverse textual data, WizardLM-70B-V1.0 generates high-quality content and assists writers in various creative and professional endeavors. + +#### Model names +``` +wizardlm/wizardlm-70b-v1.0:fp8 +wizardlm/wizardlm-70b-v1.0:fp16 +``` + +## Code models + +### Qwen2.5-coder-32b-instruct +Qwen2.5-coder is your intelligent programming assistant familiar with more than 40 programming languages. +With Qwen2.5-coder deployed at Scaleway, your company can benefit from code generation, AI-assisted code repair, and code reasoning. + +#### Model name +``` +qwen/qwen2.5-coder-32b-instruct:int8 +``` + +## Embeddings models + +### Bge-multilingual-gemma2 +BGE-Multilingual-Gemma2 tops the [MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard), scoring the number one spot in French and Polish, and number seven in English (as of Q4 2024). +As its name suggests, the model’s training data spans a broad range of languages, including English, Chinese, Polish, French, and more. + +| Attribute | Value | +|-----------|-------| +| Embedding dimensions | 3584 | +| Matryoshka embedding | No | + + + [Matryoshka embeddings](https://huggingface.co/blog/matryoshka) refers to embeddings trained on multiple dimensions number. As a result, resulting vectors dimensions will be sorted by most meaningful first. For example, a 3584 dimensions vector can be truncated to its 768 first dimensions and used directly. + + +#### Model name +``` +baai/bge-multilingual-gemma2:fp32 +``` + +### Sentence-t5-xxl +The Sentence-T5-XXL model represents a significant evolution in sentence embeddings, building on the robust foundation of the Text-To-Text Transfer Transformer (T5) architecture. +Designed for performance in various language processing tasks, Sentence-T5-XXL leverages the strengths of T5's encoder-decoder structure to generate high-dimensional vectors that encapsulate rich semantic information. +This model has been meticulously tuned for tasks such as text classification, semantic similarity, and clustering, making it a useful tool in the RAG (Retrieval-Augmented Generation) framework. It excels in sentence similarity tasks, but its performance in semantic search tasks is less optimal. + + +| Attribute | Value | +|-----------|-------| +| Embedding dimensions | 768 | +| Matryoshka embedding | No | + +#### Model name +``` +sentence-transformers/sentence-t5-xxl:fp32 +```