Skip to content

Commit 297a8c0

Browse files
committed
Add Mistral3 multimodal support with Pixtral vision encoder
This adds support for Mistral3 multimodal models (vision + text): - `Bumblebee.Vision.Pixtral`: Pixtral vision encoder with RoPE support - `Bumblebee.Text.Mistral3`: Mistral3 text decoder with interleaved attention - `Bumblebee.Multimodal.Mistral3`: Vision-language model combining Pixtral and Mistral3 with multimodal projector for image-conditioned generation Supported architectures: - PixtralVisionModel - Mistral3Model, Mistral3ForCausalLM, Mistral3ForSequenceClassification - Mistral3ForConditionalGeneration (multimodal)
1 parent bae534a commit 297a8c0

File tree

7 files changed

+1622
-0
lines changed

7 files changed

+1622
-0
lines changed

lib/bumblebee.ex

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,12 @@ defmodule Bumblebee do
170170
"MistralModel" => {Bumblebee.Text.Mistral, :base},
171171
"MistralForCausalLM" => {Bumblebee.Text.Mistral, :for_causal_language_modeling},
172172
"MistralForSequenceClassification" => {Bumblebee.Text.Mistral, :for_sequence_classification},
173+
"Mistral3Model" => {Bumblebee.Text.Mistral3, :base},
174+
"Mistral3ForCausalLM" => {Bumblebee.Text.Mistral3, :for_causal_language_modeling},
175+
"Mistral3ForSequenceClassification" =>
176+
{Bumblebee.Text.Mistral3, :for_sequence_classification},
177+
"Mistral3ForConditionalGeneration" =>
178+
{Bumblebee.Multimodal.Mistral3, :for_conditional_generation},
173179
"PhiModel" => {Bumblebee.Text.Phi, :base},
174180
"PhiForCausalLM" => {Bumblebee.Text.Phi, :for_causal_language_modeling},
175181
"PhiForSequenceClassification" => {Bumblebee.Text.Phi, :for_sequence_classification},
@@ -198,6 +204,7 @@ defmodule Bumblebee do
198204
"T5Model" => {Bumblebee.Text.T5, :base},
199205
"T5ForConditionalGeneration" => {Bumblebee.Text.T5, :for_conditional_generation},
200206
"T5EncoderModel" => {Bumblebee.Text.T5, :encoder},
207+
"PixtralVisionModel" => {Bumblebee.Vision.Pixtral, :base},
201208
"ViTForImageClassification" => {Bumblebee.Vision.Vit, :for_image_classification},
202209
"ViTForMaskedImageModeling" => {Bumblebee.Vision.Vit, :for_masked_image_modeling},
203210
"ViTModel" => {Bumblebee.Vision.Vit, :base},
@@ -255,6 +262,7 @@ defmodule Bumblebee do
255262
"layoutlm" => :layout_lm,
256263
"llama" => :llama,
257264
"mistral" => :llama,
265+
"mistral3" => :llama,
258266
"mbart" => :mbart,
259267
"phi" => :code_gen,
260268
"phi3" => :llama,

0 commit comments

Comments
 (0)