add qwen3vl_moe support

Author: openvino-dev-samples

optimum/exporters/openvino/model_configs.py

@@ -146,7 +146,6 @@
     XverseModelPatcher,
 )
 
-
 def init_model_configs():
     if "open_clip" not in TasksManager._LIBRARY_TO_SUPPORTED_MODEL_TYPES:
         TasksManager._LIBRARY_TO_SUPPORTED_MODEL_TYPES["open_clip"] = {}
@@ -170,6 +169,10 @@ def init_model_configs():
         "transformers",
         "AutoModelForImageTextToText",
     )
+    TasksManager._CUSTOM_CLASSES[("pt", "qwen3_vl_moe", "image-text-to-text")] = (
+        "transformers",
+        "AutoModelForImageTextToText",
+    )
     TasksManager._CUSTOM_CLASSES[("pt", "llava_next_video", "image-text-to-text")] = (
         "transformers",
         "AutoModelForVision2Seq",
@@ -397,6 +400,14 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
     ],
     library_name="transformers",
 )
+@register_in_tasks_manager(
+    "qwen3_vl_moe_text",
+    *[
+        "text-generation",
+        "text-generation-with-past",
+    ],
+    library_name="transformers",
+)
 class Qwen3VLTextOpenVINOConfig(TextDecoderWithPositionIdsOnnxConfig):
     MIN_TRANSFORMERS_VERSION = "4.56.0"
 
@@ -4004,6 +4015,68 @@ def outputs(self) -> Dict[str, Dict[int, str]]:
         return {}
 
 
+@register_in_tasks_manager(
+    "qwen3_vl_moe",
+    *["image-text-to-text", "video-text-to-text"],
+    library_name="transformers",
+)
+class Qwen3_VL_MOEOpenVINOConfig(Qwen3_VLOpenVINOConfig):
+    def with_behavior(
+        self,
+        behavior: Union[str, Qwen3VLConfigBehavior],
+    ):
+        """
+        Creates a config for different behaviour.
+        Args:
+            behavior ([`ConfigBehavior`]):
+                The behavior to use for the new instance.
+        """
+        if isinstance(behavior, str) and not isinstance(behavior, Qwen3VLConfigBehavior):
+            behavior = Qwen3VLConfigBehavior(behavior)
+
+        if behavior == Qwen3VLConfigBehavior.TEXT_EMBEDDINGS:
+            return get_vlm_text_embeddings_config("qwen3_vl_moe_text", self._orig_config.text_config, self.int_dtype, self.float_dtype)
+
+        if behavior == Qwen3VLConfigBehavior.LANGUAGE:
+            return get_vlm_text_generation_config(
+                "qwen3_vl_moe_text",
+                self._orig_config.text_config,
+                self.int_dtype,
+                self.float_dtype,
+                model_patcher=Qwen3VLLanguageModelPatcher,
+                dummy_input_generator=DummyQwen2VLLMInputGenerator,
+                inputs_update={"position_ids": {1: "batch_size", 2: "sequence_length"}},
+            )
+
+        if behavior == Qwen3VLConfigBehavior.VISION_EMBEDDINGS:
+            return self.__class__(
+                self._orig_config,
+                task=self.task,
+                int_dtype=self.int_dtype,
+                float_dtype=self.float_dtype,
+                behavior=behavior,
+                preprocessors=self._preprocessors,
+            )
+        if behavior == Qwen3VLConfigBehavior.VISION_EMBEDDINGS_MERGER:
+            return self.__class__(
+                self._orig_config,
+                task=self.task,
+                int_dtype=self.int_dtype,
+                float_dtype=self.float_dtype,
+                behavior=behavior,
+                preprocessors=self._preprocessors,
+            )
+        if behavior == Qwen3VLConfigBehavior.VISION_EMBEDDINGS_POS:
+            return self.__class__(
+                self._orig_config,
+                task=self.task,
+                int_dtype=self.int_dtype,
+                float_dtype=self.float_dtype,
+                behavior=behavior,
+                preprocessors=self._preprocessors,
+            )
+
+
 @register_in_tasks_manager(
     "glm",
     *[

optimum/exporters/openvino/utils.py

@@ -229,6 +229,7 @@ def get_submodels(model):
     "qwen2_vl",
     "qwen2_5_vl",
     "qwen3_vl",
+    "qwen3_vl_moe",
     "got_ocr2",
     "gemma3",
     "idefics3",

optimum/intel/openvino/modeling_visual_language.py

@@ -189,7 +189,7 @@ def prepare_inputs(
             if past_len:
                 position_ids = position_ids[:, -inputs_embeds.shape[1] :]
 
-            if (self.config.model_type == "qwen2_vl" or self.config.model_type == "qwen3_vl") and position_ids.ndim != 3:
+            if (self.config.model_type == "qwen2_vl" or self.config.model_type == "qwen3_vl" or self.config.model_type == "qwen3_vl_moe") and position_ids.ndim != 3:
                 position_ids = np.repeat(np.expand_dims(position_ids, 0), 3, axis=0)
 
             inputs["position_ids"] = position_ids
@@ -230,7 +230,6 @@ def forward(
         **kwargs,
     ):
         self.compile()
-
         inputs = self.prepare_inputs(
             input_ids=input_ids,
             attention_mask=attention_mask,
@@ -787,7 +786,7 @@ def forward(
     ):
         if pixel_values is None:
             pixel_values = images if images is not None else image_pixel_values
-        if self.config.model_type == "qwen3_vl":
+        if self.config.model_type == "qwen3_vl" or self.config.model_type == "qwen3_vl_moe":
             inputs_embeds, attention_mask, position_ids, visual_pos_masks, deepstack_visual_embeds = self.get_multimodal_embeddings(
                 input_ids,
                 pixel_values,
@@ -4986,6 +4985,7 @@ def preprocess_inputs(
     "qwen2_vl": _OVQwen2VLForCausalLM,
     "qwen2_5_vl": _OVQwen2_5_VLForCausalLM,
     "qwen3_vl": _OVQwen3VLForCausalLM,
+    "qwen3_vl_moe": _OVQwen3VLForCausalLM,
     "got_ocr2": _OVGotOCR2ForCausalLM,
     "gemma3": _OVGemma3ForCausalLM,
     "idefics3": _OVIdefics3ForCausalLM,