Add test to ensure all supported models are tested

docs/source/openvino/models.mdx

@@ -25,6 +25,7 @@ Here is the list of the supported architectures :
 - Beit
 - Bert
 - BioGPT
+- BigBirdPegasus
 - BlenderBot
 - BlenderBotSmall
 - Bloom
@@ -64,6 +65,7 @@ Here is the list of the supported architectures :
 - GPT-Neo
 - GPT-NeoX
 - GPT-NeoX-Japanese
+- GPT-OSS
 - Gemma
 - Gemma2
 - Gemma3
@@ -103,6 +105,7 @@ Here is the list of the supported architectures :
 - MobileVit
 - Nystromformer
 - OLMo
+- OLMo 2
 - OPT
 - Orion
 - Pegasus

optimum/exporters/openvino/model_configs.py

@@ -2990,9 +2990,6 @@ class Phi4MMConfigBehavior(str, enum.Enum):
 @register_in_tasks_manager(
     "phi4mm", *["image-text-to-text", "automatic-speech-recognition"], library_name="transformers"
 )
-@register_in_tasks_manager(
-    "phi4_multimodal", *["image-text-to-text", "automatic-speech-recognition"], library_name="transformers"
-)
 class Phi4MMOpenVINOConfig(BaseVLMOpenVINOConfig):
     SUPPORTED_BEHAVIORS = [model_type.value for model_type in Phi4MMConfigBehavior]
     NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig
@@ -3216,6 +3213,14 @@ def rename_ambiguous_inputs(self, inputs):
         return inputs
 
 
+@register_in_tasks_manager(
+    "phi4_multimodal", *["image-text-to-text", "automatic-speech-recognition"], library_name="transformers"
+)
+class Phi4MultimodalOpenVINOConfig(Phi4MMOpenVINOConfig):
+    MIN_TRANSFORMERS_VERSION = "4.51.0"
+    MAX_TRANSFORMERS_VERSION = "4.60.0"
+
+
 class DummyQwen2VLLMInputGenerator(DummyTextInputGenerator):
     def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"):
         generated_input = super().generate(input_name, framework, int_dtype, float_dtype)
@@ -3667,6 +3672,7 @@ class M2M100OpenVINOConfig(BartOpenVINOConfig):
 )
 @register_in_tasks_manager("deepseek", *["text-generation", "text-generation-with-past"], library_name="transformers")
 class DeepseekOpenVINOConfig(MiniCPM3OpenVINOConfig):
+    MIN_TRANSFORMERS_VERSION = "4.46.0"
     MAX_TRANSFORMERS_VERSION = "4.53.3"
     _MODEL_PATCHER = DeepseekPatcher
 

tests/openvino/test_decoder.py

@@ -8,6 +8,7 @@
 import torch
 from parameterized import parameterized
 from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig, PretrainedConfig, pipeline, set_seed
+from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES
 from transformers.testing_utils import slow
 from utils_tests import (
     F32_CONFIG,
@@ -19,7 +20,9 @@
     patch_awq_for_inference,
 )
 
+from optimum.exporters.openvino.model_configs import DeepseekOpenVINOConfig, NemotronOnnxConfig
 from optimum.exporters.openvino.model_patcher import patch_update_causal_mask
+from optimum.exporters.tasks import TasksManager
 from optimum.intel import OVModelForCausalLM, OVModelForSequenceClassification
 from optimum.intel.openvino.utils import _print_compiled_model_properties
 from optimum.intel.pipelines import pipeline as optimum_pipeline
@@ -36,6 +39,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "baichuan2",
         "baichuan2-13b",
         "gpt_bigcode",
+        "bigbird_pegasus",
         "blenderbot",
         "blenderbot-small",
         "bloom",
@@ -52,6 +56,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "mistral",
         "mixtral",
         "mpt",
+        "mbart",
         "opt",
         "pegasus",
         "qwen",
@@ -84,15 +89,15 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "gemma2",
         "exaone",
         "granite",
-        "granite-moe",
+        "granitemoe",
     )
 
-    SUPPORTED_SSM_ARCHITECTURES = ("mamba", "falcon-mamba")
+    SUPPORTED_SSM_ARCHITECTURES = ("mamba", "falcon_mamba")
 
     SUPPORTED_ARCHITECTURES += SUPPORTED_SSM_ARCHITECTURES
 
     if is_transformers_version(">=", "4.46.0"):
-        SUPPORTED_ARCHITECTURES += ("glm", "mistral-nemo", "minicpm3", "phi3-moe")
+        SUPPORTED_ARCHITECTURES += ("glm", "mistral-nemo", "minicpm3", "phimoe")
         # openvino 2025.0 required for disabling check_trace
         if is_openvino_version(">=", "2025.0"):
             SUPPORTED_ARCHITECTURES += ("deepseek",)
@@ -105,17 +110,23 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         if is_openvino_version(">=", "2024.6.0") and platform.system() != "Windows":
             SUPPORTED_ARCHITECTURES += ("mixtral_awq",)
 
+    if is_transformers_version(">", "4.47"):
+        SUPPORTED_ARCHITECTURES += ("olmo2",)
+
+    if is_transformers_version(">", "4.48"):
+        SUPPORTED_ARCHITECTURES += ("nemotron",)
+
     if is_transformers_version(">", "4.49"):
-        SUPPORTED_ARCHITECTURES += ("gemma3_text",)
+        SUPPORTED_ARCHITECTURES += ("gemma3_text", "helium")
 
     if is_transformers_version(">=", "4.51.0"):
-        SUPPORTED_ARCHITECTURES += ("qwen3", "qwen3_moe")
+        SUPPORTED_ARCHITECTURES += ("llama4", "llama4_text", "qwen3", "qwen3_moe")
 
     if is_transformers_version(">=", "4.51.3"):
         SUPPORTED_ARCHITECTURES += ("glm4",)
 
     if is_transformers_version(">=", "4.53.0"):
-        SUPPORTED_ARCHITECTURES += ("arcee",)
+        SUPPORTED_ARCHITECTURES += ("arcee", "smollm3")
 
     if is_transformers_version(">=", "4.54.0"):
         # remote code models differs after transformers v4.54
@@ -151,6 +162,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "bart": 2,
         "baichuan2": 2,
         "baichuan2-13b": 2,
+        "bigbird_pegasus": 2 if is_transformers_version(">=", "4.52") else 0,
         "gpt_bigcode": 5,
         "blenderbot": 2,
         "blenderbot-small": 2,
@@ -163,12 +175,17 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "gpt_neo": 4,
         "gpt_neox": 5,
         "llama": 2,
+        "llama4": 5,
+        "llama4_text": 2,
         "marian": 2,
+        "mbart": 2,
         "minicpm": 4,
         "mistral": 2,
         "mixtral": 2,
         "mpt": 5,
-        "opt": 5 if is_transformers_version(">=", "4.46.0") else 0,
+        "nemotron": 2,
+        "olmo2": 2,
+        "opt": 5 if is_transformers_version(">=", "4.46") else 0,
         "pegasus": 2,
         "qwen": 2,
         "phi": 2,
@@ -188,6 +205,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "olmo": 2,
         "stablelm": 2,
         "starcoder2": 2,
+        "smollm3": 2,
         "dbrx": 2,
         "cohere": 2,
         "qwen2": 2,
@@ -197,22 +215,51 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "gemma2": 4,
         "exaone": 8,
         "granite": 6,
-        "granite-moe": 6,
+        "granitemoe": 6,
         "glm": 28,
         "mistral-nemo": 8,
         "minicpm3": 6,
-        "phi3-moe": 2,
+        "phimoe": 2,
         "deepseek": 2,
         "opt_gptq": 12,
         "mixtral_awq": 2,
         "gemma3_text": 2,
         "glm4": 2,
+        "helium": 2,
         "qwen3": 2,
         "qwen3_moe": 2,
         "mamba": 0,
-        "falcon-mamba": 0,
+        "falcon_mamba": 0,
         "arcee": 2,
     }
+    TASK = "text-generation"
+
+    def test_find_untested_architectures(self):
+        if len(self.SUPPORTED_ARCHITECTURES) != len(set(self.SUPPORTED_ARCHITECTURES)):
+            raise ValueError(
+                f"For the task `{self.TASK}`, some architectures are duplicated in the list of tested architectures: "
+                f"{self.SUPPORTED_ARCHITECTURES}.\n"
+            )
+
+        tested_architectures = set(self.SUPPORTED_ARCHITECTURES)
+        transformers_architectures = set(CONFIG_MAPPING_NAMES.keys())
+        ov_architectures = set(TasksManager.get_supported_model_type_for_task(task=self.TASK, exporter="openvino"))
+        supported_architectures = ov_architectures & transformers_architectures
+
+        if is_transformers_version(">=", str(DeepseekOpenVINOConfig.MAX_TRANSFORMERS_VERSION)):
+            if "deepseek_v2" in supported_architectures:
+                supported_architectures.remove("deepseek_v2")
+            if "deepseek_v3" in supported_architectures:
+                supported_architectures.remove("deepseek_v3")
+        elif is_transformers_version("<", str(NemotronOnnxConfig.MIN_TRANSFORMERS_VERSION)):
+            supported_architectures.remove("nemotron")
+
+        untested_architectures = supported_architectures - tested_architectures
+
+        if len(untested_architectures) > 0:
+            raise ValueError(
+                f"For the task `{self.TASK}`, the OpenVINO exporter supports {untested_architectures} which are not tested"
+            )
 
     # TODO: remove gptq/awq from here
     @parameterized.expand(SUPPORTED_ARCHITECTURES)

tests/openvino/test_export.py

@@ -81,7 +81,7 @@ class ExportModelTest(unittest.TestCase):
         "speecht5": OVModelForTextToSpeechSeq2Seq,
         "clip": OVModelForZeroShotImageClassification,
         "mamba": OVModelForCausalLM,
-        "falcon-mamba": OVModelForCausalLM,
+        "falcon_mamba": OVModelForCausalLM,
         "stable-diffusion-3": OVStableDiffusion3Pipeline,
         "flux": OVFluxPipeline,
         "ltx-video": OVLTXPipeline,

tests/openvino/test_exporters_cli.py

@@ -83,7 +83,7 @@ class OVCLIExportTestCase(unittest.TestCase):
         ("text2text-generation", "t5"),
         ("text2text-generation-with-past", "t5"),
         ("text-generation-with-past", "mamba"),
-        ("text-generation-with-past", "falcon-mamba"),
+        ("text-generation-with-past", "falcon_mamba"),
         ("text-classification", "albert"),
         ("question-answering", "distilbert"),
         ("token-classification", "roberta"),
@@ -126,7 +126,7 @@ class OVCLIExportTestCase(unittest.TestCase):
         "speecht5": 2,
         "clip": 2 if is_tokenizers_version("<", "0.20.0") or is_openvino_version(">=", "2024.5") else 0,
         "mamba": 2,
-        "falcon-mamba": 2,
+        "falcon_mamba": 2,
         "qwen3": 2,
     }