Skip to content

Add test to ensure all supported models are tested #1458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 31 commits into
base: main
Choose a base branch
from
Draft

Add test to ensure all supported models are tested #1458

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9e80aba
add tests
echarlaix Oct 1, 2025
563de33
reorder
echarlaix Oct 1, 2025
e280b28
rename
echarlaix Oct 1, 2025
252368c
add olmo and smollm3
echarlaix Oct 1, 2025
e8ad17d
fix
echarlaix Oct 1, 2025
f978fce
add llama4 test
echarlaix Oct 2, 2025
226df98
add min deepseek transformers version
echarlaix Oct 2, 2025
7a8a1d7
remove min transformers constraint
echarlaix Oct 2, 2025
9b4b2e9
add tiny llama4 model
echarlaix Oct 2, 2025
9559e2f
add gpt-oss to documentation
echarlaix Oct 2, 2025
2389be1
remove reorder as it makes things less clear for review
echarlaix Oct 2, 2025
1819b85
add tests expected sdpa
echarlaix Oct 2, 2025
586d2e5
add model doc
echarlaix Oct 3, 2025
e9a7c7c
merge main
echarlaix Oct 3, 2025
65ce535
add ov attribute check for tests
echarlaix Oct 3, 2025
34ec242
style
echarlaix Oct 3, 2025
9c30790
style
echarlaix Oct 3, 2025
206246e
add vlm check_openvino_model_attributes
echarlaix Oct 3, 2025
a6035e9
set back img size
echarlaix Oct 3, 2025
a2cbaed
fix longt5
echarlaix Oct 3, 2025
11cf624
marian should not be tested depednign on openvino version
echarlaix Oct 3, 2025
76b3405
add untested architectures for vision2seq models
echarlaix Oct 6, 2025
a44664e
phi4_multimodal
echarlaix Oct 6, 2025
21689cb
add phi4_multimodal tiny model for test
echarlaix Oct 6, 2025
6c4f99d
style
echarlaix Oct 6, 2025
63f22a9
add helium and nemotron
echarlaix Oct 6, 2025
0e382c6
merge main in branch
echarlaix Oct 21, 2025
b96d316
fix doc
echarlaix Oct 22, 2025
f0f15d3
fix nemotron
echarlaix Oct 22, 2025
9c3e916
merge main in branch
echarlaix Oct 22, 2025
d67b8c4
revert
echarlaix Oct 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions optimum/exporters/openvino/model_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3969,6 +3969,7 @@ class M2M100OpenVINOConfig(BartOpenVINOConfig):
)
@register_in_tasks_manager("deepseek", *["text-generation", "text-generation-with-past"], library_name="transformers")
class DeepseekOpenVINOConfig(MiniCPM3OpenVINOConfig):
MIN_TRANSFORMERS_VERSION = "4.46.0"
Copy link
Collaborator Author

@echarlaix echarlaix Oct 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

only tested for transformers >= v4.46
https://github.com/huggingface/optimum-intel/blob/v1.25.0/tests/openvino/test_modeling.py#L1163

MAX_TRANSFORMERS_VERSION = "4.54.0"

def patch_model_for_export(
Expand Down
112 changes: 75 additions & 37 deletions tests/openvino/test_decoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@
pipeline,
set_seed,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES
from transformers.testing_utils import slow
from utils_tests import MODEL_NAMES, get_num_sdpa, mock_torch_cuda_is_available, patch_awq_for_inference

from optimum.exporters.openvino.model_configs import DeepseekOpenVINOConfig
from optimum.exporters.openvino.model_patcher import patch_update_causal_mask
from optimum.exporters.tasks import TasksManager
from optimum.intel import OVModelForCausalLM, OVModelForSequenceClassification
from optimum.intel.openvino.utils import _print_compiled_model_properties
from optimum.intel.pipelines import pipeline as optimum_pipeline
Expand All @@ -36,67 +39,71 @@

class OVModelForCausalLMIntegrationTest(unittest.TestCase):
SUPPORTED_ARCHITECTURES = (
"bart",
"aquila",
"aquila2",
"arctic",
"baichuan2",
"baichuan2-13b",
"gpt_bigcode",
"bart",
"bigbird_pegasus",
"biogpt",
"blenderbot",
"blenderbot-small",
"bloom",
"chatglm",
"chatglm4",
"codegen",
"codegen2",
"cohere",
"dbrx",
"decilm",
"exaone",
"falcon",
"falcon-40b",
"gemma",
"gemma2",
"gpt2",
"gptj",
"gpt_bigcode",
"gpt_neo",
"gpt_neox",
"gpt_neox_japanese",
"gptj",
"granite",
"granitemoe",
"internlm",
"internlm2",
"jais",
"llama",
"llama4",
"llama4_text",
"marian",
"mbart",
"minicpm",
"mistral",
"mixtral",
"mpt",
"olmo",
"opt",
"pegasus",
"qwen",
"phi",
"internlm2",
"orion",
"falcon",
"falcon-40b",
"pegasus",
"persimmon",
"biogpt",
"gpt_neox_japanese",
"xglm",
"aquila",
"aquila2",
"xverse",
"internlm",
"jais",
"chatglm4",
"decilm",
"gemma",
"olmo",
"stablelm",
"starcoder2",
"dbrx",
"cohere",
"phi",
"phi3",
"qwen",
"qwen2",
"qwen2_moe",
"arctic",
"phi3",
"gemma2",
"exaone",
"granite",
"granite-moe",
"stablelm",
"starcoder2",
"xglm",
"xverse",
)

SUPPORTED_SSM_ARCHITECTURES = ("mamba", "falcon-mamba")
SUPPORTED_SSM_ARCHITECTURES = ("mamba", "falcon_mamba")

SUPPORTED_ARCHITECTURES += SUPPORTED_SSM_ARCHITECTURES

if is_transformers_version(">=", "4.46.0"):
SUPPORTED_ARCHITECTURES += ("glm", "mistral-nemo", "minicpm3", "phi3-moe")
SUPPORTED_ARCHITECTURES += ("glm", "mistral-nemo", "minicpm3", "phimoe")
# openvino 2025.0 required for disabling check_trace
if is_openvino_version(">=", "2025.0"):
SUPPORTED_ARCHITECTURES += ("deepseek",)
Expand All @@ -109,6 +116,9 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
if is_openvino_version(">=", "2024.6.0") and platform.system() != "Windows":
SUPPORTED_ARCHITECTURES += ("mixtral_awq",)

if is_transformers_version(">", "4.47"):
SUPPORTED_ARCHITECTURES += ("olmo2",)

if is_transformers_version(">", "4.49"):
SUPPORTED_ARCHITECTURES += ("gemma3_text",)

Expand All @@ -119,7 +129,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
SUPPORTED_ARCHITECTURES += ("glm4",)

if is_transformers_version(">=", "4.53.0"):
SUPPORTED_ARCHITECTURES += ("arcee",)
SUPPORTED_ARCHITECTURES += ("arcee", "smollm3")

if is_transformers_version(">=", "4.54.0"):
# remote code models differs after transformers v4.54
Expand Down Expand Up @@ -167,6 +177,8 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
"gpt_neo": 4,
"gpt_neox": 5,
"llama": 2,
"llama4": 5,
"llama4_text": 2,
"marian": 2,
"minicpm": 4,
"mistral": 2,
Expand Down Expand Up @@ -201,11 +213,11 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
"gemma2": 4,
"exaone": 8,
"granite": 6,
"granite-moe": 6,
"granitemoe": 6,
"glm": 28,
"mistral-nemo": 8,
"minicpm3": 6,
"phi3-moe": 2,
"phimoe": 2,
"deepseek": 2,
"opt_gptq": 12,
"mixtral_awq": 2,
Expand All @@ -214,9 +226,35 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
"qwen3": 2,
"qwen3_moe": 2,
"mamba": 0,
"falcon-mamba": 0,
"falcon_mamba": 0,
"arcee": 2,
}
TASK = "text-generation"

def test_find_untested_architectures(self):
if len(self.SUPPORTED_ARCHITECTURES) != len(set(self.SUPPORTED_ARCHITECTURES)):
raise ValueError(
f"For the task `{self.TASK}`, some architectures are duplicated in the list of tested architectures: "
f"{self.SUPPORTED_ARCHITECTURES}.\n"
)

tested_architectures = set(self.SUPPORTED_ARCHITECTURES)
transformers_architectures = set(CONFIG_MAPPING_NAMES.keys())
ov_architectures = set(TasksManager.get_supported_model_type_for_task(task=self.TASK, exporter="openvino"))
supported_architectures = ov_architectures & transformers_architectures

if is_transformers_version(">=", str(DeepseekOpenVINOConfig.MAX_TRANSFORMERS_VERSION)):
if "deepseek_v2" in supported_architectures:
supported_architectures.remove("deepseek_v2")
if "deepseek_v3" in supported_architectures:
supported_architectures.remove("deepseek_v3")

untested_architectures = supported_architectures - tested_architectures

if len(untested_architectures) > 0:
raise ValueError(
f"For the task `{self.TASK}`, the OpenVINO exporter supports {untested_architectures} which are not tested"
)

# TODO: remove gptq/awq from here
@parameterized.expand(SUPPORTED_ARCHITECTURES)
Expand Down
2 changes: 1 addition & 1 deletion tests/openvino/test_export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class ExportModelTest(unittest.TestCase):
"speecht5": OVModelForTextToSpeechSeq2Seq,
"clip": OVModelForZeroShotImageClassification,
"mamba": OVModelForCausalLM,
"falcon-mamba": OVModelForCausalLM,
"falcon_mamba": OVModelForCausalLM,
"stable-diffusion-3": OVStableDiffusion3Pipeline,
"flux": OVFluxPipeline,
"ltx-video": OVLTXPipeline,
Expand Down
4 changes: 2 additions & 2 deletions tests/openvino/test_exporters_cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ class OVCLIExportTestCase(unittest.TestCase):
("text2text-generation", "t5"),
("text2text-generation-with-past", "t5"),
("text-generation-with-past", "mamba"),
("text-generation-with-past", "falcon-mamba"),
("text-generation-with-past", "falcon_mamba"),
("text-classification", "albert"),
("question-answering", "distilbert"),
("token-classification", "roberta"),
Expand Down Expand Up @@ -127,7 +127,7 @@ class OVCLIExportTestCase(unittest.TestCase):
"speecht5": 2,
"clip": 2 if is_tokenizers_version("<", "0.20.0") or is_openvino_version(">=", "2024.5") else 0,
"mamba": 2,
"falcon-mamba": 2,
"falcon_mamba": 2,
"qwen3": 2,
}

Expand Down
14 changes: 7 additions & 7 deletions tests/openvino/utils_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
"gemma3": "katuni4ka/tiny-random-gemma3",
"falcon": "fxmarty/really-tiny-falcon-testing",
"falcon-40b": "katuni4ka/tiny-random-falcon-40b",
"falcon-mamba": "rkazants/tiny-falcon-mamba",
"falcon_mamba": "rkazants/tiny-falcon-mamba",
"flaubert": "hf-internal-testing/tiny-random-flaubert",
"flux": "katuni4ka/tiny-random-flux",
"flux-fill": "katuni4ka/tiny-random-flux-fill",
Expand All @@ -88,7 +88,7 @@
"gpt_oss_mxfp4": "echarlaix/tiny-random-gpt-oss-mxfp4",
"gptj": "hf-internal-testing/tiny-random-GPTJModel",
"granite": "katuni4ka/tiny-random-granite",
"granite-moe": "katuni4ka/tiny-random-granite-moe",
"granitemoe": "katuni4ka/tiny-random-granite-moe",
"hubert": "hf-internal-testing/tiny-random-HubertModel",
"ibert": "hf-internal-testing/tiny-random-ibert",
"idefics3": "hf-internal-testing/tiny-random-Idefics3ForConditionalGeneration",
Expand All @@ -101,11 +101,13 @@
"llama": "HuggingFaceM4/tiny-random-LlamaForCausalLM",
"llama_awq": "HuggingFaceH4/tiny-random-LlamaForCausalLM",
"llama4": "hf-internal-testing/tiny-random-llama4",
"llama4_text": "trl-internal-testing/tiny-Llama4ForCausalLM",
"llava": "katuni4ka/tiny-random-llava",
"llava_next": "katuni4ka/tiny-random-llava-next",
"llava_next_mistral": "optimum-internal-testing/tiny-random-llava-next-mistral",
"llava_next_video": "katuni4ka/tiny-random-llava-next-video",
"m2m_100": "hf-internal-testing/tiny-random-m2m_100",
"olmo2": "hf-internal-testing/tiny-random-Olmo2ForCausalLM",
"opt": "hf-internal-testing/tiny-random-OPTModel",
"opt125m": "facebook/opt-125m",
"opt_gptq": "ybelkada/opt-125m-gptq-4bit",
Expand Down Expand Up @@ -139,7 +141,7 @@
"pix2struct": "fxmarty/pix2struct-tiny-random",
"phi": "echarlaix/tiny-random-PhiForCausalLM",
"phi3": "Xenova/tiny-random-Phi3ForCausalLM",
"phi3-moe": "katuni4ka/phi-3.5-moe-tiny-random",
"phimoe": "katuni4ka/phi-3.5-moe-tiny-random",
"phi3_v": "katuni4ka/tiny-random-phi3-vision",
"phi4mm": "katuni4ka/tiny-random-phi-4-multimodal",
"poolformer": "hf-internal-testing/tiny-random-PoolFormerModel",
Expand All @@ -156,6 +158,7 @@
"segformer": "hf-internal-testing/tiny-random-SegformerModel",
"sentence-transformers-bert": "sentence-transformers-testing/stsb-bert-tiny-safetensors",
"sam": "fxmarty/sam-vit-tiny-random",
"smollm3": "optimum-internal-testing/tiny-random-SmolLM3ForCausalLM",
"smolvlm": "katuni4ka/tiny-random-smolvlm2",
"speecht5": "hf-internal-testing/tiny-random-SpeechT5ForTextToSpeech",
"speech_to_text": "hf-internal-testing/tiny-random-Speech2TextModel",
Expand Down Expand Up @@ -326,7 +329,7 @@
},
"clip": {"model": 130},
"mamba": {"model": 386},
"falcon-mamba": {"model": 194},
"falcon_mamba": {"model": 194},
}

TEST_IMAGE_URL = "http://images.cocodataset.org/val2017/000000039769.jpg"
Expand Down Expand Up @@ -457,10 +460,8 @@ def get_num_sdpa(model):
"baichuan2-13b": "baichuan",
"chatglm4": "chatglm",
"codegen2": "codegen",
"falcon-mamba": "falcon_mamba",
"falcon-40b": "falcon",
"gpt_oss_mxfp4": "gpt_oss",
"granite-moe": "granitemoe",
"llama_awq": "llama",
"llava_next_mistral": "llava_next",
"mistral-nemo": "mistral",
Expand All @@ -470,7 +471,6 @@ def get_num_sdpa(model):
"opt_gptq": "opt",
"perceiver_text": "perceiver",
"perceiver_vision": "perceiver",
"phi3-moe": "phimoe",
"swin-window": "swin",
"vit-with-attentions": "vit",
"vit-with-hidden-states": "vit",
Expand Down
Loading