Skip to content

Commit 206cbf9

Browse files
eaidovaeaidova
committed
add tests
1 parent a3bbaff commit 206cbf9

File tree

1 file changed

+248
-0
lines changed

1 file changed

+248
-0
lines changed

‎tests/openvino/test_modeling.py‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
OVStableDiffusionPipeline,
105105
)
106106
from optimum.intel.openvino import OV_DECODER_NAME, OV_DECODER_WITH_PAST_NAME, OV_ENCODER_NAME, OV_XML_FILE_NAME
107+
from optimum.intel.openvino.configuration import OVConfig
107108
from optimum.intel.openvino.modeling_base import OVBaseModel
108109
from optimum.intel.openvino.modeling_seq2seq import OVDecoder, OVEncoder
109110
from optimum.intel.openvino.modeling_timm import TimmImageProcessor
@@ -863,6 +864,17 @@ def test_pipeline(self, model_arch):
863864
del pipe
864865
gc.collect()
865866

867+
def test_model_loading_with_ov_export_config(self):
868+
model_id = MODEL_NAMES["bert"]
869+
ov_config = OVConfig(dtype="fp16")
870+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
871+
with unittest.mock.patch("openvino.Core.read_model"):
872+
OVModelForSequenceClassification.from_pretrained(
873+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
874+
)
875+
for call in mock_save.calls:
876+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
877+
866878

867879
class OVModelForQuestionAnsweringIntegrationTest(unittest.TestCase):
868880
SUPPORTED_ARCHITECTURES = (
@@ -946,6 +958,17 @@ def test_metric(self):
946958
del ov_model
947959
gc.collect()
948960

961+
def test_model_loading_with_ov_export_config(self):
962+
model_id = MODEL_NAMES["bert"]
963+
ov_config = OVConfig(dtype="fp16")
964+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
965+
with unittest.mock.patch("openvino.Core.read_model"):
966+
OVModelForQuestionAnswering.from_pretrained(
967+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
968+
)
969+
for call in mock_save.calls:
970+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
971+
949972

950973
class OVModelForTokenClassificationIntegrationTest(unittest.TestCase):
951974
SUPPORTED_ARCHITECTURES = (
@@ -1018,6 +1041,17 @@ def test_default_token_type_ids(self):
10181041
del model
10191042
gc.collect()
10201043

1044+
def test_model_loading_with_ov_export_config(self):
1045+
model_id = MODEL_NAMES["bert"]
1046+
ov_config = OVConfig(dtype="fp16")
1047+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
1048+
with unittest.mock.patch("openvino.Core.read_model"):
1049+
OVModelForTokenClassification.from_pretrained(
1050+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
1051+
)
1052+
for call in mock_save.calls:
1053+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
1054+
10211055

10221056
class OVModelForFeatureExtractionIntegrationTest(unittest.TestCase):
10231057
SUPPORTED_ARCHITECTURES = (
@@ -1090,6 +1124,17 @@ def test_sentence_transformers_pipeline(self, model_arch):
10901124
OVModelForFeatureExtraction.from_pretrained(save_dir)
10911125
self.assertIn("Please use `OVSentenceTransformer`", str(context.exception))
10921126

1127+
def test_model_loading_with_ov_export_config(self):
1128+
model_id = MODEL_NAMES["bert"]
1129+
ov_config = OVConfig(dtype="fp16")
1130+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
1131+
with unittest.mock.patch("openvino.Core.read_model"):
1132+
OVModelForFeatureExtraction.from_pretrained(
1133+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
1134+
)
1135+
for call in mock_save.calls:
1136+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
1137+
10931138

10941139
class OVModelForCausalLMIntegrationTest(unittest.TestCase):
10951140
SUPPORTED_ARCHITECTURES = (
@@ -1728,6 +1773,22 @@ def test_load_with_different_dtype(self):
17281773
f"values are not close for {dtype if dtype is not None else 'None'}, max diff = {torch.abs(ov_logits - ref_logits).max()}",
17291774
)
17301775

1776+
def test_model_loading_with_ov_export_config(self):
1777+
model_id = MODEL_NAMES["gpt2"]
1778+
ov_config = OVConfig(dtype="fp16")
1779+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
1780+
with unittest.mock.patch("openvino.Core.read_model"):
1781+
OVModelForCausalLM.from_pretrained(
1782+
model_id,
1783+
export=True,
1784+
ov_config=F32_CONFIG,
1785+
compile=False,
1786+
ov_export_config=ov_config,
1787+
use_cache=False,
1788+
)
1789+
for call in mock_save.calls:
1790+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
1791+
17311792

17321793
class OVModelForMaskedLMIntegrationTest(unittest.TestCase):
17331794
SUPPORTED_ARCHITECTURES = (
@@ -1801,6 +1862,17 @@ def test_pipeline(self, model_arch):
18011862
del model
18021863
gc.collect()
18031864

1865+
def test_model_loading_with_ov_export_config(self):
1866+
model_id = MODEL_NAMES["bert"]
1867+
ov_config = OVConfig(dtype="fp16")
1868+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
1869+
with unittest.mock.patch("openvino.Core.read_model"):
1870+
OVModelForMaskedLM.from_pretrained(
1871+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
1872+
)
1873+
for call in mock_save.calls:
1874+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
1875+
18041876

18051877
class OVModelForImageClassificationIntegrationTest(unittest.TestCase):
18061878
SUPPORTED_ARCHITECTURES = (
@@ -1903,6 +1975,17 @@ def test_timm_save_and_infer(self, model_id):
19031975
model(pixel_values=torch.zeros((5, 3, model.config.image_size, model.config.image_size)))
19041976
gc.collect()
19051977

1978+
def test_model_loading_with_ov_export_config(self):
1979+
model_id = MODEL_NAMES["beit"]
1980+
ov_config = OVConfig(dtype="fp16")
1981+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
1982+
with unittest.mock.patch("openvino.Core.read_model"):
1983+
OVModelForImageClassification.from_pretrained(
1984+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
1985+
)
1986+
for call in mock_save.calls:
1987+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
1988+
19061989

19071990
class OVModelForSeq2SeqLMIntegrationTest(unittest.TestCase):
19081991
SUPPORTED_ARCHITECTURES = (
@@ -2069,6 +2152,17 @@ def test_compare_with_and_without_past_key_values(self):
20692152
del model_without_pkv
20702153
gc.collect()
20712154

2155+
def test_model_loading_with_ov_export_config(self):
2156+
model_id = MODEL_NAMES["bart"]
2157+
ov_config = OVConfig(dtype="fp16")
2158+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
2159+
with unittest.mock.patch("openvino.Core.read_model"):
2160+
OVModelForSeq2SeqLM.from_pretrained(
2161+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
2162+
)
2163+
for call in mock_save.calls:
2164+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
2165+
20722166

20732167
class OVModelForAudioClassificationIntegrationTest(unittest.TestCase):
20742168
SUPPORTED_ARCHITECTURES = (
@@ -2140,6 +2234,17 @@ def test_pipeline(self, model_arch):
21402234
del model
21412235
gc.collect()
21422236

2237+
def test_model_loading_with_ov_export_config(self):
2238+
model_id = MODEL_NAMES["data2vec_audio"]
2239+
ov_config = OVConfig(dtype="fp16")
2240+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
2241+
with unittest.mock.patch("openvino.Core.read_model"):
2242+
OVModelForAudioClassification.from_pretrained(
2243+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
2244+
)
2245+
for call in mock_save.calls:
2246+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
2247+
21432248

21442249
class OVModelForCTCIntegrationTest(unittest.TestCase):
21452250
SUPPORTED_ARCHITECTURES = [
@@ -2196,6 +2301,17 @@ def test_compare_to_transformers(self, model_arch):
21962301
del ov_model
21972302
gc.collect()
21982303

2304+
def test_model_loading_with_ov_export_config(self):
2305+
model_id = MODEL_NAMES["data2vec_audio"]
2306+
ov_config = OVConfig(dtype="fp16")
2307+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
2308+
with unittest.mock.patch("openvino.Core.read_model"):
2309+
OVModelForCTC.from_pretrained(
2310+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
2311+
)
2312+
for call in mock_save.calls:
2313+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
2314+
21992315

22002316
class OVModelForAudioXVectorIntegrationTest(unittest.TestCase):
22012317
SUPPORTED_ARCHITECTURES = [
@@ -2250,6 +2366,17 @@ def test_compare_to_transformers(self, model_arch):
22502366
del ov_model
22512367
gc.collect()
22522368

2369+
def test_model_loading_with_ov_export_config(self):
2370+
model_id = MODEL_NAMES["data2vec_audio"]
2371+
ov_config = OVConfig(dtype="fp16")
2372+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
2373+
with unittest.mock.patch("openvino.Core.read_model"):
2374+
OVModelForAudioXVector.from_pretrained(
2375+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
2376+
)
2377+
for call in mock_save.calls:
2378+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
2379+
22532380

22542381
class OVModelForAudioFrameClassificationIntegrationTest(unittest.TestCase):
22552382
SUPPORTED_ARCHITECTURES = [
@@ -2301,6 +2428,17 @@ def test_compare_to_transformers(self, model_arch):
23012428
del ov_model
23022429
gc.collect()
23032430

2431+
def test_model_loading_with_ov_export_config(self):
2432+
model_id = MODEL_NAMES["data2vec_audio"]
2433+
ov_config = OVConfig(dtype="fp16")
2434+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
2435+
with unittest.mock.patch("openvino.Core.read_model"):
2436+
OVModelForAudioFrameClassification.from_pretrained(
2437+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
2438+
)
2439+
for call in mock_save.calls:
2440+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
2441+
23042442

23052443
class OVModelForPix2StructIntegrationTest(unittest.TestCase):
23062444
SUPPORTED_ARCHITECTURES = ["pix2struct"]
@@ -2394,6 +2532,17 @@ def test_compare_with_and_without_past_key_values(self):
23942532
del model_without_pkv
23952533
gc.collect()
23962534

2535+
def test_model_loading_with_ov_export_config(self):
2536+
model_id = MODEL_NAMES["pix2struct"]
2537+
ov_config = OVConfig(dtype="fp16")
2538+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
2539+
with unittest.mock.patch("openvino.Core.read_model"):
2540+
OVModelForPix2Struct.from_pretrained(
2541+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
2542+
)
2543+
for call in mock_save.calls:
2544+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
2545+
23972546

23982547
class OVModelForVisualCausalLMIntegrationTest(unittest.TestCase):
23992548
SUPPORTED_ARCHITECTURES = ["llava"]
@@ -2792,6 +2941,20 @@ def test_model_can_be_loaded_after_saving(self, model_arch):
27922941
)
27932942
self.assertIsInstance(ov_restored_model, type(ov_model))
27942943

2944+
def test_model_loading_with_ov_export_config(self):
2945+
model_id = MODEL_NAMES["llava"]
2946+
ov_config = OVConfig(dtype="fp16")
2947+
with (
2948+
unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save,
2949+
unittest.mock.patch("openvino.Core.read_model"),
2950+
unittest.mock.patch("optimum.intel.openvino.modeling_visual_language.OVModelWithEmbedForCausalLM"),
2951+
):
2952+
OVModelForVisualCausalLM.from_pretrained(
2953+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
2954+
)
2955+
for call in mock_save.calls:
2956+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
2957+
27952958

27962959
class OVModelForSpeechSeq2SeqIntegrationTest(unittest.TestCase):
27972960
SUPPORTED_ARCHITECTURES = ("whisper",)
@@ -2882,6 +3045,17 @@ def test_pipeline(self, model_arch):
28823045
del model
28833046
gc.collect()
28843047

3048+
def test_model_loading_with_ov_export_config(self):
3049+
model_id = MODEL_NAMES["whisper"]
3050+
ov_config = OVConfig(dtype="fp16")
3051+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
3052+
with unittest.mock.patch("openvino.Core.read_model"):
3053+
OVModelForSpeechSeq2Seq.from_pretrained(
3054+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
3055+
)
3056+
for call in mock_save.calls:
3057+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
3058+
28853059

28863060
class OVModelForVision2SeqIntegrationTest(unittest.TestCase):
28873061
SUPPORTED_ARCHITECTURES = ["vision-encoder-decoder", "trocr", "donut"]
@@ -2991,6 +3165,17 @@ def test_pipeline(self, model_arch: str):
29913165

29923166
gc.collect()
29933167

3168+
def test_model_loading_with_ov_export_config(self):
3169+
model_id = MODEL_NAMES["vision-encoder-decoder"]
3170+
ov_config = OVConfig(dtype="fp16")
3171+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
3172+
with unittest.mock.patch("openvino.Core.read_model"):
3173+
OVModelForVision2Seq.from_pretrained(
3174+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
3175+
)
3176+
for call in mock_save.calls:
3177+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
3178+
29943179

29953180
class OVModelForCustomTasksIntegrationTest(unittest.TestCase):
29963181
SUPPORTED_ARCHITECTURES_WITH_ATTENTION = ["vit-with-attentions"]
@@ -3213,6 +3398,17 @@ def test_functions(self):
32133398
del model
32143399
gc.collect()
32153400

3401+
def test_model_loading_with_ov_export_config(self):
3402+
model_id = MODEL_NAMES["open-clip"]
3403+
ov_config = OVConfig(dtype="fp16")
3404+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
3405+
with unittest.mock.patch("openvino.Core.read_model"):
3406+
OVModelOpenCLIPForZeroShotImageClassification.from_pretrained(
3407+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
3408+
)
3409+
for call in mock_save.calls:
3410+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
3411+
32163412

32173413
class OVModelForSTFeatureExtractionIntegrationTest(unittest.TestCase):
32183414
SUPPORTED_ARCHITECTURES = ("st-bert", "st-mpnet")
@@ -3261,6 +3457,17 @@ def test_langchain(self, model_arch):
32613457
output = embedding.embed_query("foo bar")
32623458
self.assertTrue(len(output) > 0)
32633459

3460+
def test_model_loading_with_ov_export_config(self):
3461+
model_id = MODEL_NAMES["st-bert"]
3462+
ov_config = OVConfig(dtype="fp16")
3463+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
3464+
with unittest.mock.patch("openvino.Core.read_model"):
3465+
OVSentenceTransformer.from_pretrained(
3466+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
3467+
)
3468+
for call in mock_save.calls:
3469+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
3470+
32643471

32653472
class OVLangchainTest(unittest.TestCase):
32663473
SUPPORTED_ARCHITECTURES = ("gpt2",)
@@ -3383,6 +3590,17 @@ def test_reshape(self, model_arch):
33833590
del ov_model
33843591
gc.collect()
33853592

3593+
def test_model_loading_with_ov_export_config(self):
3594+
model_id = MODEL_NAMES["sam"]
3595+
ov_config = OVConfig(dtype="fp16")
3596+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
3597+
with unittest.mock.patch("openvino.Core.read_model"):
3598+
OVSamModel.from_pretrained(
3599+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
3600+
)
3601+
for call in mock_save.calls:
3602+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
3603+
33863604

33873605
class OVModelForTextToSpeechSeq2SeqIntegrationTest(unittest.TestCase):
33883606
SUPPORTED_ARCHITECTURES = ("speecht5",)
@@ -3448,6 +3666,25 @@ def test_compare_to_transformers(self, model_arch):
34483666
del processor
34493667
gc.collect()
34503668

3669+
def test_model_loading_with_ov_export_config(self):
3670+
model_id = MODEL_NAMES["speecht5"]
3671+
ov_config = OVConfig(dtype="fp16")
3672+
with (
3673+
unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save,
3674+
unittest.mock.patch("openvino.Core.read_model"),
3675+
unittest.mock.patch("optimum.intel.openvino.modeling_text2speech.OVTextToSpeechEncoder"),
3676+
):
3677+
OVModelForTextToSpeechSeq2Seq.from_pretrained(
3678+
model_id,
3679+
vocoder="fxmarty/speecht5-hifigan-tiny",
3680+
export=True,
3681+
ov_config=F32_CONFIG,
3682+
compile=False,
3683+
ov_export_config=ov_config,
3684+
)
3685+
for call in mock_save.calls:
3686+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)
3687+
34513688

34523689
class OVModelForZeroShotImageClassificationIntegrationTest(unittest.TestCase):
34533690
SUPPORTED_ARCHITECTURES = ["clip"]
@@ -3499,3 +3736,14 @@ def test_compare_to_transformers(self, model_arch):
34993736
del transformers_model
35003737
del ov_model
35013738
gc.collect()
3739+
3740+
def test_model_loading_with_ov_export_config(self):
3741+
model_id = MODEL_NAMES["clip"]
3742+
ov_config = OVConfig(dtype="fp16")
3743+
with unittest.mock.patch("optimum.exporters.openvino.convert._save_model") as mock_save:
3744+
with unittest.mock.patch("openvino.Core.read_model"):
3745+
OVModelForZeroShotImageClassification.from_pretrained(
3746+
model_id, export=True, ov_config=F32_CONFIG, compile=False, ov_export_config=ov_config
3747+
)
3748+
for call in mock_save.calls:
3749+
self.assertEqual(call.call_args.kwargs["ov_config"], ov_config)

0 commit comments

Comments
 (0)