Skip to content

Commit 28bbd58

Browse files
sungmancsungmanc
authored
Remove skip tests, fix regression tests (#2036)
* Remove skip tests, fix regression tests * Fix precommit
1 parent 4c36d00 commit 28bbd58

File tree

7 files changed

+8
-33
lines changed

7 files changed

+8
-33
lines changed

‎tests/e2e/cli/action/test_action_classification.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def test_otx_eval_openvino(self, template, tmp_dir_path):
7575
@pytest.mark.parametrize("template", templates, ids=templates_ids)
7676
def test_pot_optimize(self, template, tmp_dir_path):
7777
if template.name == "MoViNet":
78-
pytest.skip(reason="[CVS-106020] MoViNet fails with POT")
78+
pytest.skip(reason="[CVS-106939] MoViNet fails with POT")
7979
tmp_dir_path = tmp_dir_path / "action_cls"
8080
pot_optimize_testing(template, tmp_dir_path, otx_dir, args)
8181

@@ -84,6 +84,6 @@ def test_pot_optimize(self, template, tmp_dir_path):
8484
@pytest.mark.parametrize("template", templates, ids=templates_ids)
8585
def test_pot_eval(self, template, tmp_dir_path):
8686
if template.name == "MoViNet":
87-
pytest.skip(reason="[CVS-106020] MoViNet fails with POT")
87+
pytest.skip(reason="[CVS-106939] MoViNet fails with POT")
8888
tmp_dir_path = tmp_dir_path / "action_cls"
8989
pot_eval_testing(template, tmp_dir_path, otx_dir, args)

‎tests/e2e/cli/action/test_action_detection.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,31 +58,30 @@ def test_otx_eval(self, template, tmp_dir_path):
5858

5959
@e2e_pytest_component
6060
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
61-
@pytest.mark.skip(reason="CVS-102941 ONNX export of action detection model keeps failed")
6261
@pytest.mark.parametrize("template", templates, ids=templates_ids)
6362
def test_otx_export(self, template, tmp_dir_path):
6463
tmp_dir_path = tmp_dir_path / "action_det"
6564
otx_export_testing(template, tmp_dir_path)
6665

6766
@e2e_pytest_component
6867
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
69-
@pytest.mark.skip(reason="CVS-102941 ONNX export of action detection model keeps failed")
68+
@pytest.mark.skip(reason="CVS-104657 ONNX export of action detection model keeps failed")
7069
@pytest.mark.parametrize("template", templates, ids=templates_ids)
7170
def test_otx_eval_openvino(self, template, tmp_dir_path):
7271
tmp_dir_path = tmp_dir_path / "action_det"
7372
otx_eval_openvino_testing(template, tmp_dir_path, otx_dir, args, threshold=0.05)
7473

7574
@e2e_pytest_component
7675
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
77-
@pytest.mark.skip(reason="CVS-102941 ONNX export of action detection model keeps failed")
76+
@pytest.mark.skip(reason="CVS-104657 ONNX export of action detection model keeps failed")
7877
@pytest.mark.parametrize("template", templates, ids=templates_ids)
7978
def test_pot_optimize(self, template, tmp_dir_path):
8079
tmp_dir_path = tmp_dir_path / "action_det"
8180
pot_optimize_testing(template, tmp_dir_path, otx_dir, args)
8281

8382
@e2e_pytest_component
8483
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
85-
@pytest.mark.skip(reason="CVS-102941 ONNX export of action detection model keeps failed")
84+
@pytest.mark.skip(reason="CVS-104657 ONNX export of action detection model keeps failed")
8685
@pytest.mark.parametrize("template", templates, ids=templates_ids)
8786
def test_pot_eval(self, template, tmp_dir_path):
8887
tmp_dir_path = tmp_dir_path / "action_det"

‎tests/e2e/cli/detection/test_tiling_detection.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,6 @@ def test_otx_hpo(self, template, tmp_dir_path):
191191
@e2e_pytest_component
192192
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
193193
@pytest.mark.parametrize("template", templates, ids=templates_ids)
194-
@pytest.mark.skip(reason="CVS-98026")
195194
def test_nncf_optimize(self, template, tmp_dir_path):
196195
tmp_dir_path = tmp_dir_path / "tiling_det"
197196
if template.entrypoints.nncf is None:
@@ -202,7 +201,6 @@ def test_nncf_optimize(self, template, tmp_dir_path):
202201
@e2e_pytest_component
203202
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
204203
@pytest.mark.parametrize("template", templates, ids=templates_ids)
205-
@pytest.mark.skip(reason="CVS-98026")
206204
def test_nncf_export(self, template, tmp_dir_path):
207205
tmp_dir_path = tmp_dir_path / "tiling_det"
208206
if template.entrypoints.nncf is None:
@@ -213,7 +211,6 @@ def test_nncf_export(self, template, tmp_dir_path):
213211
@e2e_pytest_component
214212
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
215213
@pytest.mark.parametrize("template", templates, ids=templates_ids)
216-
@pytest.mark.skip(reason="CVS-98026")
217214
def test_nncf_validate_fq(self, template, tmp_dir_path):
218215
tmp_dir_path = tmp_dir_path / "tiling_det"
219216
if template.entrypoints.nncf is None:
@@ -224,7 +221,6 @@ def test_nncf_validate_fq(self, template, tmp_dir_path):
224221
@e2e_pytest_component
225222
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
226223
@pytest.mark.parametrize("template", templates, ids=templates_ids)
227-
@pytest.mark.skip(reason="CVS-98026")
228224
def test_nncf_eval(self, template, tmp_dir_path):
229225
tmp_dir_path = tmp_dir_path / "tiling_det"
230226
if template.entrypoints.nncf is None:
@@ -235,7 +231,6 @@ def test_nncf_eval(self, template, tmp_dir_path):
235231
@e2e_pytest_component
236232
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
237233
@pytest.mark.parametrize("template", templates, ids=templates_ids)
238-
@pytest.mark.skip(reason="CVS-98026")
239234
def test_nncf_eval_openvino(self, template, tmp_dir_path):
240235
tmp_dir_path = tmp_dir_path / "tiling_det"
241236
if template.entrypoints.nncf is None:

‎tests/e2e/cli/instance_segmentation/test_instance_segmentation.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,6 @@ def test_otx_hpo(self, template, tmp_dir_path):
198198
@e2e_pytest_component
199199
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
200200
@pytest.mark.parametrize("template", templates, ids=templates_ids)
201-
@pytest.mark.skip(reason="CVS-98026")
202201
def test_nncf_optimize(self, template, tmp_dir_path):
203202
tmp_dir_path = tmp_dir_path / "ins_seg"
204203
if template.entrypoints.nncf is None:
@@ -209,7 +208,6 @@ def test_nncf_optimize(self, template, tmp_dir_path):
209208
@e2e_pytest_component
210209
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
211210
@pytest.mark.parametrize("template", templates, ids=templates_ids)
212-
@pytest.mark.skip(reason="CVS-98026")
213211
def test_nncf_export(self, template, tmp_dir_path):
214212
tmp_dir_path = tmp_dir_path / "ins_seg"
215213
if template.entrypoints.nncf is None:
@@ -220,7 +218,6 @@ def test_nncf_export(self, template, tmp_dir_path):
220218
@e2e_pytest_component
221219
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
222220
@pytest.mark.parametrize("template", templates, ids=templates_ids)
223-
@pytest.mark.skip(reason="CVS-98026")
224221
def test_nncf_validate_fq(self, template, tmp_dir_path):
225222
tmp_dir_path = tmp_dir_path / "ins_seg"
226223
if template.entrypoints.nncf is None:
@@ -231,7 +228,6 @@ def test_nncf_validate_fq(self, template, tmp_dir_path):
231228
@e2e_pytest_component
232229
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
233230
@pytest.mark.parametrize("template", templates, ids=templates_ids)
234-
@pytest.mark.skip(reason="CVS-98026")
235231
def test_nncf_eval(self, template, tmp_dir_path):
236232
tmp_dir_path = tmp_dir_path / "ins_seg"
237233
if template.entrypoints.nncf is None:
@@ -242,7 +238,6 @@ def test_nncf_eval(self, template, tmp_dir_path):
242238
@e2e_pytest_component
243239
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
244240
@pytest.mark.parametrize("template", templates, ids=templates_ids)
245-
@pytest.mark.skip(reason="CVS-98026")
246241
def test_nncf_eval_openvino(self, template, tmp_dir_path):
247242
tmp_dir_path = tmp_dir_path / "ins_seg"
248243
if template.entrypoints.nncf is None:

‎tests/e2e/cli/instance_segmentation/test_tiling_instseg.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,6 @@ def test_otx_hpo(self, template, tmp_dir_path):
192192
@e2e_pytest_component
193193
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
194194
@pytest.mark.parametrize("template", templates, ids=templates_ids)
195-
@pytest.mark.skip(reason="CVS-98026")
196195
def test_nncf_optimize(self, template, tmp_dir_path):
197196
tmp_dir_path = tmp_dir_path / "tiling_ins_seg"
198197
if template.entrypoints.nncf is None:
@@ -203,7 +202,6 @@ def test_nncf_optimize(self, template, tmp_dir_path):
203202
@e2e_pytest_component
204203
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
205204
@pytest.mark.parametrize("template", templates, ids=templates_ids)
206-
@pytest.mark.skip(reason="CVS-98026")
207205
def test_nncf_export(self, template, tmp_dir_path):
208206
tmp_dir_path = tmp_dir_path / "tiling_ins_seg"
209207
if template.entrypoints.nncf is None:
@@ -214,7 +212,6 @@ def test_nncf_export(self, template, tmp_dir_path):
214212
@e2e_pytest_component
215213
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
216214
@pytest.mark.parametrize("template", templates, ids=templates_ids)
217-
@pytest.mark.skip(reason="CVS-98026")
218215
def test_nncf_validate_fq(self, template, tmp_dir_path):
219216
tmp_dir_path = tmp_dir_path / "tiling_ins_seg"
220217
if template.entrypoints.nncf is None:
@@ -225,7 +222,6 @@ def test_nncf_validate_fq(self, template, tmp_dir_path):
225222
@e2e_pytest_component
226223
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
227224
@pytest.mark.parametrize("template", templates, ids=templates_ids)
228-
@pytest.mark.skip(reason="CVS-98026")
229225
def test_nncf_eval(self, template, tmp_dir_path):
230226
tmp_dir_path = tmp_dir_path / "tiling_ins_seg"
231227
if template.entrypoints.nncf is None:
@@ -236,7 +232,6 @@ def test_nncf_eval(self, template, tmp_dir_path):
236232
@e2e_pytest_component
237233
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
238234
@pytest.mark.parametrize("template", templates, ids=templates_ids)
239-
@pytest.mark.skip(reason="CVS-98026")
240235
def test_nncf_eval_openvino(self, template, tmp_dir_path):
241236
tmp_dir_path = tmp_dir_path / "tiling_ins_seg"
242237
if template.entrypoints.nncf is None:

‎tests/e2e/cli/semantic_segmentation/test_segmentation.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -228,31 +228,22 @@ def test_nncf_eval_openvino(self, template, tmp_dir_path):
228228
@e2e_pytest_component
229229
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
230230
@pytest.mark.parametrize("template", templates, ids=templates_ids)
231-
@pytest.mark.skip(reason="Need to check POT with openvino 2022.2.0")
232231
def test_pot_optimize(self, template, tmp_dir_path):
233232
tmp_dir_path = tmp_dir_path / "segmentation"
234-
if template.model_template_id.startswith("ClassIncremental_Semantic_Segmentation_Lite-HRNet-"):
235-
pytest.skip("CVS-82482")
236233
pot_optimize_testing(template, tmp_dir_path, otx_dir, args)
237234

238235
@e2e_pytest_component
239236
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
240237
@pytest.mark.parametrize("template", templates, ids=templates_ids)
241-
@pytest.mark.skip(reason="Need to check POT with openvino 2022.2.0")
242238
def test_pot_validate_fq(self, template, tmp_dir_path):
243239
tmp_dir_path = tmp_dir_path / "segmentation"
244-
if template.model_template_id.startswith("ClassIncremental_Semantic_Segmentation_Lite-HRNet-"):
245-
pytest.skip("CVS-82482")
246240
pot_validate_fq_testing(template, tmp_dir_path, otx_dir, "semantic_segmentation", type(self).__name__)
247241

248242
@e2e_pytest_component
249243
@pytest.mark.skipif(TT_STABILITY_TESTS, reason="This is TT_STABILITY_TESTS")
250244
@pytest.mark.parametrize("template", templates, ids=templates_ids)
251-
@pytest.mark.skip(reason="Need to check POT with openvino 2022.2.0")
252245
def test_pot_eval(self, template, tmp_dir_path):
253246
tmp_dir_path = tmp_dir_path / "segmentation"
254-
if template.model_template_id.startswith("ClassIncremental_Semantic_Segmentation_Lite-HRNet-"):
255-
pytest.skip("CVS-82482")
256247
pot_eval_testing(template, tmp_dir_path, otx_dir, args)
257248

258249
@e2e_pytest_component

‎tests/regression/regression_command.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ def regression_openvino_testing(
119119

120120
if (
121121
exported_performance[k] < trained_performance[k]
122-
or abs(trained_performance[k] - exported_performance[k]) / (trained_performance[k] + 1e-10) > threshold
122+
and abs(trained_performance[k] - exported_performance[k]) / (trained_performance[k] + 1e-10) > threshold
123123
):
124124
regression_result["passed"] = False
125125
regression_result["log"] = f"{trained_performance[k]=}, {exported_performance[k]=}"
@@ -169,7 +169,7 @@ def regression_deployment_testing(
169169
] = f"Deploy performance: ({deployed_performance[k]}) < Criteria: ({modified_criteria})."
170170
if (
171171
deployed_performance[k] < exported_performance[k]
172-
or abs(exported_performance[k] - deployed_performance[k]) / (exported_performance[k] + 1e-10) > threshold
172+
and abs(exported_performance[k] - deployed_performance[k]) / (exported_performance[k] + 1e-10) > threshold
173173
):
174174
regression_result["passed"] = False
175175
regression_result["log"] = f"{exported_performance[k]=}, {deployed_performance[k]=}"
@@ -219,7 +219,7 @@ def regression_nncf_eval_testing(
219219
] = f"NNCF performance: ({evaluated_performance[k]}) < Criteria: ({modified_criteria})."
220220
if (
221221
evaluated_performance[k] < trained_performance[k]
222-
or abs(trained_performance[k] - evaluated_performance[k]) / (trained_performance[k] + 1e-10) > threshold
222+
and abs(trained_performance[k] - evaluated_performance[k]) / (trained_performance[k] + 1e-10) > threshold
223223
):
224224
regression_result["passed"] = False
225225
regression_result["log"] = f"{trained_performance[k]=}, {evaluated_performance[k]=}"

0 commit comments

Comments
 (0)