Correct selection of metrics with sub_evaluation disabled (#3902)

* Ignore subset_metrics if sub_evaluation not enabled

* Test slection of metrics with sub_evaluation

Author: pwolnows

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/model_evaluator.py

@@ -793,7 +793,7 @@ def provide_metric_references(cls, conf, return_header=True):
 def get_config_metrics(config):
     metrics = None
     sub_evaluation = config.get('sub_evaluation', False)
-    if sub_evaluation is not None:
+    if sub_evaluation:
         size = config.get('subsample_size')
         subset_metrics = config.get('subset_metrics', [])
         for item in subset_metrics:

tools/accuracy_checker/tests/test_model_evaluator.py

@@ -17,7 +17,7 @@
 from unittest.mock import Mock, MagicMock
 
 from openvino.tools.accuracy_checker.evaluators import ModelEvaluator
-
+from openvino.tools.accuracy_checker.evaluators.model_evaluator import get_config_metrics
 
 class TestModelEvaluator:
     def setup_method(self):
@@ -143,6 +143,58 @@ def test_process_dataset_with_loading_predictions_and_with_dataset_processors(se
         assert not self.postprocessor.process_dataset.called
         assert self.postprocessor.full_process.called
 
+    def test_model_evaluator_get_config_metrics(self, mocker):
+        dataset_config = { 
+            'metrics': [{'type': 'accuracy', 'top_k': 1, 'reference': 0.78}], 
+            'subset_metrics': [{'subset_size': '20%', 
+                'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.65}]}]
+        }
+        metric = {'type': 'accuracy', 'top_k': 1, 'reference': 0.78}
+        selected_metric = get_config_metrics(dataset_config)[0]
+
+        assert metric['reference'] == selected_metric['reference']
+        assert metric['top_k'] == selected_metric['top_k']
+
+    def test_model_evaluator_get_config_metrics_is_first_subset_metrics(self, mocker):
+        dataset_config_sub_evaluation = { 'sub_evaluation' : 'True', 
+            'metrics': [{'type': 'accuracy', 'top_k': 1, 'reference': 0.78}], 
+            'subset_metrics': [
+                {'subset_size': '10%', 'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.65}]},
+                {'subset_size': '20%', 'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.72}]}]
+        }
+        subset_metric = {'type': 'accuracy', 'top_k': 5, 'reference': 0.65}
+        selected_metric = get_config_metrics(dataset_config_sub_evaluation)[0]
+
+        assert subset_metric['reference'] == selected_metric['reference']
+        assert subset_metric['top_k'] == selected_metric['top_k']
+
+    def test_model_evaluator_get_config_metrics_with_subsample_size_from_subset_metrics(self, mocker):
+        dataset_config_sub_evaluation = { 'sub_evaluation' : 'True', 'subsample_size': '20%', 
+            'metrics': [{'type': 'accuracy', 'top_k': 1, 'reference': 0.78}], 
+            'subset_metrics': [
+                {'subset_size': '10%', 'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.65}]},
+                {'subset_size': '20%', 'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.72}]}]
+        }
+        subset_metric = {'type': 'accuracy', 'top_k': 5, 'reference': 0.72}
+        selected_metric = get_config_metrics(dataset_config_sub_evaluation)[0]
+
+        assert subset_metric['reference'] == selected_metric['reference']
+        assert subset_metric['top_k'] == selected_metric['top_k']
+
+
+    def test_model_evaluator_get_config_metrics_from_subset_metrics(self, mocker):
+        dataset_config_sub_evaluation = { 'sub_evaluation' : 'True', 
+            'metrics': [{'type': 'accuracy', 'top_k': 1, 'reference': 0.78}], 
+            'subset_metrics': [{'subset_size': '20%', 
+                'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.65}]}]
+        }
+        subset_metric = {'type': 'accuracy', 'top_k': 5, 'reference': 0.65}
+        selected_metric = get_config_metrics(dataset_config_sub_evaluation)[0]
+
+        assert subset_metric['reference'] == selected_metric['reference']
+        assert subset_metric['top_k'] == selected_metric['top_k']
+
+
 
 class TestModelEvaluatorAsync:
     def setup_method(self):