Update reference collection according to feedback (#2925)

Author: eaidova

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/model_evaluator.py

@@ -761,31 +761,21 @@ def release(self):
             self.adapter.release()
 
     @classmethod
-    def provide_metric_references(cls, conf, subset, return_header=True):
+    def provide_metric_references(cls, conf, return_header=True):
         processing_info = cls.get_processing_info(conf)
         dataset_config = conf['datasets'][0]
-        dataset = Dataset(dataset_config, log=False)
-        dataset_size = len(dataset)
-        ignore_config_refs = False
-        if subset is not None:
-            dataset_config['subsample_size'] = subset
-            new_dataset = Dataset(dataset_config, log=False)
-            if len(new_dataset) != len(dataset):
-                ignore_config_refs = True
-                warning('Subset is not matched with configuration. Reference values will be ignored')
-                dataset_size = len(new_dataset)
-                dataset = new_dataset
-        metric_dispatcher = MetricsExecutor(dataset_config.get('metrics', []), dataset)
+        metric_dispatcher = MetricsExecutor(dataset_config.get('metrics', []), postpone_metrics=True)
         extracted_results, extracted_meta = [], []
-        for result_presenter, metric_result in metric_dispatcher.get_metric_result_template(ignore_config_refs):
-            result, metadata = result_presenter.extract_result(metric_result)
+        for result_presenter, metric_result in metric_dispatcher.get_metric_result_template(
+            dataset_config.get('metrics', []), False):
+            result, metadata = result_presenter.extract_result(metric_result, names_from_refs=True)
             if isinstance(result, list):
                 extracted_results.extend(result)
                 extracted_meta.extend(metadata)
             else:
                 extracted_results.append(result)
                 extracted_meta.append(metadata)
-        header, report = generate_csv_report(processing_info, extracted_results, dataset_size, extracted_meta)
+        header, report = generate_csv_report(processing_info, extracted_results, 0, extracted_meta)
         if not return_header:
             return report
         return header, report

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/module_evaluator.py

@@ -20,8 +20,6 @@
 import importlib
 from pathlib import Path
 from .base_evaluator import BaseEvaluator
-from ..logging import warning
-from ..dataset import Dataset
 from ..presenters import generate_csv_report
 from ..metrics import MetricsExecutor
 
@@ -147,31 +145,21 @@ def dataset_size(self):
         return self._internal_module.dataset_size
 
     @classmethod
-    def provide_metric_references(cls, conf, subset, return_header=True):
+    def provide_metric_references(cls, conf, return_header=True):
         processing_info = cls.get_processing_info(conf)
         dataset_config = conf['module_config']['datasets'][0]
-        dataset = Dataset(dataset_config)
-        dataset_size = len(dataset)
-        ignore_config_refs = False
-        if subset is not None:
-            dataset_config['subsample_size'] = subset
-            new_dataset = Dataset(dataset_config)
-            if len(new_dataset) != len(dataset):
-                ignore_config_refs = True
-                warning('Subset is not matched with configuration. Reference values will be ignored')
-                dataset_size = len(new_dataset)
-                dataset = new_dataset
-        metric_dispatcher = MetricsExecutor(dataset_config.get('metrics', []), dataset)
+        metric_dispatcher = MetricsExecutor(dataset_config.get('metrics', []), postpone_metrics=True)
         extracted_results, extracted_meta = [], []
-        for result_presenter, metric_result in metric_dispatcher.get_metric_result_template(ignore_config_refs):
-            result, metadata = result_presenter.extract_result(metric_result)
+        for result_presenter, metric_result in metric_dispatcher.get_metric_result_template(
+            dataset_config.get('metrics', []), False):
+            result, metadata = result_presenter.extract_result(metric_result, names_from_refs=True)
             if isinstance(result, list):
                 extracted_results.extend(result)
                 extracted_meta.extend(metadata)
             else:
                 extracted_results.append(result)
                 extracted_meta.append(metadata)
-        header, report = generate_csv_report(processing_info, extracted_results, dataset_size, extracted_meta)
+        header, report = generate_csv_report(processing_info, extracted_results, 0, extracted_meta)
         if not return_header:
             return report
         return header, report

tools/accuracy_checker/openvino/tools/accuracy_checker/metrics/audio_processing.py

@@ -38,8 +38,7 @@ def parameters(cls):
     def configure(self):
         self.delay = self.get_value_from_config('delay')
         self.buffer = []
-        self.meta.update({'scale': 1, 'postfix': 'Db', 'calculate_mean': False, 'names': ['mean', 'std']})
-        self.meta['target_per_value'] = {'mean': 'higher-better', 'std': 'higher-worse'}
+        self.meta = self.get_common_meta()
 
     def reset(self):
         del self.buffer
@@ -67,3 +66,12 @@ def update(self, annotation, prediction):
 
     def evaluate(self, annotations, predictions):
         return [np.mean(self.buffer), np.std(self.buffer)]
+
+    @classmethod
+    def get_common_meta(cls):
+        meta = super().get_common_meta()
+        meta.update({
+            'scale': 1, 'postfix': 'Db', 'calculate_mean': False, 'names': ['mean', 'std'],
+            'target_per_value': {'mean': 'higher-better', 'std': 'higher-worse'}
+        })
+        return meta

tools/accuracy_checker/openvino/tools/accuracy_checker/metrics/classification.py

@@ -371,7 +371,6 @@ def __init__(self, *args, **kwargs):
         self.video_avg_prob = AverageProbMeter()
         self.previous_video_id = None
         self.previous_video_label = None
-        self.meta['names'] = ['clip_accuracy', 'video_accuracy']
 
     def update(self, annotation, prediction):
         if isinstance(annotation.identifier, list):
@@ -407,6 +406,10 @@ def reset(self):
         if self.profiler:
             self.profiler.reset()
 
+    @classmethod
+    def get_common_meta(cls):
+        return {'target': 'higher-better', 'names': ['clip_accuracy', 'video_accuracy']}
+
 
 class ClassificationF1Score(PerImageEvaluationMetric):
     __provider__ = 'classification_f1-score'
@@ -553,9 +556,6 @@ class AcerScore(PerImageEvaluationMetric):
     def configure(self):
         if isinstance(confusion_matrix, UnsupportedPackage):
             confusion_matrix.raise_error(self.__provider__)
-        self.meta.update({
-            'target': 'higher-worse'
-        })
         self.reset()
 
     def update(self, annotation, prediction):
@@ -579,3 +579,7 @@ def evaluate(self, annotations, predictions):
     def reset(self):
         self.targets = []
         self.results = []
+
+    @classmethod
+    def get_common_meta(cls):
+        return {'target': 'higher-worse'}

tools/accuracy_checker/openvino/tools/accuracy_checker/metrics/dna_seq_accuracy.py

@@ -36,10 +36,6 @@ def configure(self):
         self.balanced = self.get_value_from_config('balanced')
         self.min_coverage = self.get_value_from_config('min_coverage')
         self.accuracy = []
-        self.meta.update({
-            'names': ['mean', 'median'],
-            'calculate_mean': False
-        })
 
     def update(self, annotation, prediction):
         alignment = parasail.sw_trace_striped_32(prediction.label, annotation.label, 8, 4, parasail.dnafull)
@@ -94,3 +90,12 @@ def evaluate(self, annotations, predictions):
 
     def reset(self):
         self.accuracy = []
+
+    @classmethod
+    def get_common_meta(cls):
+        meta = super().get_common_meta()
+        meta.update({
+            'names': ['mean', 'median'],
+            'calculate_mean': False
+        })
+        return meta

tools/accuracy_checker/openvino/tools/accuracy_checker/metrics/gan_metrics.py

@@ -52,9 +52,6 @@ def parameters(cls):
         return parameters
 
     def configure(self):
-        self.meta.update({
-            'scale': 1, 'postfix': ' ', 'target': 'higher-worse'
-        })
         self.eps = self.get_value_from_config('eps')
         self.length = self.get_value_from_config('length')
 
@@ -71,6 +68,14 @@ def evaluate(self, annotations, predictions):
         generated = np.stack(generated)
         return self.score_calc(real, generated)
 
+    @classmethod
+    def get_common_meta(cls):
+        meta = super().get_common_meta()
+        meta.update({
+            'scale': 1, 'postfix': ' ', 'target': 'higher-worse'
+        })
+        return meta
+
 
 class InceptionScore(BaseGanMetric):
     """

tools/accuracy_checker/openvino/tools/accuracy_checker/metrics/hit_ratio.py

@@ -119,13 +119,6 @@ class LogLoss(PerImageEvaluationMetric):
 
     def configure(self):
         self.losses = []
-        self.meta.update({
-            'scale': 1,
-            'postfix': ' ',
-            'calculate_mean': False,
-            'target': 'higher-worse',
-            'names': ['mean', 'std'],
-        })
 
     def update(self, annotation, prediction):
         score = np.clip(prediction.scores, 1e-15, 1 - 1e-15)
@@ -138,3 +131,13 @@ def evaluate(self, annotations, predictions):
 
     def reset(self):
         self.losses = []
+
+    @classmethod
+    def get_common_meta(cls):
+        return {
+            'scale': 1,
+            'postfix': ' ',
+            'calculate_mean': False,
+            'target': 'higher-worse',
+            'names': ['mean', 'std'],
+        }

tools/accuracy_checker/openvino/tools/accuracy_checker/metrics/image_quality_assessment.py

@@ -62,8 +62,13 @@ class StructuralSimilarity(BaseRegressionMetric):
 
     def __init__(self, *args, **kwargs):
         super().__init__(_ssim, *args, **kwargs)
-        self.meta['target'] = 'higher-better'
-        self.meta['target_per_value'] = {'mean': 'higher-better', 'std': 'higher-worse', 'max_error': 'higher-worse'}
+
+    @classmethod
+    def get_common_meta(cls):
+        meta = super().get_common_meta()
+        meta['target'] = 'higher-better'
+        meta['target_per_value'] = {'mean': 'higher-better', 'std': 'higher-worse', 'max_error': 'higher-worse'}
+        return meta
 
 
 class PeakSignalToNoiseRatio(BaseRegressionMetric):
@@ -92,8 +97,6 @@ def parameters(cls):
 
     def __init__(self, *args, **kwargs):
         super().__init__(self._psnr_differ, *args, **kwargs)
-        self.meta['target'] = 'higher-better'
-        self.meta['target_per_value'] = {'mean': 'higher-better', 'std': 'higher-worse', 'max_error': 'higher-worse'}
 
     def configure(self):
         super().configure()
@@ -103,7 +106,6 @@ def configure(self):
             'BGR': [2, 1, 0],
             'RGB': [0, 1, 2],
         }
-        self.meta['postfix'] = 'Db'
         self.channel_order = channel_order[self.color_order]
         self.normalized_images = self.get_value_from_config('normalized_images')
         self.color_scale = 255 if not self.normalized_images else 1
@@ -137,6 +139,14 @@ def _psnr_differ(self, annotation_image, prediction_image):
 
         return -10 * math.log10(mse)
 
+    @classmethod
+    def get_common_meta(cls):
+        meta = super().get_common_meta()
+        meta['target'] = 'higher-better'
+        meta['target_per_value'] = {'mean': 'higher-better', 'std': 'higher-worse', 'max_error': 'higher-worse'}
+        meta['postfix'] = 'Db'
+        return meta
+
 
 class PeakSignalToNoiseRatioWithBlockingEffectFactor(PeakSignalToNoiseRatio):
     __provider__ = 'psnr-b'
@@ -280,8 +290,6 @@ def parameters(cls):
 
     def __init__(self, *args, **kwargs):
         super().__init__(self._vif_diff, *args, **kwargs)
-        self.meta['target'] = 'higher-better'
-        self.meta['target_per_value'] = {'mean': 'higher-better', 'std': 'higher-worse', 'max_error': 'higher-worse'}
         if isinstance(convolve2d, UnsupportedPackage):
             convolve2d.raise_error(self.__provider__)
 
@@ -335,6 +343,13 @@ def _vifp_single(gt, p, sigma_nsq):
 
         return num / den
 
+    @classmethod
+    def get_common_meta(cls):
+        meta = super().get_common_meta()
+        meta['target'] = 'higher-better'
+        meta['target_per_value'] = {'mean': 'higher-better', 'std': 'higher-worse', 'max_error': 'higher-worse'}
+        return meta
+
 
 def gaussian_filter(ws, sigma):
     x, y = np.mgrid[-ws // 2 + 1:ws // 2 + 1, -ws // 2 + 1:ws // 2 + 1]

tools/accuracy_checker/openvino/tools/accuracy_checker/metrics/language_modeling.py

@@ -30,7 +30,6 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.loss = 0
         self.total = 0
-        self.meta['target'] = 'higher-worse'
 
     def update(self, annotation, prediction):
         def cross_entropy(logits, target):
@@ -59,3 +58,8 @@ def evaluate(self, annotations, predictions):
     def reset(self):
         self.loss = 0
         self.total = 0
+
+    @classmethod
+    def get_common_meta(cls):
+        meta = super().get_common_meta()
+        meta['target'] = 'higher-worse'

tools/accuracy_checker/openvino/tools/accuracy_checker/metrics/machine_translation.py

@@ -82,6 +82,7 @@ def __call__(self, line):
         'none': None,   # No value is required
 }
 
+
 class BilingualEvaluationUnderstudy(PerImageEvaluationMetric):
     __provider__ = 'bleu'
     annotation_types = (MachineTranslationAnnotation, )