AC: update profiling (#2894)

Author: eaidova

tools/accuracy_checker/accuracy_checker/evaluators/model_evaluator.py

@@ -366,11 +366,16 @@ def _process_batch_results(
             self.adapter.output_blob = self.adapter.output_blob or self.launcher.output_blob
             batch_predictions = self.adapter.process(batch_predictions, batch_identifiers, batch_meta)
 
+        copy_annotations, copy_predictions = None, None
+        if self.metric_executor.profiler is not None and self.metric_executor.profiler.required_postprocessing:
+            copy_annotations, copy_predictions = copy.deepcopy(batch_annotations), copy.deepcopy(batch_predictions)
+            copy_annotations, copy_predictions = self.postprocessor.deprocess_batch(
+                copy_annotations, copy_predictions, batch_meta)
         annotations, predictions = self.postprocessor.process_batch(
             batch_annotations, batch_predictions, batch_meta
         )
         _, profile_result = self.metric_executor.update_metrics_on_batch(
-            batch_input_ids, annotations, predictions, enable_profiling
+            batch_input_ids, annotations, predictions, enable_profiling, copy_annotations, copy_predictions
         )
         if output_callback:
             callback_kwargs = {'profiling_result': profile_result} if enable_profiling else {}

tools/accuracy_checker/accuracy_checker/metrics/coco_metrics.py

@@ -69,6 +69,7 @@ def parameters(cls):
     def configure(self):
         self.max_detections = self.get_value_from_config('max_detections')
         threshold = process_threshold(self.get_value_from_config('threshold'))
+        self.config.pop('threshold', None)
         self.thresholds = get_or_parse_value(threshold, COCO_THRESHOLDS)
         if not self.dataset:
             raise ConfigError('coco metrics require dataset metadata providing in dataset_meta'
@@ -120,13 +121,19 @@ def _update_label_stat_for_non_matched_classes(self, labels_stat, predictions):
         matched_classes = set(labels_stat)
         background = self.dataset.metadata.get('background_label')
         prediction_classes = np.unique([pred.labels for pred in predictions])
+        get_polygon = hasattr(predictions[0], 'to_polygon')
         for pc in prediction_classes:
             if pc == background or pc in matched_classes:
                 continue
-            prediction_boxes, _, _ = _prepare_prediction_boxes(
-                pc, predictions, True
-            )
-            conf = prediction_boxes[:, 0]
+            if not get_polygon:
+                prediction_boxes, _, _ = _prepare_prediction_boxes(
+                    pc, predictions, True
+                )
+                conf = prediction_boxes[:, 0] if not get_polygon else []
+                prediction_boxes = prediction_boxes[:, 1:]
+            else:
+                prediction_boxes = [p.to_polygon().get(pc, []) for p in predictions][0]
+                conf = [p.scores[p.labels == pc] for p in predictions][0]
             label_report = {
                 'precision': [],
                 'recall': [],
@@ -135,7 +142,7 @@ def _update_label_stat_for_non_matched_classes(self, labels_stat, predictions):
                 'scores': conf,
                 'matched': defaultdict(list),
                 'gt': [],
-                'dt': prediction_boxes[:, 1:],
+                'dt': prediction_boxes,
                 'prediction_matches': 0,
                 'annotation_matches': 0,
                 'iou': []
@@ -413,8 +420,8 @@ def evaluate(self, annotations, predictions):
         return recalls
 
 
-class MSCOCOSegmAveragePrecision(MSCOCOAveragePrecision):
-    __provider__ = 'coco_segm_precision'
+class MSCOCOSegmBase(MSCOCOBaseMetric):
+    __provider__ = 'coco_segm'
 
     annotation_types = (CoCoInstanceSegmentationAnnotation,)
     prediction_types = (CoCoInstanceSegmentationPrediction,)
@@ -424,17 +431,37 @@ def configure(self):
         if isinstance(maskUtils, UnsupportedPackage):
             maskUtils.raise_error(self.__provider__)
 
+    def update(self, annotation, prediction):
+        compute_iou, create_boxes = select_specific_parameters(annotation)
+        per_class_results = []
+        profile_boxes = self.profiler is not None
+        if profile_boxes:
+            annotation_polygons = annotation.to_polygon()
+            prediction_polygons = prediction.to_polygon()
 
-class MSCOCOSegmRecall(MSCOCORecall):
-    __provider__ = 'coco_segm_recall'
+        for label_id, label in enumerate(self.labels):
 
-    annotation_types = (CoCoInstanceSegmentationAnnotation,)
-    prediction_types = (CoCoInstanceSegmentationPrediction,)
+            detections, scores, dt_difficult = prepare_predictions(prediction, label, self.max_detections)
+            ground_truth, gt_difficult, iscrowd, boxes, areas = prepare_annotations(annotation, label, create_boxes)
+            iou = compute_iou(ground_truth, detections, annotation_boxes=boxes, annotation_areas=areas, iscrowd=iscrowd)
+            eval_result = evaluate_image(
+                ground_truth if not profile_boxes else annotation_polygons.get(label, []),
+                gt_difficult, iscrowd, detections if not profile_boxes else prediction_polygons.get(label, []),
+                dt_difficult, scores, iou, self.thresholds,
+                profile_boxes
+            )
+            self.matching_results[label_id].append(eval_result)
+            per_class_results.append(eval_result)
 
-    def configure(self):
-        super().configure()
-        if isinstance(maskUtils, UnsupportedPackage):
-            maskUtils.raise_error(self.__provider__)
+        return per_class_results
+
+
+class MSCOCOSegmAveragePrecision(MSCOCOAveragePrecision, MSCOCOSegmBase):
+    __provider__ = 'coco_segm_precision'
+
+
+class MSCOCOSegmRecall(MSCOCORecall, MSCOCOSegmBase):
+    __provider__ = 'coco_segm_recall'
 
 
 @singledispatch

tools/accuracy_checker/accuracy_checker/metrics/coco_orig_metrics.py

@@ -34,8 +34,8 @@
 from ..logging import print_info
 from ..config import BaseField, BoolField, ConfigError
 from ..utils import get_or_parse_value, UnsupportedPackage
-from .metric import FullDatasetEvaluationMetric, PerImageEvaluationMetric, Metric
-from .coco_metrics import COCO_THRESHOLDS, process_threshold, compute_precision_recall
+from .metric import FullDatasetEvaluationMetric, Metric, PerImageEvaluationMetric
+from .coco_metrics import COCO_THRESHOLDS, process_threshold
 
 try:
     from pycocotools.coco import COCO
@@ -79,6 +79,7 @@ def parameters(cls):
 
     def configure(self):
         threshold = process_threshold(self.get_value_from_config('threshold'))
+        self.config.pop('threshold', None)
         self.threshold = get_or_parse_value(threshold, COCO_THRESHOLDS)
         self.box_side_delta = int(self.get_value_from_config('include_boundaries'))
         if not self.dataset.metadata:
@@ -99,7 +100,7 @@ def configure(self):
     def set_profiler(self, profiler):
         self.profiler = profiler
         self.profiling_helper = Metric.provide(
-            self.__provider__.replace('_orig', ''), {}, self.dataset, self.name, profiler=profiler
+            self.__provider__.replace('_orig', ''), {'max_detections': 100}, self.dataset, self.name, profiler=profiler
         )
 
     def update(self, annotation, prediction):
@@ -472,7 +473,7 @@ def evaluate(self, annotations, predictions):
         pass
 
 
-class MSCOCOorigAveragePrecision(MSCOCOorigBaseMetric):
+class MSCOCOorigAveragePrecision(MSCOCOorigBaseMetric, PerImageEvaluationMetric):
     __provider__ = 'coco_orig_precision'
 
     def evaluate(self, annotations, predictions):
@@ -490,34 +491,12 @@ class MSCOCOOrigSegmAveragePrecision(MSCOCOorigAveragePrecision, PerImageEvaluat
 
     def update(self, annotation, prediction):
         if self.profiler:
-            per_class_matching = {}
-            for _, label in enumerate(self.labels):
-                detections, scores, dt_difficult = self._prepare_predictions(prediction, label)
-                ground_truth, gt_difficult, iscrowd = self._prepare_annotations(annotation, label)
-                if not ground_truth.size:
-                    continue
-                iou = self._compute_iou(ground_truth, detections, iscrowd)
-                eval_result = self._evaluate_image(
-                    ground_truth, gt_difficult, iscrowd, detections, dt_difficult, scores, iou, self.threshold,
-                    True
-                )
-                eval_result['gt'] = annotation.to_polygon()[label]
-                eval_result['dt'] = annotation.to_polygon()[label]
-                per_class_matching[label] = eval_result
-            per_class_result = {k: compute_precision_recall(
-                self.threshold, [v])[0] for k, v in per_class_matching.items()
-                                }
-            for label in per_class_matching:
-                per_class_matching[label]['result'] = per_class_result[label]
-            self.profiler.update(
-                annotation.identifier, per_class_matching, self.name, np.nanmean(list(per_class_result.values()))
-            )
+            self.profiling_helper.update(annotation, prediction)
 
     @staticmethod
     def _compute_iou(gt, dets, iscrowd):
         return iou_calc(list(dets), list(gt), iscrowd)
 
-
     @staticmethod
     def _prepare_predictions(prediction, label):
         if prediction.size == 0:
@@ -552,7 +531,6 @@ def _prepare_annotations(annotation, label):
 
         return ann[order], difficult_label[order], iscrowd_label[order]
 
-
     @staticmethod
     def _iou_type_data_to_coco(data_to_store, data, box_side_delta):
         encoded_masks = data.mask

tools/accuracy_checker/accuracy_checker/metrics/detection.py

@@ -302,18 +302,18 @@ def _calculate_map(self, annotations, predictions, profile_boxes=False, return_l
         labels_stat = self.per_class_detection_statistics(annotations, predictions, valid_labels, profile_boxes)
 
         average_precisions = []
-        for label in labels_stat:
-            label_precision = labels_stat[label]['precision']
-            label_recall = labels_stat[label]['recall']
+        for value in labels_stat.values():
+            label_precision = value['precision']
+            label_recall = value['recall']
             label_miss_rate = 1 - label_recall
-            labels_stat[label]['miss_rate'] = label_miss_rate
+            value['miss_rate'] = label_miss_rate
             if label_recall.size:
                 ap = average_precision(label_precision, label_recall, self.integral)
                 average_precisions.append(ap)
             else:
                 average_precisions.append(np.nan)
-            labels_stat[label]['ap'] = average_precisions[-1]
-            labels_stat[label]['result'] = average_precisions[-1]
+            value['ap'] = average_precisions[-1]
+            value['result'] = average_precisions[-1]
         if profile_boxes:
             self._update_label_stat_for_non_matched_classes(labels_stat, predictions)
             self.profiler.update(annotations[0].identifier, labels_stat, self.name, np.nanmean(average_precisions))
@@ -376,16 +376,16 @@ def update(self, annotation, prediction):
             [annotation], [prediction], valid_labels, self.profiler is not None
         )
         miss_rates = []
-        for label in labels_stat:
-            label_miss_rate = 1.0 - labels_stat[label]['recall']
-            label_fppi = labels_stat[label]['fppi']
+        for value in labels_stat.values():
+            label_miss_rate = 1.0 - value['recall']
+            label_fppi = value['fppi']
 
             position = bisect.bisect_left(label_fppi, self.fppi_level)
             m0 = max(0, position - 1)
             m1 = position if position < len(label_miss_rate) else m0
             miss_rates.append(0.5 * (label_miss_rate[m0] + label_miss_rate[m1]))
             if self.profiler:
-                labels_stat[label]['result'] = miss_rates[-1]
+                value['result'] = miss_rates[-1]
         if self.profiler:
             self.profiler.update(annotation[0].identifier, labels_stat, self.name, np.nanmean(miss_rates))
 
@@ -397,9 +397,9 @@ def evaluate(self, annotations, predictions):
         labels_stat = self.per_class_detection_statistics(annotations, predictions, valid_labels)
 
         miss_rates = []
-        for label in labels_stat:
-            label_miss_rate = 1.0 - labels_stat[label]['recall']
-            label_fppi = labels_stat[label]['fppi']
+        for value in labels_stat.values():
+            label_miss_rate = 1.0 - value['recall']
+            label_fppi = value['fppi']
 
             position = bisect.bisect_left(label_fppi, self.fppi_level)
             m0 = max(0, position - 1)
@@ -437,15 +437,15 @@ def _calculate_recall(self, annotations, predictions, profile_boxes=False):
         labels_stat = self.per_class_detection_statistics(annotations, predictions, valid_labels, profile_boxes)
 
         recalls = []
-        for label in labels_stat:
-            label_recall = labels_stat[label]['recall']
+        for value in labels_stat.values():
+            label_recall = value['recall']
             if label_recall.size:
                 max_recall = label_recall[-1]
                 recalls.append(max_recall)
             else:
                 recalls.append(np.nan)
             if profile_boxes:
-                labels_stat[label]['result'] = recalls[-1]
+                value['result'] = recalls[-1]
         if profile_boxes:
             self.profiler.update(annotations[0].identifier, labels_stat, self.name, np.nanmean(recalls))
 

tools/accuracy_checker/accuracy_checker/metrics/metric_executor.py

@@ -82,7 +82,8 @@ def update_metrics_on_object(self, annotation, prediction):
 
         return metric_results
 
-    def update_metrics_on_batch(self, batch_ids, annotation, prediction, profile=False):
+    def update_metrics_on_batch(self, batch_ids, annotation, prediction,
+                                profile=False, deprocessed_annotation=None, deprocessed_prediction=None):
         """
         Updates metric value corresponding given batch.
 
@@ -94,7 +95,11 @@ def update_metrics_on_batch(self, batch_ids, annotation, prediction, profile=Fal
         results = OrderedDict()
         profile_results = OrderedDict()
 
-        for input_id, single_annotation, single_prediction in zip(batch_ids, annotation, prediction):
+        for idx, (input_id, single_annotation, single_prediction) in enumerate(zip(batch_ids, annotation, prediction)):
+            if profile:
+                if deprocessed_annotation is not None and deprocessed_prediction is not None:
+                    self.profiler.update_annotation_and_prediction(
+                        deprocessed_annotation[idx], deprocessed_prediction[idx])
             results[input_id] = self.update_metrics_on_object(single_annotation, single_prediction)
             if profile:
                 profile_results[input_id] = self.profiler.get_last_report()
@@ -115,6 +120,20 @@ def iterate_metrics(self, annotations, predictions):
                 profiling_file=profiling_file
             )
 
+    def get_metric_result_template(self, ignore_refs):
+        for name, metric_type, functor, reference, abs_threshold, rel_threshold, presenter in self.metrics:
+            profiling_file = None if functor.profiler is None else functor.profiler.report_file
+            yield presenter, EvaluationResult(
+                name=name,
+                metric_type=metric_type,
+                evaluated_value=functor.result_template,
+                reference_value=reference if not ignore_refs else None,
+                abs_threshold=abs_threshold,
+                rel_threshold=rel_threshold,
+                meta=functor.meta,
+                profiling_file=profiling_file
+            )
+
     def register_metric(self, metric_config_entry):
         type_ = 'type'
         identifier = 'name'
@@ -150,9 +169,9 @@ def register_metric(self, metric_config_entry):
             metric_type, metric_config_entry, self.dataset, metric_identifier, state=self.state, **metric_kwargs
         )
         metric_presenter = BasePresenter.provide(metric_config_entry.get(presenter, 'print_scalar'))
-        threshold_v = metric_config_entry.get(threshold)
-        abs_threshold_v = metric_config_entry.get(abs_threshold)
-        reference_v = metric_config_entry.get(reference)
+        threshold_v = metric_fn.config.get(threshold)
+        abs_threshold_v = metric_fn.config.get(abs_threshold)
+        reference_v = metric_fn.config.get(reference)
         if reference_v is not None and not isinstance(reference_v, (int, float, dict)):
             raise ConfigError(
                 'reference value should be represented as number or dictionary with numbers for each submetric'

tools/accuracy_checker/accuracy_checker/metrics/metric_profiler/base_profiler.py

@@ -69,6 +69,7 @@ def __init__(self, dump_iterations=100, report_type='csv', name=None):
         self.storage = OrderedDict()
         self.write_result = self.write_csv_result if report_type == 'csv' else self.write_json_result
         self._last_profile = None
+        self.required_postprocessing = False
 
     def register_metric(self, metric_name):
         self.fields.append('{}_result'.format(metric_name))
@@ -106,7 +107,7 @@ def write_summary(self, summary):
             out_path = self.out_dir / self.report_file
             new_file = not out_path.exists()
             if not new_file:
-                with open(str(out_path), 'r') as f:
+                with open(str(out_path), 'r', encoding='utf-8') as f:
                     out_dict = json.load(f)
             else:
                 out_dict = {
@@ -122,7 +123,7 @@ def write_summary(self, summary):
                     'dataset_meta': self.dataset_meta,
                 }
             out_dict.update(summary)
-            with open(str(out_path), 'w') as f:
+            with open(str(out_path), 'w', encoding='utf-8') as f:
                 json.dump(out_dict, f)
 
     def reset(self):
@@ -142,7 +143,7 @@ def write_csv_result(self):
             else:
                 data_to_store.append(value)
 
-        with open(str(out_path), 'a+', newline='') as f:
+        with open(str(out_path), 'a+', newline='', encoding='utf-8') as f:
             writer = DictWriter(f, fieldnames=self.fields)
             if new_file:
                 writer.writeheader()
@@ -154,7 +155,7 @@ def write_json_result(self):
         out_path = self.out_dir / self.report_file
         new_file = not out_path.exists()
         if not new_file:
-            with open(str(out_path), 'r') as f:
+            with open(str(out_path), 'r', encoding='utf-8') as f:
                 out_dict = json.load(f)
             out_dict['report'].extend(list(self.storage.values()))
         else:
@@ -170,7 +171,7 @@ def write_json_result(self):
                 'report_type': self.__provider__,
                 'dataset_meta': self.dataset_meta
             }
-        with open(str(out_path), 'w') as f:
+        with open(str(out_path), 'w', encoding='utf-8') as f:
             json.dump(out_dict, f)
 
         self._reset_storage()
@@ -193,6 +194,9 @@ def set_dataset_meta(self, meta):
     def last_report(self):
         return self._last_profile
 
+    def update_annotation_and_prediction(self, annotation, prediction):
+        pass
+
 
 def create_profiler(metric_type, metric_name):
     profiler = None