Merge pull request #2897 from openvinotoolkit/release

Wovchena · web-flow · commit 0d7aeac5eacd · 2021-11-17T17:50:57.000+03:00
Merge release into master
diff --git a/ci/documentation_updater/documentation_updater.py b/ci/documentation_updater/documentation_updater.py
@@ -113,7 +113,7 @@ def get_models_from_configs(directory):
     model_configs = directory.glob('**/model.yml')
     for model in model_configs:
         with model.open("r", encoding="utf-8") as file:
-            models[model.parent.name] = (model, yaml.load(file))
+            models[model.parent.name] = (model, yaml.safe_load(file))
         if not models[model.parent.name][1]:
             logging.error("File {} is empty. It will be ignored.".format(model))
             del models[model.parent.name]
diff --git a/demos/requirements.txt b/demos/requirements.txt
@@ -3,7 +3,7 @@ matplotlib>=3.3.4
 motmetrics>=1.2.0
 nibabel>=3.2.1
 numpy>=1.16.6,<1.20
-scikit-learn>=0.24.1
+scikit-learn~=0.24.1
 scipy~=1.5.4
 tokenizers>=0.10.1
 tensorboardX>=2.1
diff --git a/models/intel/index.md b/models/intel/index.md
@@ -299,6 +299,8 @@ Deep Learning models for speech synthesis (mel spectrogram generation and wave f
 |   text-to-speech-en-multi-0001-regression                          | 7.81  | 5.12  |
 |   text-to-speech-en-multi-0001-generation                          | 48.38 | 12.77 |
 
+## Speech Noise Suppression
+
 Deep Learning models for noise suppression.
 
 | Model Name                                                                          | Complexity (GFLOPs)  | Size (Mp)  |
diff --git a/models/public/Sphereface/model.yml b/models/public/Sphereface/model.yml
@@ -23,9 +23,7 @@ files:
   - name: Sphereface.caffemodel
     size: 90688218
     sha256: 092d8fb00291d80ed7e43f7c9323693581cd75e734be3c52764525043288a20d
-    source:
-      $type: google_drive
-      id: 0B_geeR2lTMegb2F6dmlmOXhWaVk
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/Sphereface/sphereface_model.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,112,96]
   - --input=data
diff --git a/models/public/densenet-121/model.yml b/models/public/densenet-121/model.yml
@@ -27,9 +27,7 @@ files:
   - name: densenet-121.caffemodel
     size: 32303870
     sha256: c6a6ec988d76c468c3f67501a23a39ec7bf6ebe6729fd99496a15d0e845478b2
-    source:
-      $type: google_drive
-      id: 0B7ubpZO7HnlCcHlfNmJkU2VPelE
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/densenet-121/DenseNet_121.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/densenet-161-tf/model.yml b/models/public/densenet-161-tf/model.yml
@@ -22,9 +22,7 @@ files:
   - name: tf-densenet161.tar.gz
     size: 110578586
     sha256: f4745258b9fe2f234711a70e4192368f169bf0c1956087a210b08270dbe96ca7
-    source:
-      $type: google_drive
-      id: 0B_fUSpodN0t0NmZvTnZZa2plaHc
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/densenet-161-tf/tf-densenet161.tar.gz
 postprocessing:
   - $type: unpack_archive
     format: gztar
diff --git a/models/public/densenet-161/model.yml b/models/public/densenet-161/model.yml
@@ -37,9 +37,7 @@ files:
   - name: densenet-161.caffemodel
     size: 115676075
     sha256: e124d9a8f2284f4ab160569139217f709f21be6fc132c865b6a55cb8cae7d6b5
-    source:
-      $type: google_drive
-      id: 0B7ubpZO7HnlCa0phRGJIRERoTXM
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/densenet-161/DenseNet_161.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/densenet-169/model.yml b/models/public/densenet-169/model.yml
@@ -37,9 +37,7 @@ files:
   - name: densenet-169.caffemodel
     size: 57307526
     sha256: 8d45f2ab15f6329e2f80004692c58129cc4875ffe43ddf59433ebc2216189f15
-    source:
-      $type: google_drive
-      id: 0B7ubpZO7HnlCRWVVdUJjVVAyQXc
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/densenet-169/DenseNet_169.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/densenet-201/model.yml b/models/public/densenet-201/model.yml
@@ -37,9 +37,7 @@ files:
   - name: densenet-201.caffemodel
     size: 81062969
     sha256: ba464965293d4dd5557085d57ec810bf353de362dd90e1b8293e6f0707978e4a
-    source:
-      $type: google_drive
-      id: 0B7ubpZO7HnlCV3pud2oyR3lNMWs
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/densenet-201/DenseNet_201.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/device_support.md b/models/public/device_support.md
@@ -159,4 +159,4 @@
 | yolo-v3-tf | YES | YES | YES |
 | yolo-v3-tiny-tf | YES | YES | YES |
 | yolo-v4-tf | YES | YES | YES |
-| yolo-v4-tiny-tf | YES | YES |    |
+| yolo-v4-tiny-tf | YES | YES | YES |
diff --git a/models/public/mobilenet-ssd/model.yml b/models/public/mobilenet-ssd/model.yml
@@ -33,9 +33,7 @@ files:
   - name: mobilenet-ssd.caffemodel
     size: 23147564
     sha256: 761c86fbae3d8361dd454f7c740a964f62975ed32f4324b8b85994edec30f6af
-    source:
-      $type: google_drive
-      id: 0B3gersZ2cHIxRm5PMWRoTkdHdHc
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/mobilenet-ssd/MobileNetSSD_deploy.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,300,300]
   - --input=data
diff --git a/models/public/retinanet-tf/README.md b/models/public/retinanet-tf/README.md
@@ -8,25 +8,26 @@ converted to TensorFlow\* protobuf format. For details, see [paper](https://arxi
 
 ### Steps to Reproduce Keras\* to TensorFlow\* Conversion
 
-1. Clone the original [repository](https://github.com/fizyr/keras-retinanet)(tested on `47fdf189` commit)
+1. Clone and install the original [repository](https://github.com/fizyr/keras-retinanet) (tested on `47fdf189` commit) following the [installation steps](https://github.com/fizyr/keras-retinanet#installation). Make sure you ran `python setup.py build_ext --inplace` from the root of repository.
 2. Download the original model from [here](https://github.com/fizyr/keras-retinanet/releases/download/0.5.1/resnet50_coco_best_v2.1.0.h5)
 3. Get conversion script:
    1. Get conversion script from [repository](https://github.com/amir-abdi/keras_to_tensorflow):
    ```sh
    git clone https://github.com/amir-abdi/keras_to_tensorflow.git
    ```
-   2. (Optional) Checkout the commit that the conversion was tested on:
+   2. Checkout the commit that the conversion was tested on:
    ```
    git checkout c841508a88faa5aa1ffc7a4947c3809ea4ec1228
    ```
-   3. Apply `keras_to_tensorflow.patch`:
+   3. Apply `keras_to_tensorflow.patch` from `<omz_dir>/models/public/retinanet-tf`:
    ```
    git apply keras_to_tensorflow.patch
    ```
    4. Run script:
    ```
    python keras_to_tensorflow.py --input_model=<model_in>.h5 --output_model=<model_out>.pb
    ```
+> **NOTE**: The conversion was tested with keras==2.3.1, tensorflow==1.15 and h5py<3.0.0.
 
 ## Specification
 
diff --git a/models/public/se-inception/model.yml b/models/public/se-inception/model.yml
@@ -23,9 +23,7 @@ files:
   - name: se-inception.caffemodel
     size: 47855246
     sha256: 0660aa2b867f7353794d3ac02081d9178ac0709e8d8d75c29290dba4bc10ce03
-    source:
-      $type: google_drive
-      id: 0BwHV3BlNKkWlTWRRbDZYbVB2WWc
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/se-inception/SE-BN-Inception.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/se-resnet-101/model.yml b/models/public/se-resnet-101/model.yml
@@ -23,9 +23,7 @@ files:
   - name: se-resnet-101.caffemodel
     size: 197806200
     sha256: 09777ce1202087954ee34ae2566cf611d03d2d8195ced0fe51b612ebf951d729
-    source:
-      $type: google_drive
-      id: 0BwHV3BlNKkWlTEg4YmcwQ0FoZFU
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/se-resnet-101/SE-ResNet-101.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/se-resnet-152/model.yml b/models/public/se-resnet-152/model.yml
@@ -23,9 +23,7 @@ files:
   - name: se-resnet-152.caffemodel
     size: 268009578
     sha256: bbe7fc5b5068b7cd4b12ce1bb78f61bd3f48238a372e70ab9c0724afa778e5d1
-    source:
-      $type: google_drive
-      id: 0BwHV3BlNKkWlcFE0Q2NTcWl3WUE
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/se-resnet-152/SE-ResNet-152.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/se-resnet-50/model.yml b/models/public/se-resnet-50/model.yml
@@ -23,9 +23,7 @@ files:
   - name: se-resnet-50.caffemodel
     size: 112602669
     sha256: 72880fde3bd29b324bb302c80f49a098b8970597494c56525fd410a8d2ed5993
-    source:
-      $type: google_drive
-      id: 0BwHV3BlNKkWlS2QwZHFzM3RjNzg
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/se-resnet-50/SE-ResNet-50.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/se-resnext-101/model.yml b/models/public/se-resnext-101/model.yml
@@ -23,9 +23,7 @@ files:
   - name: se-resnext-101.caffemodel
     size: 196447403
     sha256: 78dc95c9b768fdb32a0a3981a686edaf655df82973b051d818bae523c9a93c90
-    source:
-      $type: google_drive
-      id: 0BwHV3BlNKkWleklsNzBiZlprblk
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/se-resnext-101/SE-ResNeXt-101.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/se-resnext-50/model.yml b/models/public/se-resnext-50/model.yml
@@ -23,9 +23,7 @@ files:
   - name: se-resnext-50.caffemodel
     size: 110550637
     sha256: f12c6b53c2dcd0f38f388d3d5f98cbfc039357e84842e661f28a49719dd52219
-    source:
-      $type: google_drive
-      id: 0BwHV3BlNKkWlQ2Z0Q204V1RITjA
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/se-resnext-50/SE-ResNeXt-50.caffemodel
 model_optimizer_args:
   - --input_shape=[1,3,224,224]
   - --input=data
diff --git a/models/public/ssd300/model.yml b/models/public/ssd300/model.yml
@@ -38,9 +38,7 @@ files:
   - name: ssd300.tar.gz
     size: 97789219
     sha256: e3eb9794a33eb77e6798f396df453123c249a1df554e42a3302eb4aa20a8f2ee
-    source:
-      $type: google_drive
-      id: 0BzKzrI_SkD1_TkFPTEQ1Z091SUE
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/ssd300/models_VGGNet_VOC0712Plus_SSD_300x300_ft.tar.gz
 postprocessing:
   - $type: unpack_archive
     format: gztar
diff --git a/models/public/ssd512/model.yml b/models/public/ssd512/model.yml
@@ -38,9 +38,7 @@ files:
   - name: ssd512.tar.gz
     size: 100991061
     sha256: 2d98a6ddd5b4cb0ba59a37fe5d50020457cfe104d45873c3ea43d1e6dcddd12d
-    source:
-      $type: google_drive
-      id: 0BzKzrI_SkD1_MjFjNTlnempHNWs
+    source: https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2021.4/ssd512/models_VGGNet_VOC0712Plus_SSD_512x512.tar.gz
 postprocessing:
   - $type: unpack_archive
     format: gztar
diff --git a/tools/accuracy_checker/accuracy_checker/evaluators/model_evaluator.py b/tools/accuracy_checker/accuracy_checker/evaluators/model_evaluator.py
@@ -415,9 +415,9 @@ def _fill_free_irs(self, free_irs, queued_irs, infer_requests_pool, dataset_iter
                 break
 
             batch_input, batch_meta = self._get_batch_input(batch_annotation, batch_input)
+            queued_irs.append(ir_id)
             self.launcher.predict_async(infer_requests_pool[ir_id], batch_input, batch_meta,
                                         context=(batch_id, batch_input_ids, batch_annotation))
-            queued_irs.append(ir_id)
 
         return free_irs, queued_irs
 
diff --git a/tools/accuracy_checker/accuracy_checker/evaluators/quantization_model_evaluator.py b/tools/accuracy_checker/accuracy_checker/evaluators/quantization_model_evaluator.py
@@ -327,9 +327,9 @@ def _fill_free_irs(self, free_irs, queued_irs, infer_requests_pool, dataset_iter
                 break
 
             batch_input, batch_meta = self._get_batch_input(batch_inputs, batch_annotation)
+            queued_irs.append(ir_id)
             self.launcher.predict_async(infer_requests_pool[ir_id], batch_input, batch_meta,
                                         context=(batch_id, batch_input_ids, batch_annotation, batch_identifiers))
-            queued_irs.append(ir_id)
 
         return free_irs, queued_irs
 
diff --git a/tools/accuracy_checker/accuracy_checker/metrics/coco_metrics.py b/tools/accuracy_checker/accuracy_checker/metrics/coco_metrics.py
@@ -15,6 +15,7 @@
 """
 
 from functools import singledispatch
+from collections import defaultdict
 import numpy as np
 from ..config import NumberField, BaseField, ConfigError
 from ..representation import (
@@ -28,7 +29,7 @@
 from ..utils import get_or_parse_value, finalize_metric_result, UnsupportedPackage
 from .overlap import Overlap
 from .metric import PerImageEvaluationMetric
-from .detection import average_precision, APIntegralType
+from .detection import average_precision, APIntegralType, _prepare_prediction_boxes
 
 try:
     import pycocotools.mask as maskUtils
@@ -69,6 +70,9 @@ def configure(self):
         self.max_detections = self.get_value_from_config('max_detections')
         threshold = process_threshold(self.get_value_from_config('threshold'))
         self.thresholds = get_or_parse_value(threshold, COCO_THRESHOLDS)
+        if not self.dataset:
+            raise ConfigError('coco metrics require dataset metadata providing in dataset_meta'
+                              'Please provide dataset meta file or regenerate annotation')
         if not self.dataset.metadata:
             raise ConfigError('coco metrics require dataset metadata providing in dataset_meta'
                               'Please provide dataset meta file or regenerate annotation')
@@ -112,6 +116,33 @@ def reset(self):
         if self.profiler:
             self.profiler.reset()
 
+    def _update_label_stat_for_non_matched_classes(self, labels_stat, predictions):
+        matched_classes = set(labels_stat)
+        background = self.dataset.metadata.get('background_label')
+        prediction_classes = np.unique([pred.labels for pred in predictions])
+        for pc in prediction_classes:
+            if pc == background or pc in matched_classes:
+                continue
+            prediction_boxes, _, _ = _prepare_prediction_boxes(
+                pc, predictions, True
+            )
+            conf = prediction_boxes[:, 0]
+            label_report = {
+                'precision': [],
+                'recall': [],
+                'result': [],
+                'ap': -1,
+                'scores': conf,
+                'matched': defaultdict(list),
+                'gt': [],
+                'dt': prediction_boxes[:, 1:],
+                'prediction_matches': 0,
+                'annotation_matches': 0,
+                'iou': []
+            }
+            labels_stat[int(pc)] = label_report
+        return labels_stat
+
 
 class MSCOCOAveragePrecision(MSCOCOBaseMetric):
     __provider__ = 'coco_precision'
@@ -123,8 +154,9 @@ def update(self, annotation, prediction):
         ]
         precisions = [result[0] for result in per_class_result]
         if self.profiler:
-            for class_match, (precision, recall, all_precisions, all_recalls, _, _, _) in zip(
-                    per_class_matching, per_class_result
+            per_class_matching_dict = {}
+            for class_match, label, (precision, recall, all_precisions, all_recalls, _, _, _) in zip(
+                    per_class_matching, self.labels, per_class_result
             ):
                 class_match['result'] = precision
                 class_match['precision'] = precision
@@ -133,7 +165,12 @@ def update(self, annotation, prediction):
                 max_recall = all_recalls[0] if np.size(all_recalls) else np.array([])
                 ap = average_precision(max_precision, max_recall, APIntegralType.voc_max)
                 class_match['ap'] = ap
-            self.profiler.update(annotation.identifier, per_class_matching, self.name, np.nanmean(precisions))
+                per_class_matching_dict[label] = class_match
+
+            per_class_matching_dict = self._update_label_stat_for_non_matched_classes(
+                per_class_matching_dict, [prediction]
+            )
+            self.profiler.update(annotation.identifier, per_class_matching_dict, self.name, np.nanmean(precisions))
         return precisions
 
     def evaluate(self, annotations, predictions):
@@ -171,6 +208,8 @@ def prepare_summary(label_stats):
             recall_v.append(recall_v_)
             precision.append(precisions[0] if np.size(precisions) else [])
             precision_v.append(precision_v_)
+            if not np.size(precisions):
+                continue
             fppi = 1 - precisions[0]
             mr = 1 - recalls[0]
             pr = np.array([precisions[0], recalls[0]]).T
@@ -246,15 +285,17 @@ def update(self, annotation, prediction):
         ]
         recalls = [metric[1] for metric in per_class_result]
         if self.profiler:
-            for class_match, (precision, recall, all_precisions, all_recalls, _, _, _) in zip(
-                    per_class_matching, per_class_result
+            per_class_matching_dict = {}
+            for class_match, label, (precision, recall, all_precisions, all_recalls, _, _, _) in zip(
+                    per_class_matching, self.labels, per_class_result
             ):
                 class_match['result'] = recall
                 class_match['precision'] = precision
                 class_match['recall'] = recall
                 ap = average_precision(all_precisions, all_recalls, APIntegralType.voc_max)
                 class_match['ap'] = ap
-            self.profiler.update(annotation.identifier, per_class_matching, self.name, np.nanmean(recalls))
+                per_class_matching_dict[label] = class_match
+            self.profiler.update(annotation.identifier, per_class_matching_dict, self.name, np.nanmean(recalls))
         return per_class_result
 
     def evaluate(self, annotations, predictions):
diff --git a/tools/accuracy_checker/accuracy_checker/metrics/coco_orig_metrics.py b/tools/accuracy_checker/accuracy_checker/metrics/coco_orig_metrics.py
diff --git a/tools/accuracy_checker/accuracy_checker/metrics/detection.py b/tools/accuracy_checker/accuracy_checker/metrics/detection.py
diff --git a/tools/accuracy_checker/accuracy_checker/metrics/metric_profiler/base_profiler.py b/tools/accuracy_checker/accuracy_checker/metrics/metric_profiler/base_profiler.py
diff --git a/tools/accuracy_checker/accuracy_checker/metrics/metric_profiler/object_detection_metric_profiler.py b/tools/accuracy_checker/accuracy_checker/metrics/metric_profiler/object_detection_metric_profiler.py
diff --git a/tools/accuracy_checker/accuracy_checker/metrics/metric_profiler/profiling_executor.py b/tools/accuracy_checker/accuracy_checker/metrics/metric_profiler/profiling_executor.py
diff --git a/tools/accuracy_checker/accuracy_checker/presenters.py b/tools/accuracy_checker/accuracy_checker/presenters.py
diff --git a/tools/accuracy_checker/requirements.in b/tools/accuracy_checker/requirements.in
diff --git a/tools/accuracy_checker/tests/test_config_reader.py b/tools/accuracy_checker/tests/test_config_reader.py
