AC: support openvino 2.0 launcher (#2898)

Author: eaidova

tools/accuracy_checker/.pylintrc

@@ -22,7 +22,7 @@ disable = C0103,
 max-line-length = 120
 ignore-docstrings = yes
 extension-pkg-whitelist=inference_engine,cv2,numpy,mxnet,tensorflow,pycocotools,onnxruntime,kenlm,paddle,torchvision
-ignored-modules = numpy,cv2,openvino.inference_engine,caffe,mxnet,tensorflow,pycocotools,onnxruntime,torch,kenlm,paddle.fluid.core,torchvision
+ignored-modules = numpy,cv2,openvino.inference_engine,caffe,mxnet,tensorflow,pycocotools,onnxruntime,torch,kenlm,paddle.fluid.core,torchvision,openvino.pyopenvino,openvino.ie_api,openvino.impl
 load-plugins = pylint_checkers
 ignored-classes = pathlib.PurePath
 jobs=0

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/__init__.py

@@ -51,7 +51,6 @@
 from .reidentification import ReidAdapter
 from .detection import (
     TFObjectDetectionAPIAdapter,
-    MTCNNPAdapter,
     ClassAgnosticDetectionAdapter,
     FaceBoxesAdapter,
     FaceDetectionAdapter,
@@ -62,6 +61,7 @@
     UltraLightweightFaceDetectionAdapter,
     PPDetectionAdapter
 )
+from .mtcnn import MTCNNPAdapter
 from .detection_person_vehicle import (
     PersonVehicleDetectionAdapter,
     PersonVehicleDetectionRefinementAdapter
@@ -133,7 +133,6 @@
     'ClassificationAdapter',
 
     'TFObjectDetectionAPIAdapter',
-    'MTCNNPAdapter',
     'CTDETAdapter',
     'RetinaNetAdapter',
     'RetinaNetTF2',
@@ -152,6 +151,7 @@
     'UltraLightweightFaceDetectionAdapter',
     'PPDetectionAdapter',
     'FacialLandmarksAdapter',
+    'MTCNNPAdapter',
 
     'TinyYOLOv1Adapter',
     'YoloV2Adapter',

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/action_recognition.py

@@ -297,8 +297,18 @@ def find_layer(regex, output_name, all_outputs):
 
         self.loc_out = find_layer(loc_out_regex, 'loc', raw_outputs)
         self.main_conf_out = find_layer(main_conf_out_regex, 'main confidence', raw_outputs)
+        self.outputs_verified = True
+        if contains_all(raw_outputs, self.add_conf_outs):
+            return
+        add_conf_result = [layer_name + '/sink_port_0' for layer_name in self.add_conf_outs]
+        if contains_all(raw_outputs, add_conf_result):
+            self.add_conf_outs = add_conf_result
+            return
         add_conf_with_bias = [layer_name + '/add_' for layer_name in self.add_conf_outs]
-        if not contains_all(raw_outputs, self.add_conf_outs) and contains_all(raw_outputs, add_conf_with_bias):
+        if contains_all(raw_outputs, add_conf_with_bias):
             self.add_conf_outs = add_conf_with_bias
-
-        self.outputs_verified = True
+            return
+        add_conf_with_bias_result = [layer_name + '/add_/sink_port_0' for layer_name in self.add_conf_outs]
+        if contains_all(raw_outputs, add_conf_with_bias_result):
+            self.add_conf_outs = add_conf_with_bias_result
+        return

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/adapter.py

@@ -51,6 +51,19 @@ def process(self, raw, identifiers, frame_meta):
     def configure(self):
         pass
 
+    @staticmethod
+    def check_output_name(output_name, outputs, suffix='/sink_port_0'):
+        outputs = outputs[0] if isinstance(outputs, list) else outputs
+        if output_name in outputs:
+            return output_name
+        if suffix in output_name:
+            preprocessed_output_name = output_name.replace(suffix, '')
+        else:
+            preprocessed_output_name = '{}{}'.format(output_name, suffix)
+        if preprocessed_output_name in outputs:
+            return preprocessed_output_name
+        return output_name
+
     @classmethod
     def validate_config(cls, config, fetch_only=False, uri_prefix='', **kwargs):
         if cls.__name__ == Adapter.__name__:

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/attribute_classification.py

@@ -42,13 +42,21 @@ def parameters(cls):
     def configure(self):
         super().configure()
         self.output_layers = self.get_value_from_config('output_layer_map')
+        self.outputs_verified = False
 
     @classmethod
     def validate_config(cls, config, fetch_only=False, **kwargs):
         return super().validate_config(
             config, fetch_only=fetch_only, on_extra_argument=ConfigValidator.ERROR_ON_EXTRA_ARGUMENT
         )
 
+    def select_output_blob(self, outputs):
+        new_output_layers = {}
+        for attr, layer_name in self.output_layers.items():
+            new_output_layers[attr] = self.check_output_name(layer_name, outputs)
+        self.output_layers = new_output_layers
+        self.outputs_verified = True
+
     def process(self, raw, identifiers, frame_meta):
         """
         Args:
@@ -61,6 +69,8 @@ def process(self, raw, identifiers, frame_meta):
         result = []
         if isinstance(raw, dict):
             raw = [raw]
+        if not self.outputs_verified:
+            self.select_output_blob(raw)
         for identifier, raw_output in zip(identifiers, raw):
             container_dict = {}
             for layer_name, attribute in self.output_layers.items():

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/attributes_recognition.py

@@ -59,6 +59,7 @@ def configure(self):
         self.angle_yaw = self.get_value_from_config('angle_yaw')
         self.angle_pitch = self.get_value_from_config('angle_pitch')
         self.angle_roll = self.get_value_from_config('angle_roll')
+        self.outputs_verified = False
 
     def process(self, raw, identifiers, frame_meta):
         """
@@ -71,6 +72,8 @@ def process(self, raw, identifiers, frame_meta):
         """
         result = []
         raw_output = self._extract_predictions(raw, frame_meta)
+        if not self.outputs_verified:
+            self.select_output_blob(raw_output)
         for identifier, yaw, pitch, roll in zip(
                 identifiers,
                 raw_output[self.angle_yaw],
@@ -86,6 +89,12 @@ def process(self, raw, identifiers, frame_meta):
 
         return result
 
+    def select_output_blob(self, outputs):
+        self.check_output_name(self.angle_yaw, outputs)
+        self.check_output_name(self.angle_pitch, outputs)
+        self.check_output_name(self.angle_roll, outputs)
+        self.outputs_verified = True
+
 
 class VehicleAttributesRecognitionAdapter(Adapter):
     __provider__ = 'vehicle_attributes'
@@ -112,10 +121,13 @@ def configure(self):
         """
         self.color_out = self.get_value_from_config('color_out')
         self.type_out = self.get_value_from_config('type_out')
+        self.outputs_verified = False
 
     def process(self, raw, identifiers=None, frame_meta=None):
         res = []
         raw_output = self._extract_predictions(raw, frame_meta)
+        if not self.outputs_verified:
+            self.select_output_blob(raw_output)
         for identifier, colors, types in zip(identifiers, raw_output[self.color_out], raw_output[self.type_out]):
             res.append(ContainerPrediction({
                 'color': ClassificationPrediction(identifier, colors.reshape(-1)),
@@ -124,6 +136,11 @@ def process(self, raw, identifiers=None, frame_meta=None):
 
         return res
 
+    def select_output_blob(self, outputs):
+        self.check_output_name(self.color_out, outputs)
+        self.check_output_name(self.type_out, outputs)
+        self.outputs_verified = True
+
 
 class AgeGenderAdapter(Adapter):
     __provider__ = 'age_gender'
@@ -141,6 +158,7 @@ def parameters(cls):
     def configure(self):
         self.age_out = self.get_value_from_config('age_out')
         self.gender_out = self.get_value_from_config('gender_out')
+        self.outputs_verified = False
 
     @classmethod
     def validate_config(cls, config, fetch_only=False, **kwargs):
@@ -166,6 +184,8 @@ def get_age_scores(age):
     def process(self, raw, identifiers=None, frame_meta=None):
         result = []
         raw_output = self._extract_predictions(raw, frame_meta)
+        if not self.outputs_verified:
+            self.select_output_blob(raw_output)
         for identifier, age, gender in zip(identifiers, raw_output[self.age_out], raw_output[self.gender_out]):
             gender = gender.reshape(-1)
             age = age.reshape(-1)[0]*100
@@ -178,6 +198,11 @@ def process(self, raw, identifiers=None, frame_meta=None):
 
         return result
 
+    def select_output_blob(self, outputs):
+        self.age_out = self.check_output_name(self.age_out, outputs)
+        self.gender_out = self.check_output_name(self.gender_out, outputs)
+        self.outputs_verified = True
+
 
 class AgeRecognitionAdapter(Adapter):
     __provider__ = 'age_recognition'
@@ -193,6 +218,7 @@ def parameters(cls):
 
     def configure(self):
         self.age_out = self.get_value_from_config('age_out')
+        self.output_verified = False
 
     @classmethod
     def validate_config(cls, config, fetch_only=False, **kwargs):
@@ -218,8 +244,8 @@ def get_age_scores(age):
     def process(self, raw, identifiers=None, frame_meta=None):
         result = []
         raw_output = self._extract_predictions(raw, frame_meta)
-        self.select_output_blob(raw_output)
-        self.age_out = self.age_out or self.output_blob
+        if not self.output_verified:
+            self.select_output_blob(raw_output)
         prediction = raw_output[self.age_out]
         for identifier, output in zip(identifiers, prediction):
             age = np.argmax(output)
@@ -231,6 +257,15 @@ def process(self, raw, identifiers=None, frame_meta=None):
 
         return result
 
+    def select_output_blob(self, outputs):
+        self.output_verified = True
+        if self.age_out:
+            self.age_out = self.check_output_name(self.age_out, outputs)
+            return
+        super().select_output_blob(outputs)
+        self.age_out = self.output_blob
+        return
+
 
 class LandmarksRegressionAdapter(Adapter):
     __provider__ = 'landmarks_regression'

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/audio_recognition.py

@@ -91,10 +91,11 @@ def configure(self):
         self.classification_out = self.get_value_from_config('classification_out')
         self.alphabet = ' ' + string.ascii_lowercase + '\'-'
         self.alphabet = self.alphabet.encode('ascii').decode('utf-8')
+        self.output_verified = False
 
     def process(self, raw, identifiers=None, frame_meta=None):
-        if self.classification_out is not None:
-            self.output_blob = self.classification_out
+        if not self.output_verified:
+            self.select_output_blob(raw)
         multi_infer = frame_meta[-1].get('multi_infer', False) if frame_meta else False
 
         raw_output = self._extract_predictions(raw, frame_meta)
@@ -195,6 +196,15 @@ def decode(probabilities, beamwidth=10, blank_id=None):
 
         return res
 
+    def select_output_blob(self, outputs):
+        self.output_verified = True
+        if self.classification_out:
+            self.classification_out = self.check_output_name(self.classification_out, outputs)
+            return
+        super().select_output_blob(outputs)
+        self.classification_out = self.output_blob
+        return
+
 
 class CTCGreedyDecoder(Adapter):
     __provider__ = 'ctc_greedy_decoder'
@@ -217,6 +227,16 @@ def configure(self):
         self.alphabet = self.get_value_from_config('alphabet') or ' ' + string.ascii_lowercase + '\'-'
         self.softmaxed_probabilities = self.launcher_config.get('softmaxed_probabilities')
         self.classification_out = self.get_value_from_config('classification_out')
+        self.output_verified = False
+
+    def select_output_blob(self, outputs):
+        self.output_verified = True
+        if self.classification_out:
+            self.classification_out = self.check_output_name(self.classification_out, outputs)
+            return
+        super().select_output_blob(outputs)
+        self.classification_out = self.output_blob
+        return
 
     @staticmethod
     def _extract_predictions(outputs_list, meta):
@@ -232,8 +252,8 @@ def _extract_predictions(outputs_list, meta):
         return output_map
 
     def process(self, raw, identifiers, frame_meta):
-        if self.classification_out is not None:
-            self.output_blob = self.classification_out
+        if not self.output_verified:
+            self.select_output_blob(raw)
         multi_infer = frame_meta[-1].get('multi_infer', False) if frame_meta else False
 
         raw_output = self._extract_predictions(raw, frame_meta)
@@ -357,6 +377,16 @@ def configure(self):
         if self.sep not in self.alphabet  and  self.sep != '':
             raise ValueError("\"sep\" must be in alphabet or be an empty string")
         self.init_lm(lm_file, lm_vocabulary_offset, lm_vocabulary_length)
+        self.output_verified = False
+
+    def select_output_blob(self, outputs):
+        self.output_verified = True
+        if self.probability_out:
+            self.probability_out = self.check_output_name(self.probability_out, outputs)
+            return
+        super().select_output_blob(outputs)
+        self.probability_out = self.output_blob
+        return
 
     @staticmethod
     def load_python_modules():
@@ -377,6 +407,8 @@ def init_lm(self, lm_file, lm_vocabulary_offset, lm_vocabulary_length):
                 raise ValueError("Need lm_alpha and lm_beta to use lm_file")
 
     def process(self, raw, identifiers=None, frame_meta=None):
+        if not self.output_verified:
+            self.select_output_blob(raw)
         log_prob = self._extract_predictions(raw, frame_meta)
         log_prob = np.concatenate(list(log_prob))
         if not self.logarithmic_prob:

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/centernet.py

@@ -42,6 +42,13 @@ def configure(self):
         self.center_heatmap_out = self.get_value_from_config('center_heatmap_out')
         self.width_height_out = self.get_value_from_config('width_height_out')
         self.regression_out = self.get_value_from_config('regression_out')
+        self.outpus_verified = False
+
+    def select_output_blob(self, outputs):
+        self.center_heatmap_out = self.check_output_name(self.center_heatmap_out, outputs)
+        self.width_height_out = self.check_output_name(self.width_height_out, outputs)
+        self.regression_out = self.check_output_name(self.regression_out, outputs)
+        self.outpus_verified = True
 
     @staticmethod
     def _gather_feat(feat, ind):
@@ -125,6 +132,8 @@ def _transform(dets, center, scale, height, width):
     def process(self, raw, identifiers, frame_meta):
         result = []
         predictions_batch = self._extract_predictions(raw, frame_meta)
+        if not self.outpus_verified:
+            self.select_output_blob(predictions_batch)
         hm_batch = predictions_batch[self.center_heatmap_out]
         wh_batch = predictions_batch[self.width_height_out]
         reg_batch = predictions_batch[self.regression_out]

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/classification.py

@@ -59,6 +59,16 @@ def configure(self):
         self.fixed_output = self.get_value_from_config('fixed_output')
         self.fixed_output_index = int(self.get_value_from_config('fixed_output_index'))
         self.label_as_array = self.get_value_from_config('label_as_array')
+        self.output_verified = False
+
+    def select_output_blob(self, outputs):
+        self.output_verified = True
+        if self.classification_out:
+            self.classification_out = self.check_output_name(self.classification_out, outputs)
+            return
+        super().select_output_blob(outputs)
+        self.classification_out = self.output_blob
+        return
 
     def process(self, raw, identifiers, frame_meta):
         """
@@ -69,8 +79,8 @@ def process(self, raw, identifiers, frame_meta):
         Returns:
             list of ClassificationPrediction objects
         """
-        if self.classification_out is not None:
-            self.output_blob = self.classification_out
+        if not self.output_verified:
+            self.select_output_blob(raw)
         multi_infer = frame_meta[-1].get('multi_infer', False) if frame_meta else False
         raw_prediction = self._extract_predictions(raw, frame_meta)
         self.select_output_blob(raw_prediction)

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/ctpn.py

@@ -81,8 +81,16 @@ def configure(self):
             [0, -91, 15, 106],
             [0, -134, 15, 149]
         ])
+        self.outputs_verified = False
+
+    def select_output_blob(self, outputs):
+        self.cls_prob_out = self.check_output_name(self.cls_prob_out, outputs)
+        self.bbox_pred_out = self.check_output_name(self.bbox_pred_out, outputs)
+        self.outputs_verified = True
 
     def process(self, raw, identifiers, frame_meta):
+        if not self.outputs_verified:
+            self.select_output_blob(raw)
         raw_outputs = self._extract_predictions(raw, frame_meta)
         result = []
         data = zip(raw_outputs[self.bbox_pred_out], raw_outputs[self.cls_prob_out], frame_meta, identifiers)