OV2.0 AC: migrate POT evaluator on usage new OV API (#2914)

* AC: migrate POT evaluator on usage new OV API

* enable custom evaluators

* fix configuration update

* fix mtcnn reshape

* bugfix

* mtcnn remove output renaming

* fix sync mode and mtcnn callback

* return raw results for callback

* experiment with text recognition

* fix input output selection

* update custom evaluators

* fix output selection

* properly hanlde renaming

* fix num requests

* fix text recognition

* bump version

Author: eaidova

tools/accuracy_checker/openvino/tools/accuracy_checker/__init__.py

@@ -14,4 +14,4 @@
 limitations under the License.
 """
 
-__version__ = "0.9.1"
+__version__ = "0.9.2"

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/image_processing.py

@@ -72,7 +72,7 @@ def configure(self):
         self.output_verified = False
 
     def select_output_blob(self, outputs):
-        self.outputs_verified = True
+        self.output_verified = True
         if not self.target_out:
             super().select_output_blob(outputs)
             self.target_out = self.output_blob
@@ -114,9 +114,8 @@ class SuperResolutionAdapter(ImageProcessingAdapter):
     def process(self, raw, identifiers=None, frame_meta=None):
         result = []
         raw_outputs = self._extract_predictions(raw, frame_meta)
-        if not self.outputs_verified:
+        if not self.output_verified:
             self.select_output_blob(raw_outputs)
-            self.target_out = self.output_blob
 
         for identifier, img_sr in zip(identifiers, raw_outputs[self.target_out]):
             img_sr = self._basic_postprocess(img_sr)

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/asr_custom_encoder_decoder_joint.py

@@ -60,10 +60,14 @@ def predict(self, identifiers, input_data):
         # Evaluate the encoder network one feature frame at a time
         data = self.fit_to_input(input_data)
         outputs = self.infer(data)
+        if isinstance(outputs, tuple):
+            outputs, raw_outputs = outputs
+        else:
+            raw_outputs = outputs
         encoder_output = np.array(outputs[self.encoder_out]).squeeze()
         self.h0 = outputs[self.h0_out]
         self.c0 = outputs[self.c0_out]
-        return encoder_output, outputs
+        return encoder_output, raw_outputs
 
     def fit_to_input(self, input_data):
         return {self.input: input_data, self.h0_input: self.h0, self.c0_input: self.c0}
@@ -117,9 +121,13 @@ def reset(self):
     def predict(self, identifiers, input_data, hidden=None):
         data = self.fit_to_input(input_data, hidden)
         outputs = self.infer(data)
+        if isinstance(outputs, tuple):
+            outputs, raw_outputs = outputs
+        else:
+            raw_outputs = outputs
         self.h0 = outputs[self.h0_out]
         self.c0 = outputs[self.c0_out]
-        return np.array(outputs[self.decoder_out]).squeeze(), (self.h0, self.c0), outputs
+        return np.array(outputs[self.decoder_out]).squeeze(), (self.h0, self.c0), raw_outputs
 
     def fit_to_input(self, token_id, hidden):
         if hidden is None:
@@ -176,8 +184,12 @@ def predict(self, identifiers, input_data):
         encoder_out, predictor_out = input_data
         data = self.fit_to_input(encoder_out, predictor_out)
         outputs = self.infer(data)
+        if isinstance(outputs, tuple):
+            outputs, raw_outputs = outputs
+        else:
+            raw_outputs = outputs
         joint_out = outputs[self.output]
-        return log_softmax(np.array(joint_out).squeeze()), outputs
+        return log_softmax(np.array(joint_out).squeeze()), raw_outputs
 
     def fit_to_input(self, encoder_out, predictor_out):
         return {self.input1: encoder_out, self.input2: predictor_out}
@@ -282,10 +294,11 @@ def set_input_and_output(self):
             ]
             self.with_prefix = with_prefix
 
-
     def predict(self, identifiers, input_data):
         raise NotImplementedError
 
+    def infer(self, input_data, raw_results=False):
+        return super().infer(input_data, True)
 
 
 class DLSDKEncoder(Encoder, CommonDLSDKModel):

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/asr_encoder_decoder_evaluator.py

@@ -96,8 +96,12 @@ def predict(self, identifiers, input_data, encoder_callback=None):
         predictions, raw_outputs = [], []
         for data in input_data:
             encoder_prediction, decoder_inputs = self.encoder.predict(identifiers, data)
+            if isinstance(encoder_prediction, tuple):
+                encoder_prediction, raw_encoder_prediction = encoder_prediction
+            else:
+                raw_encoder_prediction = encoder_prediction
             if encoder_callback:
-                encoder_callback(encoder_prediction)
+                encoder_callback(raw_encoder_prediction)
             if self.store_encoder_predictions:
                 self._encoder_predictions.append(encoder_prediction)
             raw_output, prediction = self.decoder.predict(identifiers, decoder_inputs)
@@ -137,8 +141,8 @@ def predict(self, identifiers, input_data):
 class EncoderOVModel(BaseOpenVINOModel):
     def predict(self, identifiers, input_data):
         input_data = self.fit_to_input(input_data)
-        results = self.infer(input_data)
-        return results, results[self.output_blob]
+        results = self.infer(input_data, raw_results=True)
+        return results, results[self.output_blob] if not isinstance(results, tuple) else results[0][self.output_blob]
 
 
 class DecoderDLSDKModel(BaseDLSDKModel):
@@ -167,10 +171,10 @@ def __init__(self, network_info, launcher, suffix=None, delayed_model_loading=Fa
 
     def predict(self, identifiers, input_data):
         feed_dict = self.fit_to_input(input_data)
-        results = self.infer(feed_dict)
+        results, raw_results = self.infer(feed_dict, raw_results=True)
         result = self.adapter.process([results], identifiers, [{}])
 
-        return results, result
+        return raw_results, result
 
     def set_input_and_output(self):
         super().set_input_and_output()

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/asr_encoder_prediction_joint_evaluator.py

@@ -21,7 +21,7 @@
 
 from ...adapters import create_adapter
 from ...config import ConfigError
-from ...utils import contains_all, read_pickle, parse_partial_shape
+from ...utils import contains_all, read_pickle, parse_partial_shape, postprocess_output_name
 from .asr_encoder_decoder_evaluator import AutomaticSpeechRecognitionEvaluator
 from .base_models import (
     BaseCascadeModel, BaseDLSDKModel, BaseOpenVINOModel, BaseONNXModel, create_model, create_encoder
@@ -80,8 +80,12 @@ def predict(self, identifiers, input_data, encoder_callback=None):
         predictions, raw_outputs = [], []
         for data in input_data:
             encoder_prediction, decoder_inputs = self.encoder.predict(identifiers, data)
+            if isinstance(encoder_prediction, tuple):
+                encoder_prediction, raw_encoder_prediction = encoder_prediction
+            else:
+                raw_encoder_prediction = encoder_prediction
             if encoder_callback:
-                encoder_callback(encoder_prediction)
+                encoder_callback(raw_encoder_prediction)
             if self.store_encoder_predictions:
                 self._encoder_predictions.append(encoder_prediction)
             raw_output, prediction = self.decoder(identifiers, decoder_inputs, callback=encoder_callback)
@@ -129,8 +133,12 @@ def _greedy_decode(self, x, out_len, callback=None):
                     self._get_last_symb(label),
                     hidden
                 )
+                if isinstance(g, tuple):
+                    g, raw_g = g
+                else:
+                    raw_g = g
                 if callback:
-                    callback(g)
+                    callback(raw_g)
                 hidden_prime = (g[self.prediction.output_layers[0]], g[self.prediction.output_layers[1]])
                 g = g[self.prediction.output_layers[2]]
                 logp = self._joint_step(f, g, log_normalize=False, callback=callback)[0, :]
@@ -161,8 +169,12 @@ def _pred_step(self, label, hidden):
     def _joint_step(self, enc, pred, log_normalize=False, callback=None):
         inputs = {self.joint.input_layers[0]: enc, self.joint.input_layers[1]: pred}
         logits, logits_blob = self.joint.predict(None, inputs)
+        if isinstance(logits, tuple):
+            logits, raw_logits = logits
+        else:
+            raw_logits = logits
         if callback:
-            callback(logits)
+            callback(raw_logits)
         logits = logits_blob[:, 0, 0, :]
         if not log_normalize:
             return logits
@@ -254,8 +266,8 @@ def __init__(self, network_info, launcher, suffix=None, delayed_model_loading=Fa
 
     def predict(self, identifiers, input_data, callback=None):
         input_data = self.fit_to_input(input_data)
-        results = self.infer(input_data)
-        return results, results[self.output_blob]
+        results = self.infer(input_data, raw_results=True)
+        return results, results[self.output_blob] if not isinstance(results, tuple) else results[0][self.output_blob]
 
     def fit_to_input(self, input_data):
         if isinstance(input_data, dict):
@@ -278,12 +290,10 @@ def set_input_and_output(self):
         with_prefix = input_blob.startswith(self.default_model_suffix)
         if self.input_blob is None or with_prefix != self.with_prefix:
             if self.output_blob is None:
-                output_blob = next(iter(self.exec_network.outputs)).get_node().friendly_name
+                output_blob = next(iter(self.outputs))
             else:
-                output_blob = (
-                    '_'.join([self.default_model_suffix, self.output_blob])
-                    if with_prefix else self.output_blob.split(self.default_model_suffix + '_')[-1]
-                )
+                output_blob = postprocess_output_name(self.output_blob, self.outputs, raise_error=False)
+
             self.input_blob = input_blob
             self.output_blob = output_blob
             self.with_prefix = with_prefix
@@ -292,11 +302,8 @@ def set_input_and_output(self):
                     '_'.join([self.default_model_suffix, inp])
                     if with_prefix else inp.split(self.default_model_suffix)[-1]
                 )
-            for idx, out in enumerate(self.output_layers):
-                self.output_layers[idx] = (
-                    '_'.join([self.default_model_suffix, out])
-                    if with_prefix else out.split(self.default_model_suffix)[-1]
-                )
+        for idx, out in enumerate(self.output_layers):
+            self.output_layers[idx] = postprocess_output_name(out, self.outputs, raise_error=False)
 
 
 class EncoderDLSDKModel(CommonDLSDKModel):

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/base_custom_evaluator.py

@@ -56,7 +56,8 @@ def process_dataset(self, subset=None, num_images=None, check_progress=False, da
 
         if 'progress_reporter' in kwargs:
             _progress_reporter = kwargs['progress_reporter']
-            _progress_reporter.reset(self.dataset.size)
+            if _progress_reporter is not None:
+                _progress_reporter.reset(self.dataset.size)
         else:
             _progress_reporter = None if not check_progress else self._create_progress_reporter(
                 check_progress, self.dataset.size

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/base_models.py

@@ -266,7 +266,8 @@ def _reshape_input(self, input_shapes):
         if self.is_dynamic:
             return
         if hasattr(self, 'exec_network') and self.exec_network is not None:
-            del self.infer_request
+            if hasattr(self, 'infer_request'):
+                del self.infer_request
             del self.exec_network
         index_mapping = self.input_index_mapping()
         input_shapes_for_tensors = {index_mapping[name]: shape for name, shape in input_shapes.items()}
@@ -322,24 +323,24 @@ def set_input_and_output(self):
         with_prefix = input_blob.startswith(self.default_model_suffix)
         if self.input_blob is None or with_prefix != self.with_prefix:
             if self.output_blob is None:
-                output_blob = next(iter(outputs)).get_node().friendly_name
-            else:
-                output_blob = (
-                    '_'.join([self.default_model_suffix, self.output_blob])
-                    if with_prefix else self.output_blob.split(self.default_model_suffix + '_')[-1]
-                )
+                self.output_blob = next(iter(outputs)).get_node().friendly_name
             self.input_blob = input_blob
-            self.output_blob = output_blob
             self.with_prefix = with_prefix
             if hasattr(self, 'adapter') and self.adapter is not None:
-                self.adapter.output_blob = output_blob
+                self.adapter.output_blob = self.output_blob
 
     @property
     def inputs(self):
         if self.network:
             return {node.get_node().friendly_name: node.get_node() for node in self.network.inputs}
         return {node.get_node().friendly_name: node.get_node() for node in self.exec_network.inputs}
 
+    @property
+    def outputs(self):
+        if self.network:
+            return {node.get_node().friendly_name: node.get_node() for node in self.network.outputs}
+        return {node.get_node().friendly_name: node.get_node() for node in self.exec_network.outputs}
+
     def fit_to_input(self, input_data):
         input_info = self.inputs[self.input_blob]
         if (self.input_blob in self.dynamic_inputs or
@@ -348,16 +349,19 @@ def fit_to_input(self, input_data):
 
         return {self.input_blob: np.array(input_data)}
 
-    def infer(self, input_data):
+    def infer(self, input_data, raw_results=False):
         if not hasattr(self, 'infer_request') or self.infer_request is None:
             self.infer_request = self.exec_network.create_infer_request()
         tensors_mapping = self.input_tensors_mapping()
         feed_dict = {tensors_mapping[name]: data for name, data in input_data.items()}
         outputs = self.infer_request.infer(feed_dict)
-        return {
+        res_outputs = {
             out_node.get_node().friendly_name: out_res
             for out_node, out_res in outputs.items()
         }
+        if raw_results:
+            return res_outputs, outputs
+        return res_outputs
 
 
 class BaseONNXModel:

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/cocosnet_evaluator.py

@@ -217,14 +217,14 @@ def predict(self, identifiers, input_data):
         results = []
         prediction = None
         if self.infer_request is None:
-            self.infer_request = self.exec_network.create_infer_reuqest()
+            self.infer_request = self.exec_network.create_infer_request()
         for current_input in input_data:
             data = self.fit_to_input(current_input)
             if not self.is_dynamic and self.dynamic_inputs:
                 self._reshape_input({k: v.shape for k, v in data.items()})
-            prediction = self.infer(data)
+            prediction, raw_prediction = self.infer(data, raw_results=True)
             results.append(*self.adapter.process(prediction, identifiers, [{}]))
-        return results, prediction
+        return results, raw_prediction
 
 
 class GanCheckModel(BaseDLSDKModel):

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/colorization_evaluator.py

@@ -166,10 +166,10 @@ def predict(self, identifiers, input_data):
         self._inputs[self.input_blob] = img_l_rs
         if not self.is_dynamic and self.dynamic_inputs:
             self._reshape_input({k: v.shape for k, v in self._inputs.items()})
-        res = self.infer(self._inputs)
+        res, raw_res = self.infer(self._inputs, raw_results=True)
 
         new_result = self.postprocessing(res[self.output_blob], img_l)
-        return res, np.array(new_result)
+        return raw_res, np.array(new_result)
 
     def set_input_and_output(self):
         super().set_input_and_output()
@@ -209,9 +209,9 @@ def predict(self, identifiers, input_data):
         input_dict = self.fit_to_input(input_data)
         if not self.is_dynamic and self.dynamic_inputs:
             self._reshape_input({k: v.shape for k, v in input_dict.items()})
-        raw_result = self.infer(input_dict)
+        raw_result, raw_outputs = self.infer(input_dict, raw_results=True)
         result = self.adapter.process([raw_result], identifiers, [{}])
-        return raw_result, result
+        return raw_outputs, result
 
     def fit_to_input(self, input_data):
         constant_normalization = 255.