AC: Refactoring custom evaluators (models part) (#2880)

* Refactor models for custom evaluators

* Align network_info section for cascade models

* Fix pylint errors

* Updated text-recognition configs

* Fixed input/output setting for asr_encoder_prediction_joint_evaluator

* Fixed adapter output_blob setting for asr_encoder_decoder_evaluator

Author: Anna Grebneva

models/intel/formula-recognition-medium-scan-0001/accuracy-check.yml

@@ -3,8 +3,9 @@ evaluations:
     module: custom_evaluators.custom_text_recognition_evaluator.TextRecognitionWithAttentionEvaluator
     module_config:
       network_info:
-
-        max_seq_len: '192'
+        recognizer_encoder: {}
+        recognizer_decoder: {}
+      max_seq_len: '192'
 
       launchers:
         - framework: dlsdk

models/intel/formula-recognition-polynomials-handwritten-0001/accuracy-check.yml

@@ -3,7 +3,9 @@ evaluations:
     module: custom_evaluators.custom_text_recognition_evaluator.TextRecognitionWithAttentionEvaluator
     module_config:
       network_info:
-        max_seq_len: "192"
+        recognizer_encoder: {}
+        recognizer_decoder: {}
+      max_seq_len: "192"
 
       launchers:
         - framework: dlsdk

models/intel/text-recognition-0015/accuracy-check.yml

@@ -5,74 +5,76 @@ evaluations:
       model_type: SequentialTextRecognitionModel
       lowercase: true
       network_info:
-        max_seq_len: "24"
-        custom_label_map:
-          0: "<s>"
-          1: ""
-          2: "</s>"
-          3: "?"
-          4: "0"
-          5: "1"
-          6: "2"
-          7: "3"
-          8: "4"
-          9: "5"
-          10: "6"
-          11: "7"
-          12: "8"
-          13: "9"
-          14: "a"
-          15: "b"
-          16: "c"
-          17: "d"
-          18: "e"
-          19: "f"
-          20: "g"
-          21: "h"
-          22: "i"
-          23: "j"
-          24: "k"
-          25: "l"
-          26: "m"
-          27: "n"
-          28: "o"
-          29: "p"
-          30: "q"
-          31: "r"
-          32: "s"
-          33: "t"
-          34: "u"
-          35: "v"
-          36: "w"
-          37: "x"
-          38: "y"
-          39: "z"
-          40: "A"
-          41: "B"
-          42: "C"
-          43: "D"
-          44: "E"
-          45: "F"
-          46: "G"
-          47: "H"
-          48: "I"
-          49: "J"
-          50: "K"
-          51: "L"
-          52: "M"
-          53: "N"
-          54: "O"
-          55: "P"
-          56: "Q"
-          57: "R"
-          58: "S"
-          59: "T"
-          60: "U"
-          61: "V"
-          62: "W"
-          63: "X"
-          64: "Y"
-          65: "Z"
+        recognizer_encoder: {}
+        recognizer_decoder: {}
+      max_seq_len: "24"
+      custom_label_map:
+        0: "<s>"
+        1: ""
+        2: "</s>"
+        3: "?"
+        4: "0"
+        5: "1"
+        6: "2"
+        7: "3"
+        8: "4"
+        9: "5"
+        10: "6"
+        11: "7"
+        12: "8"
+        13: "9"
+        14: "a"
+        15: "b"
+        16: "c"
+        17: "d"
+        18: "e"
+        19: "f"
+        20: "g"
+        21: "h"
+        22: "i"
+        23: "j"
+        24: "k"
+        25: "l"
+        26: "m"
+        27: "n"
+        28: "o"
+        29: "p"
+        30: "q"
+        31: "r"
+        32: "s"
+        33: "t"
+        34: "u"
+        35: "v"
+        36: "w"
+        37: "x"
+        38: "y"
+        39: "z"
+        40: "A"
+        41: "B"
+        42: "C"
+        43: "D"
+        44: "E"
+        45: "F"
+        46: "G"
+        47: "H"
+        48: "I"
+        49: "J"
+        50: "K"
+        51: "L"
+        52: "M"
+        53: "N"
+        54: "O"
+        55: "P"
+        56: "Q"
+        57: "R"
+        58: "S"
+        59: "T"
+        60: "U"
+        61: "V"
+        62: "W"
+        63: "X"
+        64: "Y"
+        65: "Z"
 
       launchers:
         - framework: dlsdk

models/intel/text-recognition-0016/accuracy-check.yml

@@ -5,48 +5,50 @@ evaluations:
       model_type: SequentialTextRecognitionModel
       lowercase: true
       network_info:
-        max_seq_len: "24"
-        custom_label_map:
-          0: "<s>"
-          1: ""
-          2: "</s>"
-          3: "?"
-          4: "0"
-          5: "1"
-          6: "2"
-          7: "3"
-          8: "4"
-          9: "5"
-          10: "6"
-          11: "7"
-          12: "8"
-          13: "9"
-          14: "a"
-          15: "b"
-          16: "c"
-          17: "d"
-          18: "e"
-          19: "f"
-          20: "g"
-          21: "h"
-          22: "i"
-          23: "j"
-          24: "k"
-          25: "l"
-          26: "m"
-          27: "n"
-          28: "o"
-          29: "p"
-          30: "q"
-          31: "r"
-          32: "s"
-          33: "t"
-          34: "u"
-          35: "v"
-          36: "w"
-          37: "x"
-          38: "y"
-          39: "z"
+        recognizer_encoder: {}
+        recognizer_decoder: {}
+      max_seq_len: "24"
+      custom_label_map:
+        0: "<s>"
+        1: ""
+        2: "</s>"
+        3: "?"
+        4: "0"
+        5: "1"
+        6: "2"
+        7: "3"
+        8: "4"
+        9: "5"
+        10: "6"
+        11: "7"
+        12: "8"
+        13: "9"
+        14: "a"
+        15: "b"
+        16: "c"
+        17: "d"
+        18: "e"
+        19: "f"
+        20: "g"
+        21: "h"
+        22: "i"
+        23: "j"
+        24: "k"
+        25: "l"
+        26: "m"
+        27: "n"
+        28: "o"
+        29: "p"
+        30: "q"
+        31: "r"
+        32: "s"
+        33: "t"
+        34: "u"
+        35: "v"
+        36: "w"
+        37: "x"
+        38: "y"
+        39: "z"
 
       launchers:
         - framework: dlsdk

models/intel/text-spotting-0005/accuracy-check.yml

@@ -7,31 +7,27 @@ evaluations:
 
         recognizer_encoder: {}
 
-        recognizer_decoder: {}
-
-        recognizer_decoder_inputs:
-          prev_symbol: prev_symbol
-          prev_hidden: prev_hidden
-          encoder_outputs: encoder_outputs
-
-        recognizer_decoder_outputs:
-          symbols_distribution: output
-          cur_hidden: hidden
-
-        max_seq_len: '28'
-        recognizer_confidence_threshold: '0.45'
-
-        alphabet: __abcdefghijklmnopqrstuvwxyz0123456789
-        sos_index: '0'
-        eos_index: '1'
-
-        adapter:
-          type: mask_rcnn_with_text
-          classes_out: labels
-          boxes_out: boxes
-          raw_masks_out: masks
-          texts_out: texts
-          confidence_threshold: 0.65
+        recognizer_decoder:
+          inputs:
+            prev_symbol: prev_symbol
+            prev_hidden: prev_hidden
+            encoder_outputs: encoder_outputs
+          outputs:
+            symbols_distribution: output
+            cur_hidden: hidden
+
+      max_seq_len: '28'
+      alphabet: __abcdefghijklmnopqrstuvwxyz0123456789
+      sos_index: '0'
+      eos_index: '1'
+      recognizer_confidence_threshold: '0.45'
+      adapter:
+        type: mask_rcnn_with_text
+        classes_out: labels
+        boxes_out: boxes
+        raw_masks_out: masks
+        texts_out: texts
+        confidence_threshold: 0.65
 
       launchers:
         - framework: dlsdk

models/intel/text-to-speech-en-0001/accuracy-check.yml

@@ -5,20 +5,21 @@ evaluations:
       network_info:
         forward_tacotron_duration: {}
 
-        forward_tacotron_regression: {}
-        forward_tacotron_regression_inputs:
-          data: data
-          data_mask: data_mask
-          pos_mask: pos_mask
+        forward_tacotron_regression:
+          inputs:
+            data: data
+            data_mask: data_mask
+            pos_mask: pos_mask
+          max_regression_len: '512'
 
-        melgan: {}
-        max_mel_len: '128'
-        max_regression_len: '512'
-        pos_mask_window: '4'
+        melgan:
+          max_mel_len: '128'
 
-        adapter:
-          type: regression
-          keep_shape: True
+      pos_mask_window: '4'
+
+      adapter:
+        type: regression
+        keep_shape: True
 
       launchers:
         - framework: dlsdk

models/intel/text-to-speech-en-multi-0001/accuracy-check.yml

@@ -5,21 +5,22 @@ evaluations:
       network_info:
         forward_tacotron_duration: {}
 
-        forward_tacotron_regression: {}
-        forward_tacotron_regression_inputs:
-          data: data
-          data_mask: data_mask
-          pos_mask: pos_mask
-          speaker_embedding: speaker_embedding
+        forward_tacotron_regression:
+          inputs:
+            data: data
+            data_mask: data_mask
+            pos_mask: pos_mask
+            speaker_embedding: speaker_embedding
+          max_regression_len: '512'
 
-        melgan: {}
-        max_mel_len: '128'
-        max_regression_len: '512'
-        pos_mask_window: '4'
+        melgan:
+          max_mel_len: '128'
 
-        adapter:
-          type: regression
-          keep_shape: True
+      pos_mask_window: '4'
+
+      adapter:
+        type: regression
+        keep_shape: True
 
       launchers:
         - framework: dlsdk

tools/accuracy_checker/openvino/tools/accuracy_checker/adapters/audio_recognition.py

@@ -745,7 +745,8 @@ def set_alphabet(self):
         if 'vocabulary_file' in self.launcher_config:
             self.alphabet = read_txt(self.get_value_from_config('vocabulary_file'), ignore_space=True)
         else:
-            self.alphabet = self.get_value_from_config('alphabet') or ' ' + string.ascii_lowercase + '\''
+            self.alphabet = (''.join(self.get_value_from_config('alphabet')) if self.get_value_from_config('alphabet')
+                             else ' ' + string.ascii_lowercase + '\'')
             self.alphabet = self.alphabet.encode('ascii').decode('utf-8')
 
     def process(self, raw, identifiers=None, frame_meta=None):