Merge pull request #3553 from eaidova/ea/inf_precision_hint

AC: support inference precision hint setting

Author: Wovchena

tools/accuracy_checker/openvino/tools/accuracy_checker/__init__.py

@@ -14,4 +14,4 @@
 limitations under the License.
 """
 
-__version__ = "0.9.3"
+__version__ = "0.9.4"

tools/accuracy_checker/openvino/tools/accuracy_checker/argparser.py

@@ -302,6 +302,11 @@ def add_openvino_specific_args(parser):
              'static - convert undefined shapes to static before execution',
         required=False, default='default'
     )
+    openvino_specific_args.add_argument(
+        '--inference_precision_hint',
+        help='Inference Precision hint for device',
+        required=False
+    )
     openvino_specific_args.add_argument(
         '--use_new_api', type=cast_to_bool, help='switch to processing using OpenVINO 2.0 API', required=False,
         default=ov_new_api_available()

tools/accuracy_checker/openvino/tools/accuracy_checker/config/config_reader.py

@@ -840,10 +840,12 @@ def _async_evaluation_args(launcher_entry):
 
     kaldi_binaries = arguments.kaldi_bin_dir if 'kaldi_bin_dir' in arguments else None
     kaldi_logs = arguments.kaldi_log_file if 'kaldi_log_file' in arguments else None
+    precision_hint = arguments.inference_precision_hint if 'inference_precision_hint' in arguments else None
     if kaldi_binaries:
         launcher_entry['_kaldi_bin_dir'] = kaldi_binaries
         launcher_entry['_kaldi_log_file'] = kaldi_logs
-
+    if precision_hint:
+        launcher_entry['_inference_precision_hint'] = precision_hint
     if launcher_entry['framework'].lower() not in ['dlsdk', 'openvino']:
         return launcher_entry
 
@@ -936,6 +938,7 @@ def provide_model_type(launcher, arguments):
 def provide_models(launchers, arguments):
     input_precisions = arguments.input_precision if 'input_precision' in arguments else None
     input_layout = arguments.layout if 'layout' in arguments else None
+    
     provide_precision_and_layout(launchers, input_precisions, input_layout)
     if 'models' not in arguments or not arguments.models:
         return launchers

tools/accuracy_checker/openvino/tools/accuracy_checker/launcher/dlsdk_launcher_config.py

@@ -198,6 +198,10 @@ def check_model_source(entry, fetch_only=False, field_uri=None, validation_schem
     '_model_type': StringField(
         choices=['xml', 'blob', 'onnx', 'paddle', 'tf'],
         description='hint for model type in automatic model search', optional=True),
+    '_inference_precision_hint': StringField(
+        description='Model execution precision for device',
+        optional=True
+    )
 }
 
 

tools/accuracy_checker/openvino/tools/accuracy_checker/launcher/openvino_launcher.py

@@ -19,7 +19,6 @@
 import multiprocessing
 from pathlib import Path
 import re
-import warnings
 import numpy as np
 from openvino.runtime import Core, AsyncInferQueue, get_version, PartialShape, Type, Dimension
 from openvino.preprocess import PrePostProcessor
@@ -32,7 +31,6 @@
     ov_set_config
 )
 from .dlsdk_async_request import AsyncInferRequestWrapper
-
 from ..config import ConfigError
 from ..logging import warning, debug, print_info
 from ..utils import (
@@ -49,13 +47,12 @@
 format_map = {
       'f32': np.float32, 'i32': np.int32, 'i64': np.int64,
       'fp16': np.float16, 'f16': np.float16, 'i16': np.int16, 'u16': np.uint16,
-      'i8': np.int8, 'u8': np.uint8,
-      'boolean': np.uint8
+      'i8': np.int8, 'u8': np.uint8, 'boolean': np.uint8
 }
 
 PRECISION_STR_TO_TYPE = {
     'FP32': Type.f32, 'FP16': Type.f16, 'U8': Type.u8, 'U16': Type.u16, 'I8': Type.i8, 'I16': Type.i16,
-    'I32': Type.i32, 'I64': Type.i64, 'BOOL': Type.boolean
+    'I32': Type.i32, 'I64': Type.i64, 'BOOL': Type.boolean, 'INT8': Type.u8, 'BF16': Type.bf16
 }
 
 
@@ -177,19 +174,14 @@ def predict(self, inputs, metadata=None, return_raw=False, **kwargs):
         raw_results = []
         for infer_inputs in inputs:
             if self._do_reshape:
-                input_shapes = {
-                    layer_name: data.shape for layer_name, data in infer_inputs.items()
-                }
+                input_shapes = {layer_name: data.shape for layer_name, data in infer_inputs.items()}
                 self._reshape_input(input_shapes)
             if self.infer_request is None:
                 self.infer_request = self.exec_network.create_infer_request()
             feed_dict = {self.input_to_tensor_name[layer_name]: data for layer_name, data in infer_inputs.items()}
             outputs = self.infer_request.infer(inputs=feed_dict)
             raw_results.append(outputs)
-            results.append({
-                out_node.get_node().friendly_name: out_res
-                for out_node, out_res in outputs.items()
-            })
+            results.append({out_node.get_node().friendly_name: out_res for out_node, out_res in outputs.items()})
         if self.reset_memory_state:
             for state in self.infer_request.query_state():
                 state.reset()
@@ -211,10 +203,7 @@ def _predict_sequential(self, inputs, metadata=None, return_raw=False, **kwargs)
             feed_dict.update(lstm_inputs_feed)
             infer_inputs = {self.input_to_tensor_name[layer_name]: data for layer_name, data in feed_dict.items()}
             out_tensors = self.infer_request.infer(infer_inputs)
-            output_result = {
-                out_node.get_node().friendly_name: out_tensor
-                for out_node, out_tensor in out_tensors.items()
-            }
+            output_result = {out_node.get_node().friendly_name: out_tensor for out_node, out_tensor in out_tensors.items()}
             lstm_inputs_feed = self._fill_lstm_inputs(output_result)
             results.append(output_result)
             if return_raw:
@@ -395,6 +384,7 @@ def _device_specific_configuration(self):
         device_config = self.config.get('device_config')
         if device_config:
             self._set_device_config(device_config)
+        self._set_infer_precision_hint()
 
     def _set_nireq(self):
         num_requests = self.config.get('num_requests')
@@ -475,14 +465,29 @@ def _set_device_config(self, device_config):
                 if isinstance(value, dict):
                     if key in self._devices_list():
                         if key not in self.ie_core.available_devices:
-                            warnings.warn('{} device is unknown. Config loading may lead to error.'.format(key))
+                            warning('{} device is unknown. Config loading may lead to error.'.format(key))
                         ov_set_config(self.ie_core, dict(value), device=key)
                     else:
-                        warnings.warn(
+                        warning(
                             f'Configuration for {key} will be skipped as device is not listed in evaluation device')
                 else:
-                    warnings.warn('Option {key}: {value} will be skipped because device to which it should be '
-                                  'applied is not specified or option is not a dict-like'.format(key=key, value=value))
+                    warning(f'Option {key}: {value} will be skipped because device to which it should be '
+                                  f'applied is not specified or option is not a dict-like')
+
+    def _set_infer_precision_hint(self):
+        precision_hint = self.config.get('_inference_precision_hint')
+        if precision_hint is None:
+            return
+        supported_props = self.ie_core.get_property(self._device, 'SUPPORTED_PROPERTIES')
+        if 'INFERENCE_PRECISION_HINT' not in supported_props:
+            warning(f'inference precision hint is not supported for device {self._device}, option will be ingnored')
+            return
+        if not precision_hint.upper() in PRECISION_STR_TO_TYPE and not precision_hint in format_map:
+            raise ConfigError(f'Unknown precision {precision_hint} for inference precision hint')
+        precision_type = PRECISION_STR_TO_TYPE.get(precision_hint.upper(), precision_hint)
+        self.ie_core.set_property(self._device, {'INFERENCE_PRECISION_HINT': precision_type})
+        current_precision = self.ie_core.get_property(self._device, 'INFERENCE_PRECISION_HINT')
+        print_info(f'Inference precision: {current_precision.get_type_name()}')
 
     def _log_versions(self):
         versions = self.ie_core.get_versions(self._device)