Merge pull request #3405 from anzhella-pankratova/performance_test_update

Performance scope: add demos with dynamism

Author: Wovchena

demos/tests/cases.py

@@ -15,6 +15,7 @@
 import collections
 import itertools
 import sys
+from copy import deepcopy
 
 from args import (
     DataDirectoryArg, DataDirectoryOrigFileNamesArg, DataPatternArg,
@@ -41,6 +42,7 @@ def __init__(self, name, implementation, model_keys=None, device_keys=None, test
 
         self._exec_name = self.subdirectory.replace('/', '_')
         self.parser = None
+        self.supported_devices = None
 
         Demo.IMPLEMENTATION_TYPES.add(implementation)
 
@@ -50,6 +52,8 @@ def models_lst_path(self, source_dir):
     def device_args(self, device_list):
         if len(self.device_keys) == 0:
             return {'CPU': []}
+        if self.supported_devices:
+            device_list = list(set(device_list) & set(self.supported_devices))
         return {device: [arg for key in self.device_keys for arg in [key, device]] for device in device_list}
 
     def get_models(self, case):
@@ -99,6 +103,10 @@ def only_models(self, models):
                     continue
         return self
 
+    def only_devices(self, devices):
+        self.supported_devices = devices
+        return self
+
     def set_precisions(self, precisions, model_info):
         for case in self.test_cases[:]:
             updated_options = {p: {} for p in precisions}
@@ -1407,4 +1415,4 @@ def single_option_cases(key, *args):
 ]
 
 
-BASE = { demo.subdirectory : demo for demo in DEMOS }
+BASE = { demo.subdirectory : deepcopy(demo) for demo in DEMOS }

demos/tests/performance_cases.py

@@ -14,8 +14,7 @@
 
 import os
 import re
-from args import DataPatternArg
-from copy import deepcopy
+#from args import DataPatternArg
 
 from cases import BASE, single_option_cases
 
@@ -49,25 +48,46 @@ def write_to_csv(self, result, test_case, device):
 
         if not os.path.isfile(self.filename):
             models_col = [f"Model {key}" for key in self.model_keys]
-            columns = ','.join(['Device', *models_col, *result.keys()])
+            precisions_col = [f"Precision {key}" for key in self.model_keys]
+            columns = ','.join(['Device', *precisions_col, *models_col, *result.keys()])
             with open(self.filename, 'w') as f:
                 print(columns, file=f)
 
+        precisions = [test_case.options[key].precision if key in test_case.options else '-'
+                      for key in self.model_keys]
         models_names = [test_case.options[key].name if key in test_case.options else '-'
                         for key in self.model_keys]
-        data = ','.join([device, *models_names, *result.values()])
+        data = ','.join([device, *precisions, *models_names, *result.values()])
         with open(self.filename, 'a') as f:
             print(data, file=f)
 
 
 DEMOS = [
-    deepcopy(BASE['interactive_face_detection_demo/cpp']).add_parser(PerformanceParser),
+    BASE['interactive_face_detection_demo/cpp_gapi'].add_parser(PerformanceParser),
 
-    deepcopy(BASE['object_detection_demo/python'])
+    BASE['interactive_face_detection_demo/cpp'].add_parser(PerformanceParser),
+
+    BASE['object_detection_demo/python']
         .only_models(['person-detection-0200', 'yolo-v2-tf'])
-        .update_option({'-i': DataPatternArg('action-recognition')})
+    # TODO: create large -i for performance scenario
+    #    .update_option({'-i': DataPatternArg('action-recognition')})
         .add_test_cases(single_option_cases('-nireq', '3', '5'),
                         single_option_cases('-nstreams', '3', '4'),
                         single_option_cases('-nthreads', str(THREADS_NUM), str(THREADS_NUM - 2)))
-        .add_parser(PerformanceParser)
+        .add_parser(PerformanceParser),
+
+    BASE['bert_named_entity_recognition_demo/python']
+        .update_option({'--dynamic_shape': None})
+        .only_devices(['CPU'])
+        .add_parser(PerformanceParser),
+
+    BASE['gpt2_text_prediction_demo/python']
+        .update_option({'--dynamic_shape': None})
+        .only_devices(['CPU'])
+        .add_parser(PerformanceParser),
+
+    BASE['speech_recognition_wav2vec_demo/python']
+        .update_option({'--dynamic_shape': None})
+        .only_devices(['CPU'])
+        .add_parser(PerformanceParser),
 ]

demos/tests/run_tests.py

@@ -83,7 +83,7 @@ def parse_args():
     parser.add_argument('--supported-devices', type=parser_paths_list, required=False,
         help='paths to Markdown files with supported devices for each model')
     parser.add_argument('--precisions', type=str, nargs='+', default=['FP16', 'FP16-INT8'],
-        help='IR precisions for all models. By default, models are tested in FP16 precision')
+        help='IR precisions for all models. By default, models are tested in FP16, FP16-INT8 precisions')
     parser.add_argument('--models-dir', type=Path, required=False, metavar='DIR',
         help='directory with pre-converted models (IRs)')
     return parser.parse_args()