Merge branch 'main' into xinyuye/yolov4

Author: XinyuYe-Intel

vision/classification/shufflenet/README.md

@@ -30,7 +30,7 @@ ONNX ShuffleNet-v2 ==> Quantized ONNX ShuffleNet-v2
 |-------------|:--------------|:--------------|:--------------|:--------------|:--------------|:--------------|
 |ShuffleNet-v2 |[9.2MB](model/shufflenet-v2-10.onnx) |  [8.7MB](model/shufflenet-v2-10.tar.gz) | 1.6 | 10 | 30.64 | 11.68|
 |ShuffleNet-v2-fp32 |[8.79MB](model/shufflenet-v2-12.onnx) |[8.69MB](model/shufflenet-v2-12.tar.gz) |1.9 |12 |33.65 |13.43|
-|ShuffleNet-v2-int8 |[2.28MB](model/shufflenet-v2-12-int8.onnx) |[2.37MB](model/shufflenet-v2-10-int8.tar.gz) |1.9 |12 |33.85 |13.66 |
+|ShuffleNet-v2-int8 |[2.28MB](model/shufflenet-v2-12-int8.onnx) |[2.37MB](model/shufflenet-v2-12-int8.tar.gz) |1.9 |12 |33.85 |13.66 |
 > Compared with the fp32 ShuffleNet-v2, int8 ShuffleNet-v2's Top-1 error rising ratio is 0.59%, Top-5 error rising ratio is 1.71% and performance improvement is 1.62x.
 >
 > Note the performance depends on the test hardware. 

vision/classification/vgg/model/vgg16-12-int8.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:477a77f7ba31bc6bebe6e9824cb0108173d3bb0c54506d8c6663ea36eee7dfb4
+size 105964121

workflow_scripts/test_models.py

@@ -2,82 +2,85 @@
 
 import argparse
 import check_model
-import os
 from pathlib import Path
 import subprocess
 import sys
 import test_utils
 
 
 def main():
-  parser = argparse.ArgumentParser(description='Test settings')
-  # default all: test by both onnx and onnxruntime
-  # if target is specified, only test by the specified one
-  parser.add_argument('--target', required=False, default='all', type=str,
-                      help='Test the model by which (onnx/onnxruntime)?',
-                      choices=['onnx', 'onnxruntime', 'all'])
-  args = parser.parse_args()
+    parser = argparse.ArgumentParser(description='Test settings')
+    # default all: test by both onnx and onnxruntime
+    # if target is specified, only test by the specified one
+    parser.add_argument('--target', required=False, default='all', type=str,
+                        help='Test the model by which (onnx/onnxruntime)?',
+                        choices=['onnx', 'onnxruntime', 'all'])
+    args = parser.parse_args()
 
-  cwd_path = Path.cwd()
-  # git fetch first for git diff on GitHub Action
-  subprocess.run(['git', 'fetch', 'origin', 'main:main'], cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  # obtain list of added or modified files in this PR
-  obtain_diff = subprocess.Popen(['git', 'diff', '--name-only', '--diff-filter=AM', 'origin/main', 'HEAD'],
-  cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-  stdoutput, stderroutput = obtain_diff.communicate()
-  diff_list = stdoutput.split()
+    cwd_path = Path.cwd()
+    # git fetch first for git diff on GitHub Action
+    subprocess.run(['git', 'fetch', 'origin', 'main:main'],
+                   cwd=cwd_path, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE)
+    # obtain list of added or modified files in this PR
+    obtain_diff = subprocess.Popen(['git', 'diff', '--name-only', '--diff-filter=AM', 'origin/main', 'HEAD'],
+                                   cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdoutput, _ = obtain_diff.communicate()
+    diff_list = stdoutput.split()
 
-  # identify list of changed onnx models in model Zoo
-  model_list = [str(model).replace("b'","").replace("'", "") for model in diff_list if ".onnx" in str(model)]
-  # run lfs install before starting the tests
-  test_utils.run_lfs_install()
+    # identify list of changed ONNX models in ONXX Model Zoo
+    tar_ext_name = '.tar.gz'
+    onnx_ext_name = '.onnx'
+    model_list = [str(model).replace("b'", "").replace("'", "")
+                  for model in diff_list if onnx_ext_name in str(model) or tar_ext_name in str(model)]
+    # run lfs install before starting the tests
+    test_utils.run_lfs_install()
 
-  print('\n=== Running ONNX Checker on added models ===\n')
-  # run checker on each model
-  failed_models = []
-  tar_ext_name = '.tar.gz'
-  for model_path in model_list:
-      model_name = model_path.split('/')[-1]
-      tar_name = model_name.replace('.onnx', tar_ext_name)
-      print('==============Testing {}=============='.format(model_name))
+    print('\n=== Running ONNX Checker on added models ===\n')
+    # run checker on each model
+    failed_models = []
+    for model_path in model_list:
+        model_name = model_path.split('/')[-1]
+        print('==============Testing {}=============='.format(model_name))
 
-      try:
-        # Step 1: check the onnx model and test_data_set from .tar.gz by ORT
-        # replace '.onnx' with '.tar.gz'
-        tar_gz_path = model_path[:-5] + '.tar.gz'
-        print(tar_gz_path)
-        test_data_set = []
-        # if tar.gz exists, git pull and try to get test data
-        if (args.target == 'onnxruntime' or args.target == 'all') and os.path.exists(tar_gz_path):
-          test_utils.pull_lfs_file(tar_gz_path)
-          # check whether 'test_data_set_0' exists
-          model_path_from_tar, test_data_set = test_utils.extract_test_data(tar_gz_path)
-          # finally check the onnx model from .tar.gz by ORT
-          # if the test_data_set does not exist, create the test_data_set
-          check_model.run_backend_ort(model_path_from_tar, test_data_set)
-          print('[PASS] {} is checked by onnxruntime. '.format(tar_name))
+        try:
+            # check .tar.gz by ORT and ONNX
+            if tar_ext_name in model_name:
+                # Step 1: check the ONNX model and test_data_set from .tar.gz by ORT
+                test_data_set = []
+                test_utils.pull_lfs_file(model_path)
+                # check whether 'test_data_set_0' exists
+                model_path_from_tar, test_data_set = test_utils.extract_test_data(model_path)
+                # if tar.gz exists, git pull and try to get test data
+                if (args.target == 'onnxruntime' or args.target == 'all'):
+                    # finally check the ONNX model from .tar.gz by ORT
+                    # if the test_data_set does not exist, create the test_data_set
+                    check_model.run_backend_ort(model_path_from_tar, test_data_set)
+                    print('[PASS] {} is checked by onnxruntime. '.format(model_name))
+                # Step 2: check the ONNX model inside .tar.gz by ONNX
+                if args.target == 'onnx' or args.target == 'all':
+                    check_model.run_onnx_checker(model_path_from_tar)
+                    print('[PASS] {} is checked by onnx. '.format(model_name))
+            # check uploaded standalone ONNX model by ONNX
+            elif onnx_ext_name in model_name:
+                test_utils.pull_lfs_file(model_path)
+                if args.target == 'onnx' or args.target == 'all':
+                    check_model.run_onnx_checker(model_path)
+                    print('[PASS] {} is checked by onnx. '.format(model_name))
 
-        # Step 2: check the uploaded onnx model by ONNX
-        # git pull the onnx file
-        test_utils.pull_lfs_file(model_path)
-        # 2. check the uploaded onnx model by ONNX
-        if args.target == 'onnx' or args.target == 'all':
-          check_model.run_onnx_checker(model_path)
-          print('[PASS] {} is checked by onnx. '.format(model_name))
+        except Exception as e:
+            print('[FAIL] {}: {}'.format(model_name, e))
+            failed_models.append(model_path)
+            test_utils.remove_onnxruntime_test_dir()
+            # remove the produced tar directory
+            test_utils.remove_tar_dir()
 
-      except Exception as e:
-        print('[FAIL] {}: {}'.format(model_name, e))
-        failed_models.append(model_path)
-        test_utils.remove_onnxruntime_test_dir()
+    if len(failed_models) == 0:
+        print('{} models have been checked. '.format(len(model_list)))
+    else:
+        print('In all {} models, {} models failed. '.format(len(model_list), len(failed_models)))
+        sys.exit(1)
 
-      # remove the produced tar directory
-      test_utils.remove_tar_dir()
-
-  if len(failed_models) == 0:
-      print('{} models have been checked. '.format(len(model_list)))
-  else:
-      print('In all {} models, {} models failed. '.format(len(model_list), len(failed_models)))
-      sys.exit(1)
 
 if __name__ == '__main__':
     main()