Use avx512f instead of VNNI and refactor code in CIs (#533)

* Use avx512f instead of vnni and refactor code

Signed-off-by: jcwchen <[email protected]>

* Update workflow_scripts/check_model.py

Signed-off-by: jcwchen <[email protected]>

Co-authored-by: Gary Miguel <[email protected]>
Signed-off-by: jcwchen <[email protected]>

* fix flake8

Signed-off-by: jcwchen <[email protected]>

* typo

Signed-off-by: jcwchen <[email protected]>

Signed-off-by: jcwchen <[email protected]>
Co-authored-by: Gary Miguel <[email protected]>

Author: jcwchen

.github/workflows/linux_ci.yml

@@ -34,6 +34,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install onnx onnxruntime requests py-cpuinfo
+          # Print CPU info for debugging ONNX Runtime inference difference
           python -m cpuinfo
 
       - name: Test updated ONNX_HUB_MANIFEST.json

.github/workflows/windows_ci.yml

@@ -36,6 +36,7 @@ jobs:
           # TODO: now ONNX only supports Protobuf <= 3.20.1
           python -m pip install protobuf==3.20.1
           python -m pip install onnx onnxruntime requests py-cpuinfo
+          # Print CPU info for debugging ONNX Runtime inference difference
           python -m cpuinfo
 
       - name: Test new models by onnx

workflow_scripts/check_model.py

@@ -10,53 +10,55 @@
 import test_utils
 
 
-def has_vnni_support():
-    return 'avx512' in str(get_cpu_info()['flags'])
-
-
-def skip_quant_models_if_missing_vnni(model_name):
-    return ('-int8' in model_name or '-qdq' in model_name) and not has_vnni_support()
+def has_avx512f_support():
+    return "avx512f" in set(get_cpu_info()["flags"])
 
 
 def run_onnx_checker(model_path):
     model = onnx.load(model_path)
     onnx.checker.check_model(model)
 
 
+def ort_skip_reason(model_path):
+    if ("-int8" in model_path or "-qdq" in model_path) and not has_avx512f_support():
+        return f"Skip ORT test for {model_path} because this machine lacks avx512f support and the output.pb was produced with avx512f support."
+    model = onnx.load(model_path)
+    if model.opset_import[0].version < 7:
+        return f"Skip ORT test for {model_path} because ORT only supports opset version >= 7"
+    return None
+
+
 def make_tarfile(output_filename, source_dir):
     with tarfile.open(output_filename, "w:gz", format=tarfile.GNU_FORMAT) as tar:
         tar.add(source_dir, arcname=os.path.basename(source_dir))
 
 
 def run_backend_ort(model_path, test_data_set=None, tar_gz_path=None):
-    if skip_quant_models_if_missing_vnni(model_path):
-        print(f'Skip ORT test for {model_path} because this machine lacks of VNNI support and the output.pb was produced with VNNI support.')
-        return
-    model = onnx.load(model_path)
-    if model.opset_import[0].version < 7:
-        print('Skip ORT test since it only *guarantees* support for models stamped with opset version 7')
+    skip_reason = ort_skip_reason(model_path)
+    if skip_reason:
+        print(skip_reason)
         return
-    # if 'test_data_set_N' doesn't exist, create test_dir
+    # if "test_data_set_N" doesn't exist, create test_dir
     if not test_data_set:
         # Start from ORT 1.10, ORT requires explicitly setting the providers parameter if you want to use execution providers
         # other than the default CPU provider (as opposed to the previous behavior of providers getting set/registered by default
         # based on the build flags) when instantiating InferenceSession.
         # For example, if NVIDIA GPU is available and ORT Python package is built with CUDA, then call API as following:
-        # onnxruntime.InferenceSession(path/to/model, providers=['CUDAExecutionProvider'])
+        # onnxruntime.InferenceSession(path/to/model, providers=["CUDAExecutionProvider"])
         onnxruntime.InferenceSession(model_path)
         # Get model name without .onnx
         model_name = os.path.basename(os.path.splitext(model_path)[0])
         if model_name is None:
             print(f"The model path {model_path} is invalid")
             return
-        ort_test_dir_utils.create_test_dir(model_path, './', test_utils.TEST_ORT_DIR)
+        ort_test_dir_utils.create_test_dir(model_path, "./", test_utils.TEST_ORT_DIR)
         ort_test_dir_utils.run_test_dir(test_utils.TEST_ORT_DIR)
         if os.path.exists(model_name) and os.path.isdir(model_name):
             rmtree(model_name)
         os.rename(test_utils.TEST_ORT_DIR, model_name)
         make_tarfile(tar_gz_path, model_name)
         rmtree(model_name)
-    # otherwise use the existing 'test_data_set_N' as test data
+    # otherwise use the existing "test_data_set_N" as test data
     else:
         test_dir_from_tar = test_utils.get_model_directory(model_path)
         ort_test_dir_utils.run_test_dir(test_dir_from_tar)

workflow_scripts/generate_onnx_hub_manifest.py

@@ -100,7 +100,7 @@ def prep_name(col):
 def get_file_info(row, field, target_models=None):
     source_dir = split(row["source_file"])[0]
     model_file = row[field].contents[0].attrs["href"]
-    ## So that model relative path is consistent across OS
+    # So that model relative path is consistent across OS
     rel_path = "/".join(join(source_dir, model_file).split(os.sep))
     if target_models is not None and rel_path not in target_models:
         return None
@@ -261,7 +261,7 @@ def get_model_ports(source_file, metadata, model_name):
             for k, v in get_file_info(row, "model_with_data_path").items():
                 metadata[k] = v
         except (AttributeError, FileNotFoundError) as e:
-            print("no model_with_data in file {}".format(row["source_file"]))
+            print("no model_with_data in file {}: {}".format(row["source_file"], e))
 
         try:
             opset = int(row["opset_version"].contents[0])
@@ -291,7 +291,7 @@ def get_model_ports(source_file, metadata, model_name):
 
     else:
         print("Missing model in {}".format(row["source_file"]))
-output.sort(key=lambda x:x["model_path"])
-with open( "ONNX_HUB_MANIFEST.json", "w+") as f:
+output.sort(key=lambda x: x["model_path"])
+with open("ONNX_HUB_MANIFEST.json", "w+") as f:
     print("Found {} models".format(len(output)))
     json.dump(output, f, indent=4)