Merge pull request #324 from nbcsm/ci

fix CI failure, refine test code

Author: nbcsm

ci_build/azure_pipelines/pretrained_model_test-matrix.yml

@@ -5,7 +5,6 @@ jobs:
   parameters:
     platforms: ['linux', 'windows', 'mac']
     tf_versions: ['1.12', '1.11', '1.10', '1.9', '1.8', '1.7', '1.6', '1.5']
-    onnx_versions: ['1.3']
     onnx_opsets: ['8', '7']
     onnx_backends:
       onnxruntime: ['0.2.1']

ci_build/azure_pipelines/templates/setup.yml

@@ -5,6 +5,14 @@ steps:
     set -ex
     pip install pytest pytest-cov pytest-runner graphviz requests pyyaml pillow pandas
     pip install $(CI_PIP_TF_NAME) $(CI_PIP_ONNX_NAME) $(CI_PIP_ONNX_BACKEND_NAME)
+
+    # TF 1.10 requires numpy <=1.14.5 and >=1.13.3, but onnxruntime 0.2.1 does not work with numpy <= 1.14.5
+    # Upgrade numpy only within constraints from other packages if any.
+    if [[ $CI_TF_VERSION == 1.10* ]] && [[ $CI_ONNX_BACKEND == "onnxruntime" ]] ;
+    then
+        pip install $(CI_PIP_ONNX_NAME) $(CI_PIP_ONNX_BACKEND_NAME) numpy --no-deps -U
+    fi
+
     python setup.py install
     pip freeze --all
   displayName: 'Setup Environment'

ci_build/azure_pipelines/unit_test-matrix.yml

@@ -5,7 +5,6 @@ jobs:
   parameters:
     platforms: ['linux', 'windows', 'mac']
     tf_versions: ['1.12', '1.11', '1.10', '1.9', '1.8', '1.7', '1.6', '1.5']
-    onnx_versions: ['1.3']
     onnx_opsets: ['8', '7']
     onnx_backends:
       onnxruntime: ['0.2.1']

setup.cfg

@@ -2,5 +2,4 @@
 test=pytest
 
 [tool:pytest]
-addopts=--cov=tf2onnx --ignore=tests/test_custom_rnncell.py --ignore=tests/test_const_fold.py --ignore=tests/test_loops.py
-#testpaths=tests/test_*.py
+addopts=--cov=tf2onnx

tests/backend_test_base.py

@@ -8,42 +8,31 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import argparse
 import logging
 import os
-import sys
-import tempfile
 import unittest
 
 import numpy as np
 import tensorflow as tf
 from tensorflow.python.ops import variables as variables_lib
+from common import get_test_config
 from tf2onnx import utils
-from tf2onnx.tfonnx import process_tf_graph, tf_optimize, DEFAULT_TARGET, POSSIBLE_TARGETS
+from tf2onnx.tfonnx import process_tf_graph, tf_optimize
 
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,using-constant-test
 
 class Tf2OnnxBackendTestBase(unittest.TestCase):
-    # static variables
-    TMPPATH = tempfile.mkdtemp()
-    BACKEND = os.environ.get("TF2ONNX_TEST_BACKEND", "onnxruntime")
-    OPSET = int(os.environ.get("TF2ONNX_TEST_OPSET", 7))
-    TARGET = os.environ.get("TF2ONNX_TEST_TARGET", "").split(",")
-    DEBUG = None
-
-    def debug_mode(self):
-        return type(self).DEBUG
-
     def setUp(self):
+        self.config = get_test_config()
         self.maxDiff = None
         tf.reset_default_graph()
         # reset name generation on every test
         utils.INTERNAL_NAME = 1
         np.random.seed(1)  # Make it reproducible.
 
         self.log = logging.getLogger("tf2onnx.unitest." + str(type(self)))
-        if self.debug_mode():
+        if self.config.is_debug_mode:
             self.log.setLevel(logging.DEBUG)
         else:
             # suppress log info of tensorflow so that result of test can be seen much easier
@@ -83,17 +72,17 @@ def run_onnxruntime(self, model_path, inputs, output_names):
     def _run_backend(self, g, outputs, input_dict):
         model_proto = g.make_model("test")
         model_path = self.save_onnx_model(model_proto, input_dict)
-        if type(self).BACKEND == "onnxmsrtnext":
+        if self.config.backend == "onnxmsrtnext":
             y = self.run_onnxmsrtnext(model_path, input_dict, outputs)
-        elif type(self).BACKEND == "onnxruntime":
+        elif self.config.backend == "onnxruntime":
             y = self.run_onnxruntime(model_path, input_dict, outputs)
-        elif type(self).BACKEND == "caffe2":
+        elif self.config.backend == "caffe2":
             y = self.run_onnxcaffe2(model_proto, input_dict)
         else:
             raise ValueError("unknown backend")
         return y
 
-    def run_test_case(self, feed_dict, input_names_with_port, output_names_with_port, rtol=1e-07,
+    def run_test_case(self, feed_dict, input_names_with_port, output_names_with_port, rtol=1e-07, atol=0.,
                       convert_var_to_const=True, constant_fold=True, check_value=True, check_shape=False,
                       check_dtype=False, process_args=None, onnx_feed_dict=None):
         # optional - passed to process_tf_graph
@@ -104,7 +93,7 @@ def run_test_case(self, feed_dict, input_names_with_port, output_names_with_port
             onnx_feed_dict = feed_dict
 
         graph_def = None
-        save_dir = os.path.join(type(self).TMPPATH, self._testMethodName)
+        save_dir = os.path.join(self.config.temp_path, self._testMethodName)
 
         if convert_var_to_const:
             with tf.Session() as sess:
@@ -123,7 +112,7 @@ def run_test_case(self, feed_dict, input_names_with_port, output_names_with_port
                 output_dict.append(sess.graph.get_tensor_by_name(out_name))
             expected = sess.run(output_dict, feed_dict=feed_dict)
 
-        if self.debug_mode():
+        if self.config.is_debug_mode:
             if not os.path.exists(save_dir):
                 os.makedirs(save_dir)
             model_path = os.path.join(save_dir, self._testMethodName + "_original.pb")
@@ -134,7 +123,7 @@ def run_test_case(self, feed_dict, input_names_with_port, output_names_with_port
         graph_def = tf_optimize(input_names_with_port, output_names_with_port,
                                 sess.graph_def, constant_fold)
 
-        if self.debug_mode() and constant_fold:
+        if self.config.is_debug_mode and constant_fold:
             model_path = os.path.join(save_dir, self._testMethodName + "_after_tf_optimize.pb")
             with open(model_path, "wb") as f:
                 f.write(graph_def.SerializeToString())
@@ -144,45 +133,23 @@ def run_test_case(self, feed_dict, input_names_with_port, output_names_with_port
         tf.import_graph_def(graph_def, name='')
 
         with tf.Session() as sess:
-            g = process_tf_graph(sess.graph, opset=type(self).OPSET, output_names=output_names_with_port,
-                                 target=type(self).TARGET, **process_args)
+            g = process_tf_graph(sess.graph, opset=self.config.opset, output_names=output_names_with_port,
+                                 target=self.config.target, **process_args)
             actual = self._run_backend(g, output_names_with_port, onnx_feed_dict)
 
         for expected_val, actual_val in zip(expected, actual):
             if check_value:
-                self.assertAllClose(expected_val, actual_val, rtol=rtol, atol=0.)
+                self.assertAllClose(expected_val, actual_val, rtol=rtol, atol=atol)
             if check_dtype:
                 self.assertEqual(expected_val.dtype, actual_val.dtype)
             if check_shape:
                 self.assertEqual(expected_val.shape, actual_val.shape)
 
     def save_onnx_model(self, model_proto, feed_dict, postfix=""):
-        save_path = os.path.join(type(self).TMPPATH, self._testMethodName)
+        save_path = os.path.join(self.config.temp_path, self._testMethodName)
         target_path = utils.save_onnx_model(save_path, self._testMethodName + postfix, feed_dict, model_proto,
-                                            include_test_data=self.debug_mode(), as_text=self.debug_mode())
+                                            include_test_data=self.config.is_debug_mode,
+                                            as_text=self.config.is_debug_mode)
 
         self.log.debug("create model file: %s", target_path)
         return target_path
-
-    @staticmethod
-    def trigger(ut_class):
-        parser = argparse.ArgumentParser()
-        parser.add_argument('--backend', default=Tf2OnnxBackendTestBase.BACKEND,
-                            choices=["caffe2", "onnxmsrtnext", "onnxruntime"],
-                            help="backend to test against")
-        parser.add_argument('--opset', type=int, default=Tf2OnnxBackendTestBase.OPSET, help="opset to test against")
-        parser.add_argument("--target", default=",".join(DEFAULT_TARGET), choices=POSSIBLE_TARGETS,
-                            help="target platform")
-        parser.add_argument("--debug", help="output debugging information", action="store_true")
-        parser.add_argument('unittest_args', nargs='*')
-
-        args = parser.parse_args()
-        print(args)
-        ut_class.BACKEND = args.backend
-        ut_class.OPSET = args.opset
-        ut_class.DEBUG = args.debug
-        ut_class.TARGET = args.target
-
-        # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
-        sys.argv[1:] = args.unittest_args
-        unittest.main()

tests/common.py

@@ -0,0 +1,206 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT license.
+
+""" test common utilities."""
+
+import argparse
+import os
+import sys
+import tempfile
+import unittest
+
+from distutils.version import LooseVersion
+from tf2onnx.tfonnx import DEFAULT_TARGET, POSSIBLE_TARGETS
+
+__all__ = ["TestConfig", "get_test_config", "unittest_main",
+           "check_tf_min_version", "check_opset_min_version", "check_target", "skip_onnxruntime_backend",
+           "skip_caffe2_backend", "check_onnxruntime_incompatibility"]
+
+
+# pylint: disable=missing-docstring
+
+class TestConfig(object):
+    def __init__(self):
+        self.platform = sys.platform
+        self.tf_version = self._get_tf_version()
+        self.opset = int(os.environ.get("TF2ONNX_TEST_OPSET", 7))
+        self.target = os.environ.get("TF2ONNX_TEST_TARGET", ",".join(DEFAULT_TARGET)).split(',')
+        self.backend = os.environ.get("TF2ONNX_TEST_BACKEND", "onnxruntime")
+        self.backend_version = self._get_backend_version()
+        self.is_debug_mode = False
+        self.temp_path = tempfile.mkdtemp()
+
+    @property
+    def is_mac(self):
+        return self.platform == "darwin"
+
+    @property
+    def is_onnxruntime_backend(self):
+        return self.backend == "onnxruntime"
+
+    @property
+    def is_caffe2_backend(self):
+        return self.backend == "caffe2"
+
+    def _get_tf_version(self):
+        import tensorflow as tf
+        return LooseVersion(tf.__version__)
+
+    def _get_backend_version(self):
+        version = None
+        if self.backend == "onnxruntime":
+            import onnxruntime as ort
+            version = ort.__version__
+        elif self.backend == "caffe2":
+            # TODO: get caffe2 version
+            pass
+
+        if version:
+            version = LooseVersion(version)
+        return version
+
+    def __str__(self):
+        return "\n\t".join(["TestConfig:",
+                            "platform={}".format(self.platform),
+                            "tf_version={}".format(self.tf_version),
+                            "opset={}".format(self.opset),
+                            "target={}".format(self.target),
+                            "backend={}".format(self.backend),
+                            "backend_version={}".format(self.backend_version),
+                            "is_debug_mode={}".format(self.is_debug_mode)])
+
+    @staticmethod
+    def load():
+        config = TestConfig()
+        # if not launched by pytest, parse console arguments to override config
+        if "pytest" not in sys.argv[0]:
+            parser = argparse.ArgumentParser()
+            parser.add_argument('--backend', default=config.backend,
+                                choices=["caffe2", "onnxmsrtnext", "onnxruntime"],
+                                help="backend to test against")
+            parser.add_argument('--opset', type=int, default=config.opset, help="opset to test against")
+            parser.add_argument("--target", default=",".join(config.target), choices=POSSIBLE_TARGETS,
+                                help="target platform")
+            parser.add_argument("--debug", help="output debugging information", action="store_true")
+            parser.add_argument('unittest_args', nargs='*')
+
+            args = parser.parse_args()
+            config.backend = args.backend
+            config.opset = args.opset
+            config.target = args.target.split(',')
+            config.is_debug_mode = args.debug
+
+            # Now set the sys.argv to the unittest_args (leaving sys.argv[0] alone)
+            sys.argv[1:] = args.unittest_args
+
+        return config
+
+
+# need to load config BEFORE main is executed when launched from script
+# otherwise, it will be too late for test filters to take effect
+_config = TestConfig.load()
+
+
+def get_test_config():
+    global _config
+    return _config
+
+
+def unittest_main():
+    print(get_test_config())
+    unittest.main()
+
+
+def _append_message(reason, message):
+    if message:
+        reason = reason + ": " + message
+    return reason
+
+
+def check_tf_min_version(min_required_version, message=""):
+    """ Skip if tf_version < min_required_version """
+    config = get_test_config()
+    reason = _append_message("conversion requires tf >= {}".format(min_required_version), message)
+    return unittest.skipIf(config.tf_version < LooseVersion(min_required_version), reason)
+
+
+def skip_tf_versions(excluded_versions, message=""):
+    """ Skip if tf_version SEMANTICALLY matches any of excluded_versions. """
+    config = get_test_config()
+    condition = False
+    reason = _append_message("conversion excludes tf {}".format(excluded_versions), message)
+
+    current_tokens = str(config.tf_version).split('.')
+    for excluded_version in excluded_versions:
+        exclude_tokens = excluded_version.split('.')
+        # assume len(exclude_tokens) <= len(current_tokens)
+        for i, exclude in enumerate(exclude_tokens):
+            if not current_tokens[i] == exclude:
+                break
+        condition = True
+
+    return unittest.skipIf(condition, reason)
+
+
+def check_opset_min_version(min_required_version, message=""):
+    """ Skip if opset < min_required_version """
+    config = get_test_config()
+    reason = _append_message("conversion requires opset >= {}".format(min_required_version), message)
+    return unittest.skipIf(config.opset < min_required_version, reason)
+
+
+def check_target(required_target, message=""):
+    """ Skip if required_target is NOT specified """
+    config = get_test_config()
+    reason = _append_message("conversion requires target {} specified".format(required_target), message)
+    return unittest.skipIf(required_target not in config.target, reason)
+
+
+def skip_onnxruntime_backend(message=""):
+    """ Skip if backend is onnxruntime """
+    config = get_test_config()
+    reason = _append_message("not supported by onnxruntime", message)
+    return unittest.skipIf(config.is_onnxruntime_backend, reason)
+
+
+def skip_caffe2_backend(message=""):
+    """ Skip if backend is caffe2 """
+    config = get_test_config()
+    reason = _append_message("not supported by caffe2", message)
+    return unittest.skipIf(config.is_caffe2_backend, reason)
+
+
+def check_onnxruntime_incompatibility(op):
+    """ Skip if backend is onnxruntime AND op is NOT supported in current opset """
+    config = get_test_config()
+
+    if not config.is_onnxruntime_backend:
+        return unittest.skipIf(False, None)
+
+    support_since = {
+        "Abs": 6,  # Abs-1
+        "Add": 7,  # Add-1, Add-6
+        "AveragePool": 7,  # AveragePool-1
+        "Div": 7,  # Div-1, Div-6
+        "Elu": 6,  # Elu-1
+        "Exp": 6,  # Exp-1
+        "Greater": 7,  # Greater-1
+        "Less": 7,  # Less-1
+        "Log": 6,  # Log-1
+        "Max": 6,  # Max-1
+        "Min": 6,  # Min-1
+        "Mul": 7,  # Mul-1, Mul-6
+        "Neg": 6,  # Neg-1
+        "Pow": 7,  # Pow-1
+        "Reciprocal": 6,  # Reciprocal-1
+        "Relu": 6,  # Relu-1
+        "Sqrt": 6,  # Sqrt-1
+        "Sub": 7,  # Sub-1, Sub-6
+        "Tanh": 6,  # Tanh-1
+    }
+
+    if op not in support_since or config.opset >= support_since[op]:
+        return unittest.skipIf(False, None)
+
+    reason = "{} is not supported by onnxruntime before opset {}".format(op, support_since[op])
+    return unittest.skipIf(True, reason)