Add windows keras2onnx applications tests (#1598)

* Add windows keras2onnx applications tests

Signed-off-by: Tom Wildenhain <[email protected]>

* Update tf versions

Signed-off-by: Tom Wildenhain <[email protected]>

Author: TomWildenhain-Microsoft

ci_build/azure_pipelines/keras2onnx_application_tests.yml

@@ -5,7 +5,7 @@
 
 jobs:
 
-- job: 'Test'
+- job: 'Linux'
   timeoutInMinutes: 180
   pool:
     vmImage: 'Ubuntu-16.04'
@@ -110,3 +110,99 @@ jobs:
       testResultsFiles: '**/test-results-*.xml'
       testRunTitle: 'Python $(python.version)'
     condition: succeededOrFailed()
+
+- job: 'Win'
+  timeoutInMinutes: 180
+  pool:
+    vmImage: 'vs2017-win2016'
+  strategy:
+    matrix:
+      Python36-onnx1.5:
+        python.version: '3.6'
+        ONNX_PATH: onnx==1.5.0
+        INSTALL_KERAS: pip install keras==2.2.4
+        UNINSTALL_KERAS:
+        INSTALL_TENSORFLOW: pip install tensorflow==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+        INSTALL_KERAS_RESNET: pip install keras-resnet
+        INSTALL_TRANSFORMERS:
+        NIGHTLY_BUILD_TEST: python run_all.py --exclude "test_keras_applications_v2.py test_mask_rcnn.py"
+
+      Python37-onnx1.6:
+        python.version: '3.7'
+        ONNX_PATH: onnx==1.6.0
+        INSTALL_KERAS: pip install keras==2.3.1
+        UNINSTALL_KERAS:
+        INSTALL_TENSORFLOW: pip install tensorflow==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+        INSTALL_KERAS_RESNET: pip install keras-resnet
+        INSTALL_TRANSFORMERS:
+        NIGHTLY_BUILD_TEST: python run_all.py --exclude "test_keras_applications_v2.py test_mask_rcnn.py"
+
+      Python37-onnx1.9:
+        python.version: '3.7'
+        ONNX_PATH: onnx==1.9.0
+        INSTALL_KERAS: pip install keras==2.3.1
+        UNINSTALL_KERAS:
+        INSTALL_TENSORFLOW: pip install tensorflow==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+        INSTALL_KERAS_RESNET: pip install keras-resnet
+        INSTALL_TRANSFORMERS:
+        NIGHTLY_BUILD_TEST: python run_all.py --exclude "test_keras_applications_v2.py test_mask_rcnn.py"
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+      architecture: 'x64'
+
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: Add conda to PATH
+
+  - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf
+    displayName: Create Anaconda environment
+
+  - script: |
+      call activate py$(python.version)
+      python -m pip install --upgrade pip numpy
+      echo Test numpy installation... && python -c "import numpy"
+      pip install %ONNX_PATH%
+      pip uninstall -y protobuf
+      pip install protobuf
+      pip install h5py==2.9.0
+      pip install numpy==1.19
+      pip install parameterized
+      %INSTALL_TENSORFLOW%
+      %INSTALL_KERAS%
+      pip install git+https://github.com/microsoft/onnxconverter-common
+      %INSTALL_ORT%
+      pip install Pillow==8.2.0
+      pip install opencv-python
+      pip install tqdm
+      pip install keras-segmentation==0.2.0
+      pip install matplotlib
+      git clone https://github.com/qqwweee/keras-yolo3
+      %INSTALL_KERAS_RESNET%
+      pip install git+https://www.github.com/keras-team/keras-contrib.git
+      pip install keras-tcn==2.8.3
+      %UNINSTALL_KERAS%
+      pip install git+https://github.com/qubvel/efficientnet
+      %INSTALL_TRANSFORMERS%
+      pip install keras-self-attention
+      pip install pytest pytest-cov pytest-runner
+    displayName: 'Install dependencies'
+
+  - script: |
+      call activate py$(python.version)
+      pip install -e .
+      echo Test onnxruntime installation... && python -c "import onnxruntime"
+      pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
+      cd tests/keras2onnx_applications/nightly_build
+      %NIGHTLY_BUILD_TEST%
+    displayName: 'pytest'
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/test-results-*.xml'
+      testRunTitle: 'Python $(python.version)'
+    condition: succeededOrFailed()

ci_build/azure_pipelines/templates/unit_test.yml

@@ -12,8 +12,7 @@ steps:
       export TF2ONNX_TEST_OPSET=$CI_ONNX_OPSET
       export TF2ONNX_SKIP_TFLITE_TESTS=$CI_SKIP_TFLITE_TESTS
       export TF2ONNX_SKIP_TF_TESTS=$CI_SKIP_TF_TESTS
-      python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append \
-        --ignore tests/keras2onnx_unit_tests --ignore tests/keras2onnx_applications
+      python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append
     timeoutInMinutes: 15
     displayName: ${{ format('Run UnitTest - Opset{0}', onnx_opset) }}
     condition: succeededOrFailed()

setup.cfg

@@ -5,3 +5,4 @@ test=pytest
 
 [tool:pytest]
 addopts=--cov=tf2onnx
+norecursedirs=tests/keras2onnx_applications tests/keras2onnx_unit_tests

tests/keras2onnx_applications/nightly_build/run_all.py

@@ -12,6 +12,7 @@
 
 os.environ["PYTHONPATH"] = \
     os.environ.get("PYTHONPATH", "") + os.pathsep + "../../keras2onnx_unit_tests" + os.pathsep + "../../../"
+os.environ["TF2ONNX_CATCH_ERRORS"] = "FALSE"
 
 mypath = '.'
 files = [f for f in listdir(mypath) if isfile(join(mypath, f)) and f.find("test_") == 0]

tests/keras2onnx_applications/nightly_build/run_all_v2.py

@@ -3,6 +3,7 @@
 import os
 os.environ["PYTHONPATH"] = \
     os.environ.get("PYTHONPATH", "") + os.pathsep + "../../keras2onnx_unit_tests" + os.pathsep + "../../../"
+os.environ["TF2ONNX_CATCH_ERRORS"] = "FALSE"
 
 files = ['test_keras_applications_v2.py', 'test_transformers.py', 'test_chatbot.py', 'test_efn.py', \
          'test_resnext.py']

tests/keras2onnx_unit_tests/conftest.py

@@ -4,16 +4,20 @@
 import pytest
 
 import numpy as np
+import tensorflow as tf
 
-from mock_keras2onnx.proto import keras
+from mock_keras2onnx.proto import keras, is_tf_keras
 from test_utils import run_onnx_runtime
+from mock_keras2onnx.proto.tfcompat import is_tf2
 
 K = keras.backend
 
 
 @pytest.fixture(scope='function')
 def runner():
     np.random.seed(42)
+    if is_tf_keras and is_tf2:
+        tf.random.set_seed(42)
     model_files = []
 
     def runner_func(*args, **kwargs):

tf2onnx/constants.py

@@ -43,6 +43,7 @@
 
 # Environment variables
 ENV_TF2ONNX_DEBUG_MODE = "TF2ONNX_DEBUG_MODE"
+ENV_TF2ONNX_CATCH_ERRORS = "TF2ONNX_CATCH_ERRORS"
 
 # Mapping opset to IR version.
 # Note: opset 7 and opset 8 came out with IR3 but we need IR4 because of PlaceholderWithDefault

tf2onnx/convert.py

@@ -152,7 +152,11 @@ def _convert_common(frozen_graph, name="unknown", large_model=False, output_path
         if not kwargs.get("tflite_path"):
             tf.import_graph_def(frozen_graph, name='')
         g = process_tf_graph(tf_graph, const_node_values=const_node_values, **kwargs)
-        onnx_graph = optimizer.optimize_graph(g, catch_errors=not large_model)
+        if constants.ENV_TF2ONNX_CATCH_ERRORS in os.environ:
+            catch_errors = constants.ENV_TF2ONNX_CATCH_ERRORS.upper() == "TRUE"
+        else:
+            catch_errors = not large_model
+        onnx_graph = optimizer.optimize_graph(g, catch_errors)
         model_proto = onnx_graph.make_model("converted from {}".format(name),
                                             external_tensor_storage=external_tensor_storage)
     if output_path:

tf2onnx/optimizer/__init__.py

@@ -43,7 +43,7 @@ def _get_optimizers():
 
 
 def optimize_graph(graph, catch_errors=True, optimizers=None):
-    """ Optimize graph, return optimized graph. No throw if catch_errors is true"""
+    """ Optimize graph, return optimized graph. Catch errors and restore old graph if catch_errors is True"""
     logger = logging.getLogger(__name__)
     logger.info("Optimizing ONNX model")