Added tests from keras2onnx (#1563)

* Added tests from keras2onnx

Signed-off-by: Tom Wildenhain <[email protected]>

* Add pytest to setup

Signed-off-by: Tom Wildenhain <[email protected]>

* Add keras2onnx api to tf2onnx

Signed-off-by: Tom Wildenhain <[email protected]>

* Mock keras2onnx utility methods

Signed-off-by: Tom Wildenhain <[email protected]>

* Update keras2onnx tests to use ORT 1.8.0

Signed-off-by: Tom Wildenhain <[email protected]>

* Bugfix for Slice

Signed-off-by: Tom Wildenhain <[email protected]>

* Fix RU rewriter

Signed-off-by: Tom Wildenhain <[email protected]>

* Remove tf1.11 tests

Signed-off-by: Tom Wildenhain <[email protected]>

* Improve keras API documentation

Signed-off-by: Tom Wildenhain <[email protected]>

Author: TomWildenhain-Microsoft

ci_build/azure_pipelines/keras2onnx_unit_test.yml

@@ -0,0 +1,76 @@
+# Tests copied from keras2onnx
+
+jobs:
+
+- job: 'Test'
+  pool:
+    vmImage: 'Ubuntu-16.04'
+  strategy:
+    matrix:
+
+      Python36-tf1.15:
+        python.version: '3.6'
+        ONNX_PATH: onnx==1.5.0
+        TENSORFLOW_PATH: tensorflow==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+
+      Python37-tf2.1:
+        python.version: '3.7'
+        ONNX_PATH: onnx==1.6.0
+        TENSORFLOW_PATH: tensorflow-cpu==2.1.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+
+      Python38-tf2.2:
+        python.version: '3.8'
+        ONNX_PATH: onnx==1.7.0
+        TENSORFLOW_PATH: tensorflow-cpu==2.2.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+
+      Python38-tf2.3:
+        python.version: '3.8'
+        ONNX_PATH: onnx==1.8.0
+        TENSORFLOW_PATH: tensorflow-cpu==2.3.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+
+      Python38-tf2.5:
+        python.version: '3.8'
+        ONNX_PATH: onnx==1.8.0
+        TENSORFLOW_PATH: tensorflow-cpu==2.5.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+
+  steps:
+  - script: sudo install -d -m 0777 /home/vsts/.conda/envs
+    displayName: Fix Conda permissions
+
+  - task: CondaEnvironment@1
+    inputs:
+      createCustomEnvironment: true
+      environmentName: 'py$(python.version)'
+      packageSpecs: 'python=$(python.version)'
+
+  - script: |
+      python -m pip install --upgrade pip
+      conda config --set always_yes yes --set changeps1 no
+      pip install $(ONNX_PATH)
+      pip install h5py==2.9.0
+      pip install numpy==1.19
+      pip install $(TENSORFLOW_PATH)
+      pip install git+https://github.com/microsoft/onnxconverter-common
+      pip install -r requirements.txt
+      pip install -r requirements-dev.txt
+      pip install pytest pytest-cov pytest-runner
+      $(INSTALL_ORT)
+    displayName: 'Install dependencies'
+
+  - script: |
+      pip install -e .
+      python -c "import onnxruntime"
+      python -c "import onnxconverter_common"
+      pytest keras2onnx_tests --doctest-modules --junitxml=junit/test-results.xml
+    displayName: 'pytest'
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/test-results.xml'
+      testRunTitle: 'Python $(python.version)'
+    condition: succeededOrFailed()

keras2onnx_tests/conftest.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import pytest
+
+from mock_keras2onnx.proto import keras
+from test_utils import run_onnx_runtime
+
+K = keras.backend
+
+
+@pytest.fixture(scope='function')
+def runner():
+    model_files = []
+
+    def runner_func(*args, **kwargs):
+        return run_onnx_runtime(*args, model_files, **kwargs)
+
+    # Ensure Keras layer naming is reset for each function
+    K.reset_uids()
+    # Reset the TensorFlow session to avoid resource leaking between tests
+    K.clear_session()
+
+    # Provide wrapped run_onnx_runtime function
+    yield runner_func
+
+    # Remove model files
+    for fl in model_files:
+        os.remove(fl)

keras2onnx_tests/mock_keras2onnx/proto/__init__.py

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import tensorflow
+from distutils.version import StrictVersion
+
+# Rather than using ONNX protobuf definition throughout our codebase, we import ONNX protobuf definition here so that
+# we can conduct quick fixes by overwriting ONNX functions without changing any lines elsewhere.
+from onnx import onnx_pb as onnx_proto
+from onnx import helper
+from onnx import save_model as save_model
+
+
+def _check_onnx_version():
+    import pkg_resources
+    min_required_version = pkg_resources.parse_version('1.0.1')
+    current_version = pkg_resources.get_distribution('onnx').parsed_version
+    assert current_version >= min_required_version, 'Keras2ONNX requires ONNX version 1.0.1 or a newer one'
+
+
+_check_onnx_version()
+
+
+def is_tensorflow_older_than(version_str):
+    return StrictVersion(tensorflow.__version__.split('-')[0]) < StrictVersion(version_str)
+
+
+def is_tensorflow_later_than(version_str):
+    return StrictVersion(tensorflow.__version__.split('-')[0]) > StrictVersion(version_str)
+
+
+is_tf_keras = False
+str_tk_keras = os.environ.get('TF_KERAS', None)
+if str_tk_keras is None:
+    # With tensorflow 2.x, be default we loaded tf.keras as the framework, instead of Keras
+    is_tf_keras = not is_tensorflow_older_than('2.0.0')
+else:
+    is_tf_keras = str_tk_keras != '0'
+
+if is_tf_keras:
+    from tensorflow.python import keras
+else:
+    try:
+        import keras
+
+        if keras.Model == tensorflow.keras.Model:  # since keras 2.4, keras and tf.keras is unified.
+            is_tf_keras = True
+    except ImportError:
+        is_tf_keras = True
+        from tensorflow.python import keras
+
+
+def is_keras_older_than(version_str):
+    return StrictVersion(keras.__version__.split('-')[0]) < StrictVersion(version_str)
+
+
+def is_keras_later_than(version_str):
+    return StrictVersion(keras.__version__.split('-')[0]) > StrictVersion(version_str)

keras2onnx_tests/mock_keras2onnx/proto/tfcompat.py

@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import tensorflow as _tf
+
+from distutils.version import StrictVersion
+
+is_tf2 = StrictVersion(_tf.__version__.split('-')[0]) >= StrictVersion('2.0.0')
+
+
+def normalize_tensor_shape(tensor_shape):
+    if is_tf2:
+        return [d for d in tensor_shape]
+    else:
+        return [d.value for d in tensor_shape]
+
+
+def dump_graph_into_tensorboard(tf_graph):
+    # type: (_tf.Graph) -> None
+    _tb_log_dir = os.environ.get('TB_LOG_DIR')
+    if _tb_log_dir:
+        if is_tf2:
+            from tensorflow.python.ops.summary_ops_v2 import graph as write_graph
+            pb_visual_writer = _tf.summary.create_file_writer(_tb_log_dir)
+            with pb_visual_writer.as_default():
+                write_graph(tf_graph)
+        else:
+            from tensorflow.python.summary import summary
+            pb_visual_writer = summary.FileWriter(_tb_log_dir)
+            pb_visual_writer.add_graph(tf_graph)
+
+
+if is_tf2:
+    tensorflow = _tf.compat.v1
+
+    def is_subclassed(layer):
+        """Returns True if the object is a subclassed layer or subclassed model."""
+        return (layer.__module__.find('keras.engine') == -1 and
+                layer.__module__.find('keras.layers') == -1)
+else:
+    tensorflow = _tf
+
+    def is_subclassed(layer):
+        return False

keras2onnx_tests/test_cgan.py

@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+import tensorflow as tf
+import mock_keras2onnx
+import numpy as np
+from mock_keras2onnx.proto import keras, is_tf_keras
+from tf2onnx.keras2onnx_api import convert_keras
+from distutils.version import StrictVersion
+
+Activation = keras.layers.Activation
+BatchNormalization = keras.layers.BatchNormalization
+Dense = keras.layers.Dense
+Dropout = keras.layers.Dropout
+Embedding = keras.layers.Embedding
+Flatten = keras.layers.Flatten
+Input = keras.layers.Input
+LeakyReLU = keras.layers.LeakyReLU
+multiply = keras.layers.multiply
+Reshape = keras.layers.Reshape
+UpSampling2D = keras.layers.UpSampling2D
+
+Sequential = keras.models.Sequential
+Model = keras.models.Model
+
+
+# From https://github.com/eriklindernoren/Keras-GAN/blob/master/cgan/cgan.py
+class CGAN():
+    def __init__(self):
+        # Input shape
+        self.img_rows = 28
+        self.img_cols = 28
+        self.channels = 1
+        self.img_shape = (self.img_rows, self.img_cols, self.channels)
+        self.num_classes = 10
+        self.latent_dim = 100
+
+        # Build and compile the discriminator
+        self.discriminator = self.build_discriminator()
+
+        # Build the generator
+        self.generator = self.build_generator()
+
+        # The generator takes noise and the target label as input
+        # and generates the corresponding digit of that label
+        noise = Input(shape=(self.latent_dim,))
+        label = Input(shape=(1,))
+        img = self.generator([noise, label])
+
+        # For the combined model we will only train the generator
+        self.discriminator.trainable = False
+
+        # The discriminator takes generated image as input and determines validity
+        # and the label of that image
+        valid = self.discriminator([img, label])
+
+        # The combined model  (stacked generator and discriminator)
+        # Trains generator to fool discriminator
+        self.combined = Model([noise, label], valid)
+
+    def get_model(self):
+        return self.combined
+
+    def build_generator(self):
+        model = Sequential()
+
+        model.add(Dense(256, input_dim=self.latent_dim))
+
+        model.add(LeakyReLU(alpha=0.2))
+        model.add(BatchNormalization(momentum=0.8))
+        model.add(Dense(512))
+        model.add(LeakyReLU(alpha=0.2))
+        model.add(BatchNormalization(momentum=0.8))
+        model.add(Dense(1024))
+        model.add(LeakyReLU(alpha=0.2))
+        model.add(BatchNormalization(momentum=0.8))
+
+        model.add(Dense(np.prod(self.img_shape), activation='tanh'))
+        model.add(Reshape(self.img_shape))
+
+        noise = Input(shape=(self.latent_dim,))
+        label = Input(shape=(1,), dtype='int32')
+        label_embedding = Flatten()(Embedding(self.num_classes, self.latent_dim)(label))
+
+        model_input = multiply([noise, label_embedding])
+        img = model(model_input)
+
+        return Model([noise, label], img)
+
+    def build_discriminator(self):
+        model = Sequential()
+
+        model.add(Dense(512, input_dim=np.prod(self.img_shape)))
+        model.add(LeakyReLU(alpha=0.2))
+        model.add(Dense(512))
+        model.add(LeakyReLU(alpha=0.2))
+        model.add(Dropout(0.4))
+        model.add(Dense(512))
+        model.add(LeakyReLU(alpha=0.2))
+        model.add(Dropout(0.4))
+        model.add(Dense(1, activation='sigmoid'))
+
+        model.add(Dense(1, activation='sigmoid'))
+
+        img = Input(shape=self.img_shape)
+        label = Input(shape=(1,), dtype='int32')
+
+        label_embedding = Flatten()(Embedding(self.num_classes, np.prod(self.img_shape))(label))
+        flat_img = Flatten()(img)
+
+        model_input = multiply([flat_img, label_embedding])
+
+        validity = model(model_input)
+
+        return Model([img, label], validity)
+
+
+@pytest.mark.skipif(mock_keras2onnx.proto.tfcompat.is_tf2 and is_tf_keras, reason="Tensorflow 1.x only tests.")
+@pytest.mark.skipif(is_tf_keras and StrictVersion(tf.__version__.split('-')[0]) < StrictVersion("1.14.0"),
+                    reason="Not supported before tensorflow 1.14.0 for tf_keras")
+def test_CGAN(runner):
+    keras_model = CGAN().combined
+    batch = 5
+    x = np.random.rand(batch, 100).astype(np.float32)
+    y = np.random.rand(batch, 1).astype(np.float32)
+    expected = keras_model.predict([x, y])
+    onnx_model = convert_keras(keras_model, keras_model.name)
+    assert runner(onnx_model.graph.name, onnx_model,
+                  {keras_model.input_names[0]: x, keras_model.input_names[1]: y}, expected)