Tom/keras2onnx application tests (#1590)

* Add keras2onnx application tests

Signed-off-by: Tom Wildenhain <[email protected]>

* Pin Pillow version

Signed-off-by: Tom Wildenhain <[email protected]>

* Revmoe coverage reporting and old onnx version tests

Signed-off-by: Tom Wildenhain <[email protected]>

Co-authored-by: Guenther Schmuelling <[email protected]>

Author: TomWildenhain-Microsoft

ci_build/azure_pipelines/keras2onnx_application_tests.yml

@@ -0,0 +1,112 @@
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+jobs:
+
+- job: 'Test'
+  timeoutInMinutes: 180
+  pool:
+    vmImage: 'Ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36-onnx1.5:
+        python.version: '3.6'
+        ONNX_PATH: onnx==1.5.0
+        INSTALL_KERAS: pip install keras==2.2.4
+        UNINSTALL_KERAS:
+        INSTALL_TENSORFLOW: pip install tensorflow==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+        INSTALL_KERAS_RESNET: pip install keras-resnet
+        INSTALL_TRANSFORMERS:
+        NIGHTLY_BUILD_TEST: python run_all.py --exclude "test_keras_applications_v2.py"
+
+      Python37-onnx1.6:
+        python.version: '3.7.3'
+        ONNX_PATH: onnx==1.6.0
+        INSTALL_KERAS: pip install keras==2.3.1
+        UNINSTALL_KERAS:
+        INSTALL_TENSORFLOW: pip install tensorflow==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+        INSTALL_KERAS_RESNET: pip install keras-resnet
+        INSTALL_TRANSFORMERS:
+        NIGHTLY_BUILD_TEST: python run_all.py --exclude "test_keras_applications_v2.py"
+
+      Python37-onnx1.9:
+        python.version: '3.7.3'
+        ONNX_PATH: onnx==1.9.0
+        INSTALL_KERAS: pip install keras==2.3.1
+        UNINSTALL_KERAS:
+        INSTALL_TENSORFLOW: pip install tensorflow==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+        INSTALL_KERAS_RESNET: pip install keras-resnet
+        INSTALL_TRANSFORMERS:
+        NIGHTLY_BUILD_TEST: python run_all.py --exclude "test_keras_applications_v2.py"
+
+      Python38-tf2:
+        python.version: '3.8'
+        ONNX_PATH: onnx==1.9.0
+        INSTALL_KERAS:
+        UNINSTALL_KERAS: pip uninstall keras -y
+        INSTALL_TENSORFLOW: pip install tensorflow==2.2.0
+        INSTALL_ORT: pip install onnxruntime==1.8.0
+        INSTALL_KERAS_RESNET: pip install keras-resnet
+        INSTALL_TRANSFORMERS: pip install transformers==3.4.0
+        NIGHTLY_BUILD_TEST: python run_all_v2.py
+
+  steps:
+  - script: sudo install -d -m 0777 /home/vsts/.conda/envs
+    displayName: Fix Conda permissions
+
+  - task: CondaEnvironment@1
+    inputs:
+      createCustomEnvironment: true
+      environmentName: 'py$(python.version)'
+      packageSpecs: 'python=$(python.version)'
+
+  - script: |
+      python -m pip install --upgrade pip
+      conda config --set always_yes yes --set changeps1 no
+      pip install $(ONNX_PATH)
+      pip install h5py==2.9.0
+      pip install numpy==1.19
+      pip install parameterized
+      $(INSTALL_TENSORFLOW)
+      $(INSTALL_KERAS)
+      pip install git+https://github.com/microsoft/onnxconverter-common
+      $(INSTALL_ORT)
+      pip install Pillow==8.2.0
+      pip install opencv-python
+      pip install tqdm
+      pip install keras-segmentation==0.2.0
+      git clone https://github.com/matterport/Mask_RCNN
+      cd Mask_RCNN
+      pip install -r requirements.txt
+      python setup.py install
+      cd ..
+      pip install matplotlib
+      git clone https://github.com/qqwweee/keras-yolo3
+      $(INSTALL_KERAS_RESNET)
+      pip install git+https://www.github.com/keras-team/keras-contrib.git
+      pip install keras-tcn==2.8.3
+      $(UNINSTALL_KERAS)
+      pip install git+https://github.com/qubvel/efficientnet
+      $(INSTALL_TRANSFORMERS)
+      pip install keras-self-attention
+      pip install pytest pytest-cov pytest-runner
+    displayName: 'Install dependencies'
+
+  - script: |
+      pip install -e .
+      python -c "import onnxruntime"
+      pytest tests/keras2onnx_unit_tests --doctest-modules --junitxml=junit/test-results.xml
+      cd tests/keras2onnx_applications/nightly_build
+      $(NIGHTLY_BUILD_TEST)
+    displayName: 'pytest'
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/test-results-*.xml'
+      testRunTitle: 'Python $(python.version)'
+    condition: succeededOrFailed()

ci_build/azure_pipelines/templates/unit_test.yml

@@ -13,7 +13,7 @@ steps:
       export TF2ONNX_SKIP_TFLITE_TESTS=$CI_SKIP_TFLITE_TESTS
       export TF2ONNX_SKIP_TF_TESTS=$CI_SKIP_TF_TESTS
       python -m pytest --cov=tf2onnx --cov-report=term --disable-pytest-warnings -r s tests --cov-append \
-        --ignore tests/keras2onnx_unit_tests
+        --ignore tests/keras2onnx_unit_tests --ignore tests/keras2onnx_applications
     timeoutInMinutes: 15
     displayName: ${{ format('Run UnitTest - Opset{0}', onnx_opset) }}
     condition: succeededOrFailed()

tests/keras2onnx_applications/.gitignore

@@ -0,0 +1,2 @@
+*.h5
+*.onnx

tests/keras2onnx_applications/data/street.jpg

@@ -0,0 +1,11 @@
+<!--- SPDX-License-Identifier: Apache-2.0 -->
+
+# Introduction
+This tool converts the lpcnet model to onnx.
+To run this code, we need first install the original lpcnet model from <https://github.com/mozilla/LPCNet/>.
+Note that lpcnet is not a package, so please add its directory to the path.
+Then run
+```
+python convert_lpcnet_to_onnx.py [model_file]
+```
+model_file is the model with trained weights, it is a *.h5 file.

tests/keras2onnx_applications/lpcnet/README.md

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import lpcnet
+import sys
+
+model, enc, dec = lpcnet.new_lpcnet_model(use_gpu=False)
+model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
+model_file = sys.argv[1]
+model.load_weights(model_file)
+
+import mock_keras2onnx
+oxml_enc = mock_keras2onnx.convert_keras(enc, 'lpcnet_enc')
+oxml_dec = mock_keras2onnx.convert_keras(dec, 'lpcnet_dec')
+
+import onnx
+onnx.save(oxml_enc, "lpcnet_enc.onnx")
+onnx.save(oxml_dec, "lpcnet_dec.onnx")

tests/keras2onnx_applications/lpcnet/convert_lpcnet_to_onnx.py

@@ -0,0 +1,17 @@
+<!--- SPDX-License-Identifier: Apache-2.0 -->
+
+# Introduction
+The original Keras project of Masked RCNN is: <https://github.com/matterport/Mask_RCNN>. Follow the 'Installation' section in README.md to set up the model.
+There is also a good [tutorial](https://github.com/matterport/Mask_RCNN#step-by-step-detection) to learn about the object detection.
+
+The conversion supports since opset 11, And the converted model need to be working with ONNXRuntime latest version which supports ONNX opset 11 and contrib ops needed by this model.
+
+# Convert and Run the model.
+```
+cd <mask_rcnn directory>
+pip install -e .
+cd <mock_keras2onnx directory>/applications/mask_rcnn
+# convert the model to onnx and test it with an image.
+python mask_rcnn.py <image_file_path>
+```
+The unit test is added in our nightly build, see [here](https://github.com/onnx/keras-onnx/blob/master/applications/nightly_build/test_mask_rcnn.py)

tests/keras2onnx_applications/mask_rcnn/README.md

@@ -0,0 +1,217 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+import numpy as np
+import skimage
+import onnx
+import mock_keras2onnx
+
+from mrcnn.config import Config
+from mrcnn.model import BatchNorm, DetectionLayer
+from mrcnn import model as modellib
+from mrcnn import visualize
+
+from mock_keras2onnx import set_converter
+from mock_keras2onnx.ke2onnx.batch_norm import convert_keras_batch_normalization
+from os.path import dirname, abspath
+sys.path.insert(0, os.path.join(dirname(abspath(__file__)), '../../keras2onnx_tests/'))
+from test_utils import convert_tf_crop_and_resize
+
+
+ROOT_DIR = os.path.abspath("./")
+
+# Directory to save logs and trained model
+MODEL_DIR = os.path.join(ROOT_DIR, "logs")
+
+# Path to trained weights file
+COCO_MODEL_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")
+
+
+class CocoConfig(Config):
+    """Configuration for training on MS COCO.
+    Derives from the base Config class and overrides values specific
+    to the COCO dataset.
+    """
+    # Give the configuration a recognizable name
+    NAME = "coco"
+
+    # We use a GPU with 12GB memory, which can fit two images.
+    # Adjust down if you use a smaller GPU.
+    IMAGES_PER_GPU = 2
+
+    # Uncomment to train on 8 GPUs (default is 1)
+    # GPU_COUNT = 8
+
+    # Number of classes (including background)
+    NUM_CLASSES = 1 + 80  # COCO has 80 classes
+
+
+class InferenceConfig(CocoConfig):
+    # Set batch size to 1 since we'll be running inference on
+    # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
+    GPU_COUNT = 1
+    IMAGES_PER_GPU = 1
+
+
+config = InferenceConfig()
+config.display()
+
+model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=config)
+
+# Load weights trained on MS-COCO
+model.load_weights(COCO_MODEL_PATH, by_name=True)
+
+
+def convert_BatchNorm(scope, operator, container):
+    convert_keras_batch_normalization(scope, operator, container)
+
+
+def norm_boxes_graph(scope, operator, container, oopb, image_meta):
+    image_shapes = oopb.add_node('Slice',
+                         [image_meta,
+                          ('_start', oopb.int64, np.array([4], dtype='int64')),
+                          ('_end', oopb.int64, np.array([7], dtype='int64')),
+                          ('_axes', oopb.int64, np.array([1], dtype='int64'))
+                          ],
+                         operator.inputs[0].full_name + '_image_shapes')
+    image_shape = oopb.add_node('Slice',
+                                 [image_shapes,
+                                  ('_start', oopb.int64, np.array([0], dtype='int64')),
+                                  ('_end', oopb.int64, np.array([1], dtype='int64')),
+                                  ('_axes', oopb.int64, np.array([0], dtype='int64'))
+                                  ],
+                                 operator.inputs[0].full_name + '_image_shape')
+    image_shape_squeeze = oopb.apply_squeeze(image_shape, name=operator.full_name + '_image_shape_squeeze', axes=[0])[0]
+
+    window = oopb.add_node('Slice',
+                            [image_meta,
+                             ('_start', oopb.int64, np.array([7], dtype='int64')),
+                             ('_end', oopb.int64, np.array([11], dtype='int64')),
+                             ('_axes', oopb.int64, np.array([1], dtype='int64'))
+                             ],
+                            operator.inputs[0].full_name + '_window')
+    h_norm = oopb.add_node('Slice',
+                         [image_shape_squeeze,
+                          ('_start', oopb.int64, np.array([0], dtype='int64')),
+                          ('_end', oopb.int64, np.array([1], dtype='int64')),
+                          ('_axes', oopb.int64, np.array([0], dtype='int64'))
+                          ],
+                         operator.inputs[0].full_name + '_h_norm')
+    w_norm = oopb.add_node('Slice',
+                           [image_shape_squeeze,
+                            ('_start', oopb.int64, np.array([1], dtype='int64')),
+                            ('_end', oopb.int64, np.array([2], dtype='int64')),
+                            ('_axes', oopb.int64, np.array([0], dtype='int64'))
+                            ],
+                           operator.inputs[0].full_name + '_w_norm')
+    h_norm_float = scope.get_unique_variable_name('h_norm_float')
+    attrs = {'to': 1}
+    container.add_node('Cast', h_norm, h_norm_float, op_version=operator.target_opset,
+                       **attrs)
+    w_norm_float = scope.get_unique_variable_name('w_norm_float')
+    attrs = {'to': 1}
+    container.add_node('Cast', w_norm, w_norm_float, op_version=operator.target_opset,
+                       **attrs)
+    hw_concat = scope.get_unique_variable_name(operator.inputs[0].full_name + '_hw_concat')
+    attrs = {'axis': -1}
+    container.add_node("Concat",
+                       [h_norm_float, w_norm_float, h_norm_float, w_norm_float],
+                       hw_concat,
+                       op_version=operator.target_opset,
+                       name=operator.inputs[0].full_name + '_hw_concat', **attrs)
+    scale = oopb.add_node('Sub',
+                          [hw_concat,
+                           ('_sub', oopb.float, np.array([1.0], dtype='float32'))
+                           ],
+                          operator.inputs[0].full_name + '_scale')
+    boxes_shift = oopb.add_node('Sub',
+                          [window,
+                           ('_sub', oopb.float, np.array([0.0, 0.0, 1.0, 1.0], dtype='float32'))
+                           ],
+                          operator.inputs[0].full_name + '_boxes_shift')
+    divide = oopb.add_node('Div',
+                            [boxes_shift, scale],
+                            operator.inputs[0].full_name + '_divide')
+    # output shape: [batch, 4]
+    return divide
+
+
+def convert_DetectionLayer(scope, operator, container):
+    # type: (mock_keras2onnx.common.InterimContext, mock_keras2onnx.common.Operator, mock_keras2onnx.common.OnnxObjectContainer) -> None
+    pass
+
+
+set_converter(DetectionLayer, convert_DetectionLayer)
+set_converter(BatchNorm, convert_BatchNorm)
+
+
+# Run detection
+class_names = ['BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane',
+               'bus', 'train', 'truck', 'boat', 'traffic light',
+               'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird',
+               'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear',
+               'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',
+               'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball',
+               'kite', 'baseball bat', 'baseball glove', 'skateboard',
+               'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup',
+               'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',
+               'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza',
+               'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed',
+               'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote',
+               'keyboard', 'cell phone', 'microwave', 'oven', 'toaster',
+               'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors',
+               'teddy bear', 'hair drier', 'toothbrush']
+
+
+def generate_image(images, molded_images, windows, results):
+    results_final = []
+    for i, image in enumerate(images):
+        final_rois, final_class_ids, final_scores, final_masks = \
+            model.unmold_detections(results[0][i], results[3][i], # detections[i], mrcnn_mask[i]
+                                   image.shape, molded_images[i].shape,
+                                   windows[i])
+        results_final.append({
+            "rois": final_rois,
+            "class_ids": final_class_ids,
+            "scores": final_scores,
+            "masks": final_masks,
+        })
+        r = results_final[i]
+        visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
+                                    class_names, r['scores'])
+    return results_final
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print("Need an image file for object detection.")
+        exit(-1)
+
+    model_file_name = './mrcnn.onnx'
+    if not os.path.exists(model_file_name):
+        # use opset 11 or later
+        set_converter('CropAndResize', convert_tf_crop_and_resize)
+        oml = mock_keras2onnx.convert_keras(model.keras_model, target_opset=11)
+        onnx.save_model(oml, model_file_name)
+
+    # run with ONNXRuntime
+    import onnxruntime
+    filename = sys.argv[1]
+    image = skimage.io.imread(filename)
+    images = [image]
+
+    sess = onnxruntime.InferenceSession(model_file_name)
+
+    # preprocessing
+    molded_images, image_metas, windows = model.mold_inputs(images)
+    anchors = model.get_anchors(molded_images[0].shape)
+    anchors = np.broadcast_to(anchors, (model.config.BATCH_SIZE,) + anchors.shape)
+
+    results = \
+        sess.run(None, {"input_image": molded_images.astype(np.float32),
+                        "input_anchors": anchors,
+                        "input_image_meta": image_metas.astype(np.float32)})
+
+    # postprocessing
+    results_final = generate_image(images, molded_images, windows, results)