Add OnnxruntimeEvaluator

.github/workflows/ci.yml

@@ -56,12 +56,19 @@ jobs:
         run: |
           pip install pytest
           export PYTHONPATH=.
-          UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py
+          UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py
           export PYTHONPATH=
 
-      - name: run backend tests
+      - name: run backend tests python
         run: |
           pip install pytest
           export PYTHONPATH=.
           UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py
           export PYTHONPATH=
+
+      - name: run backend tests onnxruntime
+        run: |
+          pip install pytest
+          export PYTHONPATH=.
+          UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15
+          export PYTHONPATH=

.github/workflows/documentation.yml

@@ -59,7 +59,7 @@ jobs:
           pip install pytest
           pip install pytest-cov
           export PYTHONPATH=.
-          UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py
+          UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py  --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py
           export PYTHONPATH=
 
       - name: Upload coverage reports to Codecov

CHANGELOGS.rst

@@ -4,6 +4,7 @@ Change Logs
 0.2.0
 +++++
 
+* :pr:`9`: adds ``OnnxruntimeEvaluator``
 * :pr:`8`: adds ``ExtendedReferenceEvaluator``
 * :pr:`7`: improves function ``investigate_onnxruntime_issue``
 

README.rst

@@ -11,10 +11,6 @@ onnx-diagnostic: investigate onnx models
 .. image:: https://badge.fury.io/py/onnx-diagnostic.svg
     :target: http://badge.fury.io/py/onnx-diagnostic
 
-.. image:: http://img.shields.io/github/issues/sdpython/onnx-diagnostic.png
-    :alt: GitHub Issues
-    :target: https://github.com/sdpython/onnx-diagnostic/issues
-
 .. image:: https://img.shields.io/badge/license-MIT-blue.svg
     :alt: MIT License
     :target: https://opensource.org/license/MIT/

_doc/api/reference/index.rst

@@ -13,13 +13,20 @@ onnx_diagnostic.reference
 
     evaluator
     quantized_tensor
+    ort_evaluator
 
 ExtendedReferenceEvaluator
 ++++++++++++++++++++++++++
 
 .. autoclass:: onnx_diagnostic.reference.ExtendedReferenceEvaluator
     :members:
 
+OnnxruntimeEvaluator
+++++++++++++++++++++
+
+.. autoclass:: onnx_diagnostic.reference.OnnxruntimeEvaluator
+    :members:
+
 Other functions
 +++++++++++++++
 

_doc/api/reference/ort_evaluator.rst

@@ -0,0 +1,8 @@
+
+onnx_diagnostic.reference.ort_evaluator
+=======================================
+
+.. automodule:: onnx_diagnostic.reference.ort_evaluator
+    :members:
+    :no-undoc-members:
+    :exclude-members: OnnxruntimeEvaluator

_doc/conf.py

@@ -104,11 +104,12 @@
     ("py:class", "False"),
     ("py:class", "True"),
     ("py:class", "Argument"),
-    ("py:class", "onnxscript.ir.Tuple"),
-    ("py:class", "pipeline.Pipeline"),
     ("py:class", "default=sklearn.utils.metadata_routing.UNCHANGED"),
     ("py:class", "ModelProto"),
     ("py:class", "Module"),
+    ("py:class", "np.ndarray"),
+    ("py:class", "onnxscript.ir.Tuple"),
+    ("py:class", "pipeline.Pipeline"),
     ("py:class", "torch.fx.passes.operator_support.OperatorSupport"),
     ("py:class", "torch.fx.proxy.TracerBase"),
     ("py:class", "torch.utils._pytree.Context"),
@@ -177,6 +178,7 @@
     "GraphModule": "https://pytorch.org/docs/stable/fx.html#torch.fx.GraphModule",
     "HuggingFace": "https://huggingface.co/docs/hub/en/index",
     "Linux": "https://www.linux.org/",
+    "ml_dtypes": "https://github.com/jax-ml/ml_dtypes",
     "monai": "https://monai.io/",
     "numpy": "https://numpy.org/",
     "onnx": "https://onnx.ai/onnx/",
@@ -186,6 +188,7 @@
     "onnxrt backend": "https://pytorch.org/docs/stable/onnx_dynamo_onnxruntime_backend.html",
     "onnxruntime": "https://onnxruntime.ai/",
     "onnxruntime-training": "https://onnxruntime.ai/docs/get-started/training-on-device.html",
+    "onnxruntime kernels": "https://onnxruntime.ai/docs/reference/operators/OperatorKernels.html",
     "onnx-array-api": "https://sdpython.github.io/doc/onnx-array-api/dev/",
     "onnx-diagnostic": "https://sdpython.github.io/doc/onnx-diagnostic/dev/",
     "onnx-extended": "https://sdpython.github.io/doc/onnx-extended/dev/",

_doc/examples/plot_failing_onnxruntime_evaluator.py

@@ -0,0 +1,106 @@
+"""
+.. _l-plot-failing-onnxruntime-evaluator:
+
+Running OnnxruntimeEvaluator on a failing model
+===============================================
+
+Example :ref:`l-plot-failing-reference-evaluator` demonstrated
+how to run a python runtime on a model but it may very slow sometimes
+and it could show some discrepancies if the only provider is not CPU.
+Let's use :class:`OnnxruntimeEvaluator <onnx_diagnostic.reference.OnnxruntimeEvaluator>`.
+It splits the model into node and runs them independantly until it succeeds
+or fails. This class converts every node into model based on the types
+discovered during the execution. It relies on :class:`InferenceSessionForTorch
+<onnx_diagnostic.ort_session.InferenceSessionForTorch>` or
+:class:`InferenceSessionForNumpy
+<onnx_diagnostic.ort_session.InferenceSessionForNumpy>`
+for the execution. This example uses torch tensor and
+bfloat16.
+
+A failing model
++++++++++++++++
+
+The issue here is a an operator ``Cast`` trying to convert a result
+into a non-existing type.
+"""
+
+import onnx
+import onnx.helper as oh
+import torch
+import onnxruntime
+from onnx_diagnostic.ext_test_case import has_cuda
+from onnx_diagnostic.helpers import from_array_extended
+from onnx_diagnostic.reference import OnnxruntimeEvaluator
+
+TBFLOAT16 = onnx.TensorProto.BFLOAT16
+
+model = oh.make_model(
+    oh.make_graph(
+        [
+            oh.make_node("Mul", ["X", "Y"], ["xy"], name="n0"),
+            oh.make_node("Sigmoid", ["xy"], ["sy"], name="n1"),
+            oh.make_node("Add", ["sy", "one"], ["C"], name="n2"),
+            oh.make_node("Cast", ["C"], ["X999"], to=999, name="failing"),
+            oh.make_node("CastLike", ["X999", "Y"], ["Z"], name="n4"),
+        ],
+        "nd",
+        [
+            oh.make_tensor_value_info("X", TBFLOAT16, ["a", "b", "c"]),
+            oh.make_tensor_value_info("Y", TBFLOAT16, ["a", "b", "c"]),
+        ],
+        [oh.make_tensor_value_info("Z", TBFLOAT16, ["a", "b", "c"])],
+        [from_array_extended(torch.tensor([1], dtype=torch.bfloat16), name="one")],
+    ),
+    opset_imports=[oh.make_opsetid("", 18)],
+    ir_version=9,
+)
+
+# %%
+# We check it is failing.
+
+try:
+    onnxruntime.InferenceSession(model.SerializeToString(), providers=["CPUExecutionProvider"])
+except onnxruntime.capi.onnxruntime_pybind11_state.Fail as e:
+    print(e)
+
+
+# %%
+# OnnxruntimeEvaluator
+# ++++++++++++++++++++++++++
+#
+# This class extends :class:`onnx.reference.ReferenceEvaluator`
+# with operators outside the standard but defined by :epkg:`onnxruntime`.
+# `verbose=10` tells the class to print as much as possible,
+# `verbose=0` prints nothing. Intermediate values for more or less verbosity.
+
+ref = OnnxruntimeEvaluator(model, verbose=10)
+feeds = dict(
+    X=torch.rand((3, 4), dtype=torch.bfloat16), Y=torch.rand((3, 4), dtype=torch.bfloat16)
+)
+try:
+    ref.run(None, feeds)
+except Exception as e:
+    print("ERROR", type(e), e)
+
+
+# %%
+# :epkg:`onnxruntime` may not support bfloat16 on CPU.
+# See :epkg:`onnxruntime kernels`.
+
+if has_cuda():
+    ref = OnnxruntimeEvaluator(model, providers="cuda", verbose=10)
+    feeds = dict(
+        X=torch.rand((3, 4), dtype=torch.bfloat16), Y=torch.rand((3, 4), dtype=torch.bfloat16)
+    )
+    try:
+        ref.run(None, feeds)
+    except Exception as e:
+        print("ERROR", type(e), e)
+
+# %%
+# We can see it run until it reaches `Cast` and stops.
+# The error message is not always obvious to interpret.
+# It gets improved everytime from time to time.
+# This runtime is useful when it fails for a numerical reason.
+# It is possible to insert prints in the python code to print
+# more information or debug if needed.

_doc/index.rst

@@ -8,18 +8,10 @@ onnx-diagnostic: investigate onnx models
 .. image:: https://badge.fury.io/py/onnx-diagnostic.svg
     :target: http://badge.fury.io/py/onnx-diagnostic
 
-.. image:: http://img.shields.io/github/issues/sdpython/onnx-diagnostic.png
-    :alt: GitHub Issues
-    :target: https://github.com/sdpython/onnx-diagnostic/issues
-
 .. image:: https://img.shields.io/badge/license-MIT-blue.svg
     :alt: MIT License
     :target: https://opensource.org/license/MIT/
 
-.. image:: https://img.shields.io/github/repo-size/sdpython/onnx-diagnostic
-    :target: https://github.com/sdpython/onnx-diagnostic/
-    :alt: size
-
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/psf/black
 
@@ -51,6 +43,7 @@ Source are `sdpython/onnx-diagnostic
 * :ref:`l-plot-sxport-with-dynamio-shapes-auto`
 * :ref:`l-plot-tiny-llm-export`
 * :ref:`l-plot-failing-reference-evaluator`
+* :ref:`l-plot-failing-onnxruntime-evaluator`
 * :ref:`l-plot-failing-model-extract`
 
 **Some Usefuls Tools**

_unittests/ut_reference/test_array_tensor.py

@@ -2,9 +2,8 @@
 import numpy as np
 from onnx import TensorProto
 from onnx.helper import make_graph, make_model, make_node, make_tensor_value_info
-from onnx.reference.op_run import to_array_extended
 from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings
-from onnx_diagnostic.helpers import from_array_extended
+from onnx_diagnostic.helpers import from_array_extended, to_array_extended
 from onnx_diagnostic.reference import ExtendedReferenceEvaluator
 
 

_unittests/ut_reference/test_backend_extended_reference_evaluator.py

@@ -43,13 +43,13 @@ def run(self, inputs, **kwargs):
 
 class ExtendedReferenceEvaluatorBackend(onnx.backend.base.Backend):
     @classmethod
-    def is_opset_supported(cls, model):  # pylint: disable=unused-argument
-        return True, ""
+    def is_compatible(cls, model) -> bool:
+        return True
 
     @classmethod
     def supports_device(cls, device: str) -> bool:
         d = Device(device)
-        return d.type == DeviceType.CPU  # type: ignore[no-any-return]
+        return d.type == DeviceType.CPU
 
     @classmethod
     def create_inference_session(cls, model):