improve examples

Author: xadupre

_doc/api/reference/index.rst

@@ -30,10 +30,10 @@ OnnxruntimeEvaluator
 .. autoclass:: onnx_diagnostic.reference.OnnxruntimeEvaluator
     :members:
 
-ReportResultsComparison
-+++++++++++++++++++++++
+ReportResultComparison
+++++++++++++++++++++++
 
-.. autoclass:: onnx_diagnostic.reference.ReportResultsComparison
+.. autoclass:: onnx_diagnostic.reference.ReportResultComparison
     :members:
 
 TorchOnnxEvaluator

_doc/api/reference/report_results_comparison.rst

@@ -5,4 +5,4 @@ onnx_diagnostic.reference.report_results_comparison
 .. automodule:: onnx_diagnostic.reference.report_results_comparison
     :members:
     :no-undoc-members:
-    :exclude-members: ReportResultsComparison
+    :exclude-members: ReportResultComparison

_doc/conf.py

@@ -5,6 +5,7 @@
 from sphinx_runpython.conf_helper import has_dvipng, has_dvisvgm
 import torch
 from onnx_diagnostic import __version__
+from onnx_diagnostic.doc import update_version_package
 
 extensions = [
     "sphinx.ext.autodoc",
@@ -40,8 +41,8 @@
 project = "onnx-diagnostic"
 copyright = "2025"
 author = "Xavier Dupré"
-version = __version__
-release = __version__
+version = update_version_package(__version__)
+release = version
 language = "en"
 exclude_patterns = []
 pygments_style = "sphinx"

_doc/examples/plot_dump_intermediate_results.py

@@ -19,21 +19,30 @@
 
 See :func:`onnx_diagnostic.helpers.torch_helper.dummy_llm`
 for its definition. It is mostly used for unit test or example.
-
 """
 
+import numpy as np
+import pandas
 import onnx
 import torch
+import onnxruntime
 from onnx_array_api.plotting.graphviz_helper import plot_dot
 from onnx_diagnostic import doc
-from onnx_diagnostic.helpers import string_type
-from onnx_diagnostic.helpers.torch_helper import dummy_llm
+from onnx_diagnostic.helpers import max_diff, string_diff, string_type
+from onnx_diagnostic.helpers.torch_helper import dummy_llm, steal_forward
 from onnx_diagnostic.helpers.mini_onnx_builder import create_input_tensors_from_onnx_model
-from onnx_diagnostic.helpers.torch_helper import steal_forward
+from onnx_diagnostic.reference import OnnxruntimeEvaluator, ReportResultComparison
 
 
 model, inputs, ds = dummy_llm(dynamic_shapes=True)
 
+# %%
+# We use float16.
+model = model.to(torch.float16)
+
+# %%
+# Let's check.
+
 print(f"type(model)={type(model)}")
 print(f"inputs={string_type(inputs, with_shape=True)}")
 print(f"ds={string_type(ds, with_shape=True)}")
@@ -65,7 +74,7 @@
     verbose=1,
     storage_limit=2**28,
 ):
-    model(*inputs)
+    expected = model(*inputs)
 
 
 # %%
@@ -124,7 +133,74 @@
 epo.save("plot_dump_intermediate_results.onnx")
 
 # %%
-# It looks like the following.
+# Discrepancies
+# +++++++++++++
+#
+# We have a torch model, intermediate results and an ONNX graph
+# equivalent to the torch model.
+# Let's see how we can check the discrepancies.
+# First the discrepancies of the whole model.
+
+sess = onnxruntime.InferenceSession(
+    "plot_dump_intermediate_results.onnx", providers=["CPUExecutionProvider"]
+)
+feeds = dict(
+    zip([i.name for i in sess.get_inputs()], [t.detach().cpu().numpy() for t in inputs])
+)
+got = sess.run(None, feeds)
+diff = max_diff(expected, got)
+print(f"discrepancies torch/ORT: {string_diff(diff)}")
+
+# %%
+# What about intermediate results?
+# Let's use a runtime still based on :epkg:`onnxruntime`
+# running an eager evaluation.
+
+sess_eager = OnnxruntimeEvaluator(
+    "plot_dump_intermediate_results.onnx",
+    providers=["CPUExecutionProvider"],
+    torch_or_numpy=True,
+)
+feeds_tensor = dict(zip([i.name for i in sess.get_inputs()], inputs))
+got = sess_eager.run(None, feeds_tensor)
+diff = max_diff(expected, got)
+print(f"discrepancies torch/eager ORT: {string_diff(diff)}")
+
+# %%
+# They are almost the same. That's good.
+# Let's now dig into the intermediate results.
+# They are compared to the outputs stored in saved_tensors
+# during the execution of the model.
+baseline = {}
+for k, v in saved_tensors.items():
+    if k[-1] == "I":  # inputs are excluded
+        continue
+    if isinstance(v, torch.Tensor):
+        baseline[f"{k[0]}.{k[1]}".replace("model.decoder", "decoder")] = v
+
+report_cmp = ReportResultComparison(baseline)
+sess_eager.run(None, feeds_tensor, report_cmp=report_cmp)
+
+# %%
+# Let's see the results.
+
+data = report_cmp.data
+df = pandas.DataFrame(data)
+piv = df.pivot(index=("run_index", "run_name"), columns="ref_name", values="abs")
+print(piv)
+
+# %%
+# Let's clean a little bit.
+piv[piv >= 1] = np.nan
+print(piv.dropna(axis=0, how="all"))
+
+# %%
+# We can identity which results is mapped to which expected tensor.
+
+# %%
+# Picture of the model
+# ++++++++++++++++++++
+
 onx = onnx.load("plot_dump_intermediate_results.onnx")
 plot_dot(onx)
 

_doc/index.rst

@@ -236,6 +236,7 @@ With the following versions:
     import ml_dtypes
     import sklearn
     import onnx
+    import onnx_ir
     import onnxruntime
     import onnxscript
     import torch
@@ -247,6 +248,7 @@ With the following versions:
         ml_dtypes,
         sklearn,
         onnx,
+        onnx_ir,
         onnxruntime,
         onnxscript,
         torch,

_unittests/ut_reference/test_onnxruntime_evaluator.py

@@ -9,7 +9,7 @@
 from onnx_diagnostic.reference import (
     OnnxruntimeEvaluator,
     ExtendedReferenceEvaluator,
-    ReportResultsComparison,
+    ReportResultComparison,
 )
 
 try:
@@ -217,7 +217,7 @@ def test_report_results_comparison_ort(self):
         )
         x = torch.rand(5, 6, dtype=torch.float32)
         onnx.checker.check_model(model)
-        cmp = ReportResultsComparison(dict(r_x=x, r_cos=x.cos(), r_exp=x.cos().sin().exp()))
+        cmp = ReportResultComparison(dict(r_x=x, r_cos=x.cos(), r_exp=x.cos().sin().exp()))
         cmp.clear()
         feeds = dict(zip([i.name for i in model.graph.input], (x,)))
         rt = OnnxruntimeEvaluator(model, verbose=10)

_unittests/ut_reference/test_torch_onnx_evaluator.py

@@ -11,7 +11,7 @@
 from onnx_diagnostic.reference import (
     ExtendedReferenceEvaluator,
     TorchOnnxEvaluator,
-    ReportResultsComparison,
+    ReportResultComparison,
 )
 from onnx_diagnostic.reference.torch_ops import OpRunKernel, OpRunTensor
 from onnx_diagnostic.reference.torch_evaluator import get_kernels
@@ -1496,7 +1496,7 @@ def test_report_results_comparison(self):
         )
         x = torch.rand(5, 6, dtype=torch.float32)
         onnx.checker.check_model(model)
-        cmp = ReportResultsComparison(dict(r_x=x, r_cos=x.cos(), r_exp=x.cos().sin().exp()))
+        cmp = ReportResultComparison(dict(r_x=x, r_cos=x.cos(), r_exp=x.cos().sin().exp()))
         cmp.clear()
         feeds = dict(zip([i.name for i in model.graph.input], (x,)))
         rt = TorchOnnxEvaluator(model, verbose=10)

onnx_diagnostic/doc.py

@@ -2,6 +2,28 @@
 import numpy as np
 
 
+def get_latest_pypi_version(package_name="onnx-diagnostic") -> str:
+    """Returns the latest published version."""
+
+    import requests
+
+    url = f"https://pypi.org/pypi/{package_name}/json"
+    response = requests.get(url)
+
+    assert response.status_code == 200, f"Unable to retrieve the version response={response}"
+    data = response.json()
+    version = data["info"]["version"]
+    return version
+
+
+def update_version_package(version: str, package_name="onnx-diagnostic") -> str:
+    "Adds dev if the major version is different from the latest published one."
+    released = get_latest_pypi_version(package_name)
+    shorten_r = ".".join(released.split(".")[:2])
+    shorten_v = ".".join(version.split(".")[:2])
+    return version if shorten_r == shorten_v else f"{shorten_v}.dev"
+
+
 def reset_torch_transformers(gallery_conf, fname):
     "Resets torch dynamo for :epkg:`sphinx-gallery`."
     import matplotlib.pyplot as plt

onnx_diagnostic/reference/__init__.py

@@ -1,4 +1,4 @@
 from .evaluator import ExtendedReferenceEvaluator
 from .ort_evaluator import OnnxruntimeEvaluator
 from .torch_evaluator import TorchOnnxEvaluator
-from .report_results_comparison import ReportResultsComparison
+from .report_results_comparison import ReportResultComparison

onnx_diagnostic/reference/ort_evaluator.py

@@ -22,7 +22,8 @@
     InferenceSessionForNumpy,
     _InferenceSession,
 )
-from .report_results_comparison import ReportResultsComparison
+from ..helpers.torch_helper import to_tensor
+from .report_results_comparison import ReportResultComparison
 from .evaluator import ExtendedReferenceEvaluator
 
 
@@ -51,6 +52,8 @@ class OnnxruntimeEvaluator:
     :param ir_version: ir version to use when unknown
     :param opsets: opsets to use when unknown
     :param whole: if True, do not split node by node
+    :param torch_or_numpy: force the use of one of them, Ture for torch,
+        False for numpy, None to let the class choose
     """
 
     def __init__(
@@ -73,6 +76,7 @@ def __init__(
         ir_version: int = 10,
         opsets: Optional[Union[int, Dict[str, int]]] = None,
         whole: bool = False,
+        torch_or_numpy: Optional[bool] = None,
     ):
         if isinstance(proto, str):
             self.proto: Proto = load(proto)
@@ -104,8 +108,10 @@ def __init__(
             disable_aot_function_inlining=disable_aot_function_inlining,
             use_training_api=use_training_api,
         )
+        self.to_tensor_or_array = to_array_extended if not torch_or_numpy else to_tensor
 
         self.verbose = verbose
+        self.torch_or_numpy = torch_or_numpy
         self.sess_: Optional[_InferenceSession] = None
         if whole:
             self.nodes: Optional[List[NodeProto]] = None
@@ -124,7 +130,10 @@ def __init__(
                 )
             )
             self.rt_inits_ = (
-                {init.name: to_array_extended(init) for init in self.proto.graph.initializer}
+                {
+                    init.name: self.to_tensor_or_array(init)
+                    for init in self.proto.graph.initializer
+                }
                 if hasattr(self.proto, "graph")
                 else {}
             )
@@ -192,13 +201,14 @@ def _log_arg(self, a: Any) -> Any:
             return a
         device = f"D{a.get_device()}:" if hasattr(a, "detach") else ""
         if hasattr(a, "shape"):
+            prefix = "A:" if hasattr(a, "astype") else "T:"
             if self.verbose < 4:  # noqa: PLR2004
-                return f"{device}{a.dtype}:{a.shape} in [{a.min()}, {a.max()}]"
+                return f"{prefix}{device}{a.dtype}:{a.shape} in [{a.min()}, {a.max()}]"
             elements = a.ravel().tolist()
             if len(elements) > 10:  # noqa: PLR2004
                 elements = elements[:10]
-                return f"{device}{a.dtype}:{a.shape}:{','.join(map(str, elements))}..."
-            return f"{device}{a.dtype}:{a.shape}:{elements}"
+                return f"{prefix}{device}{a.dtype}:{a.shape}:{','.join(map(str, elements))}..."
+            return f"{prefix}{device}{a.dtype}:{a.shape}:{elements}"
         if hasattr(a, "append"):
             return ", ".join(map(self._log_arg, a))
         return a
@@ -216,7 +226,7 @@ def run(
         outputs: Optional[List[str]],
         feed_inputs: Dict[str, Any],
         intermediate: bool = False,
-        report_cmp: Optional[ReportResultsComparison] = None,
+        report_cmp: Optional[ReportResultComparison] = None,
     ) -> Union[Dict[str, Any], List[Any]]:
         """
          Runs the model.
@@ -228,7 +238,7 @@ def run(
          :param report_cmp: used as a reference,
              every intermediate results is compare to every existing one,
              if not empty, it is an instance of
-             :class:`onnx_diagnostic.reference.ReportResultsComparison`
+             :class:`onnx_diagnostic.reference.ReportResultComparison`
         :return: outputs, as a list if return_all is False,
              as a dictionary if return_all is True
         """
@@ -437,8 +447,12 @@ def _get_sess(
         cls = (
             InferenceSessionForNumpy
             if any(isinstance(i, np.ndarray) for i in inputs)
+            and (not isinstance(self.torch_or_numpy, bool) or not self.torch_or_numpy)
             else InferenceSessionForTorch
         )
+        assert (
+            cls is InferenceSessionForTorch
+        ), f"ERROR: {string_type(inputs, with_shape=True)}"
         try:
             sess = cls(onx, **self.session_kwargs)
         except (
@@ -497,6 +511,7 @@ def _get_sess_if(
             verbose=self.verbose,
             ir_version=self.ir_version,
             opsets=self.opsets,
+            torch_or_numpy=self.torch_or_numpy,
             **self.session_kwargs,
         )
         return onx, sess
@@ -511,6 +526,7 @@ def _get_sess_local(
             verbose=self.verbose,
             ir_version=self.ir_version,
             opsets=self.opsets,
+            torch_or_numpy=self.torch_or_numpy,
             **self.session_kwargs,
         )
         return ev.proto, sess
@@ -586,6 +602,7 @@ def _get_sess_scan(
             verbose=self.verbose,
             ir_version=self.ir_version,
             opsets=self.opsets,
+            torch_or_numpy=self.torch_or_numpy,
             whole=True,
             **self.session_kwargs,
         )