Improves documentation

.github/workflows/ci.yml

@@ -15,20 +15,28 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: ['3.10', '3.11', '3.12']
-        transformers: ['4.48.3', '4.51.3', 'main']
-        torch: ['2.6', '2.7', 'main']
+        python: ['3.10', '3.11', '3.12', '3.13']
+        transformers: ['4.48.3', '4.51.3', '4.52.1', 'main']
+        torch: ['2.7', 'main']
         exclude:
           - python: '3.10'
-            transformers: 'main'
+            torch: 'main'
+          - python: '3.11'
+            torch: 'main'
           - python: '3.10'
-            torch: '2.7'
+            transformers: '4.52.1'
+          - python: '3.10'
+            transformers: 'main'
           - python: '3.11'
-            transformers: '4.51.3'
+            transformers: '4.52.1'
           - python: '3.11'
+            transformers: 'main'
+          - python: '3.13'
             torch: '2.7'
-          - python: '3.12'
-            torch: '2.6'
+          - python: '3.13'
+            transformers: '4.48.3'
+          - python: '3.13'
+            transformers: '4.51.3'
     steps:
       - uses: actions/checkout@v3
 

CHANGELOGS.rst

@@ -1,6 +1,9 @@
 Change Logs
 ===========
 
+0.5.1
++++++
+
 0.5.0
 +++++
 

_doc/api/torch_export_patches/eval/index.rst

@@ -0,0 +1,12 @@
+onnx_diagnostic.torch_export_patches.eval
+=========================================
+
+.. toctree::
+    :maxdepth: 1
+    :caption: modules
+
+    model_cases
+
+.. automodule:: onnx_diagnostic.torch_export_patches.eval
+    :members:
+    :no-undoc-members:

_doc/api/torch_export_patches/eval/model_cases.rst

@@ -0,0 +1,7 @@
+
+onnx_diagnostic.torch_export_patches.eval.model_cases
+=====================================================
+
+.. automodule:: onnx_diagnostic.torch_export_patches.eval.model_cases
+    :members:
+    :undoc-members:

_doc/api/torch_export_patches/index.rst

@@ -5,6 +5,7 @@ onnx_diagnostic.torch_export_patches
     :maxdepth: 1
     :caption: submodules
 
+    eval/index
     onnx_export_errors
     onnx_export_serialization
     patches/index

_doc/conf.py

@@ -92,6 +92,8 @@ def linkcode_resolve(domain, info):
     "matplotlib": ("https://matplotlib.org/stable/", None),
     "numpy": ("https://numpy.org/doc/stable", None),
     "onnx": ("https://onnx.ai/onnx/", None),
+    "onnxruntime": ("https://onnxruntime.ai/docs/api/python/", None),
+    "onnxscript": ("https://microsoft.github.io/onnxscript/", None),
     "onnx_array_api": ("https://sdpython.github.io/doc/onnx-array-api/dev/", None),
     "onnx_diagnostic": ("https://sdpython.github.io/doc/onnx-diagnostic/dev/", None),
     "onnx_extended": ("https://sdpython.github.io/doc/onnx-extended/dev/", None),

_doc/index.rst

@@ -36,6 +36,7 @@ It also implements tools to investigate, validate exported models (ExportedProgr
     :caption: Contents
 
     patches
+    status/index
     api/index
     cmds/index
     auto_examples/index
@@ -195,14 +196,18 @@ See :meth:`onnx_diagnostic.export.ModelInputs.guess_dynamic_shapes`.
 
     >>> (({0: 'dim_0I0', 1: 'dim_0I1'}, {1: 'dim_1I1'}), {})
 
-use_dyn_for_str
+use_dyn_not_str
 +++++++++++++++
 
+See :meth:`onnx_diagnostic.torch_export_patches.patch_inputs.use_dyn_not_str`.
 
+The function replaces dynamic dimensions defined as strings by
+``torch.export.Dim.DYNAMIC``.
 
 Older versions
 ++++++++++++++
 
+* `0.5.1 <../v0.5.1/index.html>`_
 * `0.5.0 <../v0.5.0/index.html>`_
 * `0.4.4 <../v0.4.4/index.html>`_
 * `0.4.3 <../v0.4.3/index.html>`_

_doc/status/exported_program_dynamic.rst

@@ -0,0 +1,90 @@
+=====================================
+Exported Programs with Dynamic Shapes
+=====================================
+
+The following script shows the exported program for many short cases
+and various l-plot-export-with-dynamic-shape to retrieve an ONNX model equivalent
+to the original model.
+
+.. runpython::
+    :showcode:
+    :rst:
+    :toggle: code
+    :warningout: UserWarning
+
+    import inspect
+    import textwrap
+    import pandas
+    from onnx_diagnostic.helpers import string_type
+    from onnx_diagnostic.torch_export_patches.eval import discover, run_exporter
+    from onnx_diagnostic.ext_test_case import unit_test_going
+
+    cases = discover()
+    print()
+    print(":ref:`Summary <led-summary-exported-program>`")
+    print()
+    sorted_cases = sorted(cases.items())
+    if unit_test_going():
+        sorted_cases = sorted_cases[:3]
+    for name, cls_model in sorted_cases:
+        print(f"* :ref:`{name} <led-model-case-export-{name}>`")
+    print()
+
+    obs = []
+    for name, cls_model in sorted(cases.items()):
+        print()
+        print(f".. _led-model-case-export-{name}:")
+        print()
+        print(name)
+        print("=" * len(name))
+        print()
+        print("forward")
+        print("+++++++")
+        print()
+        print("::")
+        print()
+        print(textwrap.indent(textwrap.dedent(inspect.getsource(cls_model.forward)), "    "))
+        print()
+        for exporter in (
+            "export-strict",
+            "export-nostrict",
+            "export-nostrict-decall",
+        ):
+            expname = exporter.replace("export-", "")
+            print()
+            print(expname)
+            print("+" * len(expname))
+            print()
+            res = run_exporter(exporter, cls_model, True, quiet=True)
+            case_ref = f":ref:`{name} <led-model-case-export-{name}>`"
+            expo = exporter.split("-", maxsplit=1)[-1]
+            if "inputs" in res:
+                print(f"* **inputs:** ``{string_type(res['inputs'], with_shape=True)}``")
+            if "dynamic_shapes" in res:
+                print(f"* **shapes:** ``{string_type(res['dynamic_shapes'])}``")
+            print()
+            if "exported" in res:
+                print("::")
+                print()
+                print(textwrap.indent(str(res["exported"].graph), "    "))
+                print()
+                obs.append(dict(case=case_ref, error="", exporter=expo))
+            else:
+                print("**FAILED**")
+                print()
+                print("::")
+                print()
+                print(textwrap.indent(str(res["error"]), "    "))
+                print()
+                obs.append(dict(case=case_ref, error="FAIL", exporter=expo))
+
+    print()
+    print(".. _led-summary-exported-program:")
+    print()
+    print("Summary")
+    print("+++++++")
+    print()
+    df = pandas.DataFrame(obs)
+    piv = df.pivot(index="case", columns="exporter", values="error")
+    print(piv.to_markdown(tablefmt="rst"))
+    print()

_doc/status/index.rst

@@ -0,0 +1,12 @@
+===============
+Exporter Status
+===============
+
+Following sections tries to capture what patches are put in place to export,
+what works and what does not with :func:`torch.export.export`.
+
+.. toctree::
+    :maxdepth: 1
+
+    exported_program_dynamic
+    patches_coverage

_doc/status/patches_coverage.rst

@@ -0,0 +1,45 @@
+=======================
+Coverage of the Patches
+=======================
+
+Serialized Classes
+==================
+
+The following code shows the list of serialized classes in transformers.
+
+.. runpython::
+    :showcode:
+
+    import onnx_diagnostic.torch_export_patches.onnx_export_serialization as p
+
+    print('\n'.join(sorted(p.serialization_functions())))
+
+Patched Classes
+===============
+
+The following script shows the list of methods patched
+for transformers.
+
+.. runpython::
+    :showcode:
+
+    import onnx_diagnostic.torch_export_patches.patches.patch_transformers as p
+
+    for name, cls in p.__dict__.items():
+        if name.startswith("patched_"):
+            print(f"{cls._PATCHED_CLASS_.__name__}: {', '.join(cls._PATCHES_)}")
+
+Half Automated Rewrites for Control Flows
+=========================================
+
+The following script shows the list of methods automatically rewritten
+due to control flows.
+
+.. runpython::
+    :showcode:
+
+    import onnx_diagnostic.torch_export_patches.patch_module_helper as p
+
+    for name, f in p.__dict__.items():
+        if name.startswith("_rewrite_"):
+            print(f.__doc__)

_unittests/ut_tasks/test_tasks_image_classification.py

@@ -16,7 +16,7 @@ def test_image_classification(self):
         model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"]
         model(**inputs)
         model(**data["inputs2"])
-        if not has_transformers("4.51.999"):
+        if not has_transformers("4.52.999"):
             raise unittest.SkipTest("Requires transformers>=4.52")
         with torch_export_patches(patch_transformers=True, verbose=10):
             torch.export.export(

_unittests/ut_torch_export_patches/test_eval.py

@@ -0,0 +1,51 @@
+import unittest
+from onnx_diagnostic.ext_test_case import ExtTestCase, requires_torch
+from onnx_diagnostic.torch_export_patches.eval import discover, evaluation
+
+
+class TestEval(ExtTestCase):
+    @requires_torch("2.7", "scan")
+    def test_discover(self):
+        res = discover()
+        self.assertNotEmpty(res)
+        for mod in res.values():
+            if mod.__name__ == "ControlFlowCondIdentity_153832":
+                continue
+            with self.subTest(name=mod.__name__):
+                m = mod()
+                if isinstance(m._inputs, tuple):
+                    m(*m._inputs)
+                else:
+                    m(*m._inputs[0])
+
+    def test_eval(self):
+        d = list(discover().items())[0]  # noqa: RUF015
+        ev = evaluation(
+            quiet=False,
+            cases={d[0]: d[1]},
+            exporters=(
+                "export-strict",
+                "export-nostrict",
+                "custom",
+                "dynamo",
+                "dynamo-ir",
+                "export-tracing",
+            ),
+        )
+        self.assertIsInstance(ev, list)
+        self.assertIsInstance(ev[0], dict)
+
+    def test_run_exporter(self):
+        evaluation(
+            cases="SignatureListFixedLength",
+            exporters="custom-strict",
+            quiet=False,
+            dynamic=False,
+        )
+
+    def test_run_exporter_regex(self):
+        evaluation(cases=".*Aten.*", exporters="custom-strict", quiet=False, dynamic=False)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

_unittests/ut_torch_models/test_llm_phi2.py

@@ -13,7 +13,7 @@ def test_get_phi2(self):
         model(**inputs)
 
     @ignore_warnings(UserWarning)
-    @requires_transformers("4.52")
+    @requires_transformers("4.53")
     def test_export_phi2_1(self):
         data = get_phi2(num_hidden_layers=2)
         model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"]

_unittests/ut_torch_models/test_tiny_llms.py

@@ -14,7 +14,7 @@ def test_get_tiny_llm(self):
         model(**inputs)
 
     @ignore_warnings(UserWarning)
-    @requires_transformers("4.52")
+    @requires_transformers("4.53")
     def test_export_tiny_llm_1(self):
         data = get_tiny_llm()
         model, inputs = data["model"], data["inputs"]

onnx_diagnostic/__init__.py

@@ -1,7 +1,7 @@
 """
-Investigates onnx models.
+Patches, Investigates onnx models.
 Functions, classes to dig into a model when this one is right, slow, wrong...
 """
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 __author__ = "Xavier Dupré"