From 82ad2e990f0601aeb45b1a25456148f40b4262e9 Mon Sep 17 00:00:00 2001 From: xadupre Date: Wed, 2 Jul 2025 14:25:50 +0200 Subject: [PATCH 1/4] improves documentation --- _doc/status/exported_program_dynamic.rst | 6 +- _doc/status/exporter_dynamic.rst | 100 ++++++++++++++++++ _doc/status/index.rst | 1 + .../ut_torch_export_patches/test_eval.py | 44 +++++++- .../torch_export_patches/eval/__init__.py | 4 +- 5 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 _doc/status/exporter_dynamic.rst diff --git a/_doc/status/exported_program_dynamic.rst b/_doc/status/exported_program_dynamic.rst index 22f96e00..b3cec3a0 100644 --- a/_doc/status/exported_program_dynamic.rst +++ b/_doc/status/exported_program_dynamic.rst @@ -2,9 +2,8 @@ Exported Programs with Dynamic Shapes ===================================== -The following script shows the exported program for many short cases -and various l-plot-export-with-dynamic-shape to retrieve an ONNX model equivalent -to the original model. +The following script shows the exported program for many short cases exported +with different options. This steps happens before converting into ONNX. .. runpython:: :showcode: @@ -55,6 +54,7 @@ to the original model. "export-strict", "export-nostrict", "export-nostrict-decall", + "export-tracing", ): expname = exporter.replace("export-", "") print() diff --git a/_doc/status/exporter_dynamic.rst b/_doc/status/exporter_dynamic.rst new file mode 100644 index 00000000..05737480 --- /dev/null +++ b/_doc/status/exporter_dynamic.rst @@ -0,0 +1,100 @@ +================================= +Exported ONNX with Dynamic Shapes +================================= + +The following script shows the exported program for many short cases +and various l-plot-export-with-dynamic-shape to retrieve an ONNX model equivalent +to the original model. + +.. runpython:: + :showcode: + :rst: + :toggle: code + :warningout: UserWarning + + import inspect + import textwrap + import pandas + from onnx_diagnostic.helpers import string_type + from onnx_diagnostic.helpers.onnx_helper import pretty_onnx + from onnx_diagnostic.torch_export_patches.eval import discover, run_exporter + from onnx_diagnostic.ext_test_case import unit_test_going + + cases = discover() + print() + print(":ref:`Summary `") + print() + sorted_cases = sorted(cases.items()) + if unit_test_going(): + sorted_cases = sorted_cases[:3] + for name, cls_model in sorted_cases: + print(f"* :ref:`{name} `") + print() + print() + + obs = [] + for name, cls_model in sorted(cases.items()): + print() + print(f".. _ledx-model-case-export-{name}:") + print() + print(name) + print("=" * len(name)) + print() + print("forward") + print("+++++++") + print() + print(".. code-block:: python") + print() + src = inspect.getsource(cls_model.forward) + if src: + print(textwrap.indent(textwrap.dedent(src), " ")) + else: + print(" # code is missing") + print() + print() + for exporter in ("custom", "dynamo-ir"): + expname = exporter.replace("export-", "") + print() + print(expname) + print("+" * len(expname)) + print() + res = run_exporter(exporter, cls_model, True, quiet=True) + case_ref = f":ref:`{name} `" + expo = exporter.split("-", maxsplit=1)[-1] + if "inputs" in res: + print(f"* **inputs:** ``{string_type(res['inputs'], with_shape=True)}``") + if "dynamic_shapes" in res: + print(f"* **shapes:** ``{string_type(res['dynamic_shapes'])}``") + print() + print() + if "onx" in res: + print(".. code-block:: text") + print() + print(textwrap.indent(pretty_onnx(res["onx"]), " ")) + print() + print() + obs.append(dict(case=case_ref, error="", exporter=expo)) + if "error" in res: + print("**FAILED**") + print() + print(".. code-block:: text") + print() + err = str(res["error"]) + if err: + print(textwrap.indent(err, " ")) + else: + print(" # no error found for the failure") + print() + print() + obs.append(dict(case=case_ref, error="FAIL", exporter=expo)) + + print() + print(".. _ledx-summary-exported-program:") + print() + print("Summary") + print("+++++++") + print() + df = pandas.DataFrame(obs) + piv = df.pivot(index="case", columns="exporter", values="error") + print(piv.to_markdown(tablefmt="rst")) + print() diff --git a/_doc/status/index.rst b/_doc/status/index.rst index e509862d..26cfdd66 100644 --- a/_doc/status/index.rst +++ b/_doc/status/index.rst @@ -9,6 +9,7 @@ what works and what does not with :func:`torch.export.export`. :maxdepth: 1 exported_program_dynamic + exporter_dynamic patches_coverage Some PRs in :epkg:`transformers` to keep in mind when it comes to export diff --git a/_unittests/ut_torch_export_patches/test_eval.py b/_unittests/ut_torch_export_patches/test_eval.py index 6c898b62..fabb8f0c 100644 --- a/_unittests/ut_torch_export_patches/test_eval.py +++ b/_unittests/ut_torch_export_patches/test_eval.py @@ -35,10 +35,42 @@ def test_eval(self): self.assertIsInstance(ev, list) self.assertIsInstance(ev[0], dict) - def test_run_exporter(self): + def test_run_exporter_custom(self): evaluation( cases="SignatureListFixedLength", - exporters="custom-strict", + exporters="custom", + quiet=False, + dynamic=False, + ) + + def test_run_exporter_dynamo(self): + evaluation( + cases="SignatureListFixedLength", + exporters="dynamo", + quiet=False, + dynamic=False, + ) + + def test_run_exporter_dynamo_ir(self): + evaluation( + cases="SignatureListFixedLength", + exporters="dynamo-ir", + quiet=False, + dynamic=False, + ) + + def test_run_exporter_nostrict(self): + evaluation( + cases="SignatureListFixedLength", + exporters="export-nostrict", + quiet=False, + dynamic=False, + ) + + def test_run_exporter_tracing(self): + evaluation( + cases="SignatureListFixedLength", + exporters="export-tracing", quiet=False, dynamic=False, ) @@ -46,6 +78,14 @@ def test_run_exporter(self): def test_run_exporter_regex(self): evaluation(cases=".*Aten.*", exporters="custom-strict", quiet=False, dynamic=False) + def test_run_exporter_custom_nested_cond(self): + evaluation( + cases="ControlFlowNestCond", + exporters="custom", + quiet=False, + dynamic=False, + ) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/onnx_diagnostic/torch_export_patches/eval/__init__.py b/onnx_diagnostic/torch_export_patches/eval/__init__.py index 4c69ad3f..71194f4e 100644 --- a/onnx_diagnostic/torch_export_patches/eval/__init__.py +++ b/onnx_diagnostic/torch_export_patches/eval/__init__.py @@ -337,7 +337,7 @@ def _make_exporter_onnx( from experimental_experiment.torch_interpreter import to_onnx, ExportOptions opts = {} - opts["strict"] = "-nostrict" not in exporter + opts["strict"] = "-strict" in exporter opts["fallback"] = "-fallback" in exporter opts["tracing"] = "-tracing" in exporter opts["jit"] = "-jit" in exporter @@ -520,6 +520,8 @@ def run_exporter( return res onx, builder = res + base["onx"] = onx + base["builder"] = builder if verbose >= 9: print("[run_exporter] onnx model") print( From da18e90b8e2e1757c7d254a6f9c726d0c8b16584 Mon Sep 17 00:00:00 2001 From: xadupre Date: Wed, 2 Jul 2025 14:42:09 +0200 Subject: [PATCH 2/4] fix doc --- _doc/status/exporter_dynamic.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/_doc/status/exporter_dynamic.rst b/_doc/status/exporter_dynamic.rst index 05737480..940788a1 100644 --- a/_doc/status/exporter_dynamic.rst +++ b/_doc/status/exporter_dynamic.rst @@ -73,7 +73,8 @@ to the original model. print(textwrap.indent(pretty_onnx(res["onx"]), " ")) print() print() - obs.append(dict(case=case_ref, error="", exporter=expo)) + if "error" not in res: + obs.append(dict(case=case_ref, error="", exporter=expo)) if "error" in res: print("**FAILED**") print() From 0f52906342bc5921e998670ff15644f6c0b9f8a0 Mon Sep 17 00:00:00 2001 From: xadupre Date: Wed, 2 Jul 2025 15:08:14 +0200 Subject: [PATCH 3/4] doc --- .github/workflows/documentation.yml | 4 ++-- onnx_diagnostic/torch_export_patches/README.md | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 onnx_diagnostic/torch_export_patches/README.md diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index bc1214c7..f8554e5e 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -111,9 +111,9 @@ jobs: grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' exit 1 fi - if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export') ]]; then + if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition') ]]; then echo "Documentation produces warnings." - grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' + grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' exit 1 fi diff --git a/onnx_diagnostic/torch_export_patches/README.md b/onnx_diagnostic/torch_export_patches/README.md new file mode 100644 index 00000000..34a77d50 --- /dev/null +++ b/onnx_diagnostic/torch_export_patches/README.md @@ -0,0 +1,11 @@ +# Patches to export HuggingFace models + +See [Patches Explained](https://sdpython.github.io/doc/onnx-diagnostic/dev/patches.html). + +```python +from onnx_diagnostic.torch_export_patches import torch_export_patches + +with torch_export_patches(patch_transformers=True) as f: + ep = torch.export.export(model, args, kwargs=kwargs, dynamic_shapes=dynamic_shapes) + # ... +``` From 2af7569878f74f91d7a53d226937d1f7982a06be Mon Sep 17 00:00:00 2001 From: xadupre Date: Wed, 2 Jul 2025 15:40:48 +0200 Subject: [PATCH 4/4] doc --- .github/workflows/documentation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index f8554e5e..33504ade 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -111,9 +111,9 @@ jobs: grep ERROR doc.txt | grep -v 'l-plot-tiny-llm-export' exit 1 fi - if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition') ]]; then + if [[ $(grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string') ]]; then echo "Documentation produces warnings." - grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' + grep WARNING doc.txt | grep -v 'l-plot-tiny-llm-export' | grep -v 'Inline emphasis start-string' | grep -v 'Definition list ends without a blank line' | grep -v 'Unexpected section title or transition' | grep -v 'Inline strong start-string' exit 1 fi