Enable a disabled test

.github/workflows/ci.yml

@@ -67,45 +67,30 @@ jobs:
         run: python -m pip freeze
 
       - name: tiny-llm torch.export.export
-        run: |
-          export PYTHONPATH=.
-          python _unittests/ut_torch_models/test_tiny_llms.py
+        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms.py
 
       - name: tiny-llm onnx
-        run: |
-          export PYTHONPATH=.
-          python _unittests/ut_torch_models/test_tiny_llms_onnx.py
+        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_onnx.py
         continue-on-error: true  # connectivity issues
 
       - name: tiny-llm example
-        run: |
-          export PYTHONPATH=.
-          python _doc/examples/plot_export_tiny_llm.py
+        run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm.py
         continue-on-error: true  # connectivity issues
 
       - name: tiny-llm bypass
-        run: |
-          export PYTHONPATH=.
-          python _doc/examples/plot_export_tiny_llm_patched.py
+        run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm_patched.py
         continue-on-error: true  # connectivity issues
 
+      - name: run tests bypassed
+        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_bypassed.py
+
       - name: run tests
         run: |
           pip install pytest
-          export PYTHONPATH=.
-          UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py
-          export PYTHONPATH=
+          PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py
 
       - name: run backend tests python
-        run: |
-          pip install pytest
-          export PYTHONPATH=.
-          UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py
-          export PYTHONPATH=
+        run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py
 
       - name: run backend tests onnxruntime
-        run: |
-          pip install pytest
-          export PYTHONPATH=.
-          UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15
-          export PYTHONPATH=
+        run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15

.github/workflows/documentation.yml

@@ -54,33 +54,25 @@ jobs:
         run: python -m pip freeze
 
       - name: tiny-llm torch.export.export
-        run: |
-          export PYTHONPATH=.
-          python _unittests/ut_torch_models/test_tiny_llms.py
+        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms.py
 
       - name: tiny-llm onnx
-        run: |
-          export PYTHONPATH=.
-          python _unittests/ut_torch_models/test_tiny_llms_onnx.py
+        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_onnx.py
         continue-on-error: true
 
       - name: tiny-llm example
-        run: |
-          export PYTHONPATH=.
-          python _doc/examples/plot_export_tiny_llm.py
+        run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm.py
 
       - name: tiny-llm bypass
-        run: |
-          export PYTHONPATH=.
-          python _doc/examples/plot_export_tiny_llm_patched.py
+        run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm_patched.py
+
+      - name: run tests bypassed
+        run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_bypassed.py
 
       - name: Generate coverage report
         run: |
-          pip install pytest
-          pip install pytest-cov
-          export PYTHONPATH=.
-          UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py  --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py
-          export PYTHONPATH=
+          pip install pytest pytest-cov          
+          PYTHONPATH=. UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py  --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3

_unittests/ut_torch_models/test_llm_phi2.py

@@ -3,7 +3,6 @@
 from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings, requires_transformers
 from onnx_diagnostic.torch_models.llms import get_phi2
 from onnx_diagnostic.helpers import string_type
-from onnx_diagnostic.torch_export_patches import bypass_export_some_errors
 
 
 class TestLlmPhi(ExtTestCase):
@@ -24,23 +23,6 @@ def test_export_phi2_1(self):
         ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds)
         assert ep
 
-    @ignore_warnings(UserWarning)
-    @requires_transformers("4.52")  # TODO
-    def test_export_phi2_2_bypassed(self):
-        data = get_phi2(num_hidden_layers=2)
-        model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"]
-        self.assertEqual(
-            {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs)
-        )
-        with bypass_export_some_errors(patch_transformers=True) as modificator:
-            inputs = modificator(inputs)
-            ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False)
-            assert ep
-        with bypass_export_some_errors(patch_transformers=True) as modificator:
-            inputs = modificator(inputs)
-            ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False)
-            assert ep
-
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

_unittests/ut_torch_models/test_tiny_llms.py

@@ -1,14 +1,9 @@
 import copy
 import unittest
 import torch
-from transformers.cache_utils import DynamicCache
 from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings, requires_transformers
 from onnx_diagnostic.torch_models.llms import get_tiny_llm
 from onnx_diagnostic.helpers import string_type
-from onnx_diagnostic.torch_export_patches import bypass_export_some_errors
-from onnx_diagnostic.torch_export_patches.patches.patch_transformers import (
-    patched_DynamicCache,
-)
 
 
 class TestTinyLlm(ExtTestCase):
@@ -33,47 +28,6 @@ def test_export_tiny_llm_1(self):
         got = ep.module()(**inputs)
         self.assertEqualArrayAny(expected, got)
 
-    @ignore_warnings(UserWarning)
-    def test_export_tiny_llm_2_bypassed(self):
-        data = get_tiny_llm()
-        model, inputs = data["model"], data["inputs"]
-        expected = model(**copy.deepcopy(inputs))
-        self.assertEqual(
-            {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs)
-        )
-
-        with bypass_export_some_errors(
-            patch_torch=False, patch_transformers=True, catch_constraints=False, verbose=10
-        ) as modificator:
-
-            for k in patched_DynamicCache._PATCHES_:
-                self.assertEqual(getattr(patched_DynamicCache, k), getattr(DynamicCache, k))
-
-            inputs = modificator(copy.deepcopy(inputs))
-
-            def debug():
-                print("***", string_type(inputs, with_shape=True))
-                print("***", data["dynamic_shapes"])
-                import torch.export._draft_export
-
-                ep, report = torch.export._draft_export.draft_export(
-                    model,
-                    (),
-                    kwargs=inputs,
-                    dynamic_shapes=data["dynamic_shapes"],
-                    strict=False,
-                )
-                print(report)
-
-            if self._debug():
-                debug()
-
-            ep = torch.export.export(
-                model, (), kwargs=inputs, dynamic_shapes=data["dynamic_shapes"], strict=False
-            )
-            got = ep.module()(**inputs)
-            self.assertEqualArrayAny(expected, got)
-
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

_unittests/ut_torch_models/test_tiny_llms_bypassed.py

@@ -0,0 +1,75 @@
+import copy
+import unittest
+import torch
+from transformers.cache_utils import DynamicCache
+from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings
+from onnx_diagnostic.torch_models.llms import get_tiny_llm
+from onnx_diagnostic.torch_models.llms import get_phi2
+from onnx_diagnostic.helpers import string_type
+from onnx_diagnostic.torch_export_patches import bypass_export_some_errors
+from onnx_diagnostic.torch_export_patches.patches.patch_transformers import (
+    patched_DynamicCache,
+)
+
+
+class TestTinyLlmBypassed(ExtTestCase):
+    @ignore_warnings(UserWarning)
+    def test_export_tiny_llm_2_bypassed(self):
+        data = get_tiny_llm()
+        model, inputs = data["model"], data["inputs"]
+        expected = model(**copy.deepcopy(inputs))
+        self.assertEqual(
+            {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs)
+        )
+
+        with bypass_export_some_errors(
+            patch_torch=False, patch_transformers=True, catch_constraints=False, verbose=10
+        ) as modificator:
+
+            for k in patched_DynamicCache._PATCHES_:
+                self.assertEqual(getattr(patched_DynamicCache, k), getattr(DynamicCache, k))
+
+            inputs = modificator(copy.deepcopy(inputs))
+
+            def debug():
+                print("***", string_type(inputs, with_shape=True))
+                print("***", data["dynamic_shapes"])
+                import torch.export._draft_export
+
+                ep, report = torch.export._draft_export.draft_export(
+                    model,
+                    (),
+                    kwargs=inputs,
+                    dynamic_shapes=data["dynamic_shapes"],
+                    strict=False,
+                )
+                print(report)
+
+            if self._debug():
+                debug()
+
+            ep = torch.export.export(
+                model, (), kwargs=inputs, dynamic_shapes=data["dynamic_shapes"], strict=False
+            )
+            got = ep.module()(**inputs)
+            self.assertEqualArrayAny(expected, got)
+
+    @ignore_warnings(UserWarning)
+    def test_export_phi2_2_bypassed(self):
+        data = get_phi2(num_hidden_layers=2)
+        model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"]
+        self.assertEqual(
+            {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs)
+        )
+        with bypass_export_some_errors(patch_transformers=True) as modificator:
+            inputs = modificator(inputs)
+            ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False)
+            assert ep
+        with bypass_export_some_errors(patch_transformers=True) as modificator:
+            inputs = modificator(inputs)
+            ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False)
+            assert ep
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

onnx_diagnostic/ext_test_case.py

@@ -399,6 +399,18 @@ def has_cuda() -> bool:
     return torch.cuda.device_count() > 0
 
 
+def requires_python(version: Tuple[int, ...], msg: str = ""):
+    """
+    Skips a test if python is too old.
+
+    :param msg: to overwrite the message
+    :param version: minimum version
+    """
+    if sys.version_info[: len(version)] < version:
+        return unittest.skip(msg or f"python not recent enough {sys.version_info} < {version}")
+    return lambda x: x
+
+
 def requires_cuda(msg: str = "", version: str = "", memory: int = 0):
     """
     Skips a test if cuda is not available.