From a67c7719dec77a79c022524b1bf7615427a54990 Mon Sep 17 00:00:00 2001 From: xadupre Date: Fri, 11 Apr 2025 18:33:43 +0200 Subject: [PATCH 1/6] fix issues --- _unittests/ut_torch_models/test_llm_phi2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/_unittests/ut_torch_models/test_llm_phi2.py b/_unittests/ut_torch_models/test_llm_phi2.py index 9a00d1e8..c843fafd 100644 --- a/_unittests/ut_torch_models/test_llm_phi2.py +++ b/_unittests/ut_torch_models/test_llm_phi2.py @@ -25,7 +25,6 @@ def test_export_phi2_1(self): assert ep @ignore_warnings(UserWarning) - @requires_transformers("4.52") # TODO def test_export_phi2_2_bypassed(self): data = get_phi2(num_hidden_layers=2) model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"] From b2f469fffb7181bbc8160fea258a463fb5f368eb Mon Sep 17 00:00:00 2001 From: xadupre Date: Fri, 11 Apr 2025 19:05:06 +0200 Subject: [PATCH 2/6] try --- _unittests/ut_torch_models/test_llm_phi2.py | 8 +++++++- onnx_diagnostic/ext_test_case.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/_unittests/ut_torch_models/test_llm_phi2.py b/_unittests/ut_torch_models/test_llm_phi2.py index c843fafd..4384d846 100644 --- a/_unittests/ut_torch_models/test_llm_phi2.py +++ b/_unittests/ut_torch_models/test_llm_phi2.py @@ -1,6 +1,11 @@ import unittest import torch -from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings, requires_transformers +from onnx_diagnostic.ext_test_case import ( + ExtTestCase, + ignore_warnings, + requires_transformers, + requires_python, +) from onnx_diagnostic.torch_models.llms import get_phi2 from onnx_diagnostic.helpers import string_type from onnx_diagnostic.torch_export_patches import bypass_export_some_errors @@ -25,6 +30,7 @@ def test_export_phi2_1(self): assert ep @ignore_warnings(UserWarning) + @requires_python((3, 12)) def test_export_phi2_2_bypassed(self): data = get_phi2(num_hidden_layers=2) model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"] diff --git a/onnx_diagnostic/ext_test_case.py b/onnx_diagnostic/ext_test_case.py index 4e2d1c60..a3e76eaa 100644 --- a/onnx_diagnostic/ext_test_case.py +++ b/onnx_diagnostic/ext_test_case.py @@ -399,6 +399,18 @@ def has_cuda() -> bool: return torch.cuda.device_count() > 0 +def requires_python(version: Tuple[int,...], msg: str = ""): + """ + Skips a test if python is too old. + + :param msg: to overwrite the message + :param version: minimum version + """ + if sys.version_info[:len(version)] < version: + return unittest.skip(msg or f"python not recent enough {sys.version_info} < {version}") + return lambda x: x + + def requires_cuda(msg: str = "", version: str = "", memory: int = 0): """ Skips a test if cuda is not available. From ef9f070350d896f8ff724cc26aa9069fe9f0fc00 Mon Sep 17 00:00:00 2001 From: xadupre Date: Fri, 11 Apr 2025 19:34:41 +0200 Subject: [PATCH 3/6] unstable --- onnx_diagnostic/ext_test_case.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnx_diagnostic/ext_test_case.py b/onnx_diagnostic/ext_test_case.py index a3e76eaa..fb786e38 100644 --- a/onnx_diagnostic/ext_test_case.py +++ b/onnx_diagnostic/ext_test_case.py @@ -399,14 +399,14 @@ def has_cuda() -> bool: return torch.cuda.device_count() > 0 -def requires_python(version: Tuple[int,...], msg: str = ""): +def requires_python(version: Tuple[int, ...], msg: str = ""): """ Skips a test if python is too old. :param msg: to overwrite the message :param version: minimum version """ - if sys.version_info[:len(version)] < version: + if sys.version_info[: len(version)] < version: return unittest.skip(msg or f"python not recent enough {sys.version_info} < {version}") return lambda x: x From 8f22a2279575edf83cd6d8d52859a47200d3007c Mon Sep 17 00:00:00 2001 From: xadupre Date: Sat, 12 Apr 2025 10:37:27 +0200 Subject: [PATCH 4/6] ci --- .github/workflows/ci.yml | 35 ++++------- .github/workflows/documentation.yml | 26 +++------ _unittests/ut_torch_models/test_tiny_llms.py | 46 --------------- .../test_tiny_llms_bypassed.py | 58 +++++++++++++++++++ 4 files changed, 77 insertions(+), 88 deletions(-) create mode 100644 _unittests/ut_torch_models/test_tiny_llms_bypassed.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff6a7916..e9aa96db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,45 +67,30 @@ jobs: run: python -m pip freeze - name: tiny-llm torch.export.export - run: | - export PYTHONPATH=. - python _unittests/ut_torch_models/test_tiny_llms.py + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms.py - name: tiny-llm onnx - run: | - export PYTHONPATH=. - python _unittests/ut_torch_models/test_tiny_llms_onnx.py + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_onnx.py continue-on-error: true # connectivity issues - name: tiny-llm example - run: | - export PYTHONPATH=. - python _doc/examples/plot_export_tiny_llm.py + run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm.py continue-on-error: true # connectivity issues - name: tiny-llm bypass - run: | - export PYTHONPATH=. - python _doc/examples/plot_export_tiny_llm_patched.py + run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm_patched.py continue-on-error: true # connectivity issues + - name: run tests bypassed + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_bypassed.py + - name: run tests run: | pip install pytest - export PYTHONPATH=. - UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py - export PYTHONPATH= + PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py - name: run backend tests python - run: | - pip install pytest - export PYTHONPATH=. - UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py - export PYTHONPATH= + run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py - name: run backend tests onnxruntime - run: | - pip install pytest - export PYTHONPATH=. - UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15 - export PYTHONPATH= + run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 894a214f..807d6753 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -54,33 +54,25 @@ jobs: run: python -m pip freeze - name: tiny-llm torch.export.export - run: | - export PYTHONPATH=. - python _unittests/ut_torch_models/test_tiny_llms.py + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms.py - name: tiny-llm onnx - run: | - export PYTHONPATH=. - python _unittests/ut_torch_models/test_tiny_llms_onnx.py + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_onnx.py continue-on-error: true - name: tiny-llm example - run: | - export PYTHONPATH=. - python _doc/examples/plot_export_tiny_llm.py + run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm.py - name: tiny-llm bypass - run: | - export PYTHONPATH=. - python _doc/examples/plot_export_tiny_llm_patched.py + run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm_patched.py + + - name: run tests bypassed + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_bypassed.py - name: Generate coverage report run: | - pip install pytest - pip install pytest-cov - export PYTHONPATH=. - UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py - export PYTHONPATH= + pip install pytest pytest-cov + export PYTHONPATH=. UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 diff --git a/_unittests/ut_torch_models/test_tiny_llms.py b/_unittests/ut_torch_models/test_tiny_llms.py index 4350bb68..909bdc05 100644 --- a/_unittests/ut_torch_models/test_tiny_llms.py +++ b/_unittests/ut_torch_models/test_tiny_llms.py @@ -1,14 +1,9 @@ import copy import unittest import torch -from transformers.cache_utils import DynamicCache from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings, requires_transformers from onnx_diagnostic.torch_models.llms import get_tiny_llm from onnx_diagnostic.helpers import string_type -from onnx_diagnostic.torch_export_patches import bypass_export_some_errors -from onnx_diagnostic.torch_export_patches.patches.patch_transformers import ( - patched_DynamicCache, -) class TestTinyLlm(ExtTestCase): @@ -33,47 +28,6 @@ def test_export_tiny_llm_1(self): got = ep.module()(**inputs) self.assertEqualArrayAny(expected, got) - @ignore_warnings(UserWarning) - def test_export_tiny_llm_2_bypassed(self): - data = get_tiny_llm() - model, inputs = data["model"], data["inputs"] - expected = model(**copy.deepcopy(inputs)) - self.assertEqual( - {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs) - ) - - with bypass_export_some_errors( - patch_torch=False, patch_transformers=True, catch_constraints=False, verbose=10 - ) as modificator: - - for k in patched_DynamicCache._PATCHES_: - self.assertEqual(getattr(patched_DynamicCache, k), getattr(DynamicCache, k)) - - inputs = modificator(copy.deepcopy(inputs)) - - def debug(): - print("***", string_type(inputs, with_shape=True)) - print("***", data["dynamic_shapes"]) - import torch.export._draft_export - - ep, report = torch.export._draft_export.draft_export( - model, - (), - kwargs=inputs, - dynamic_shapes=data["dynamic_shapes"], - strict=False, - ) - print(report) - - if self._debug(): - debug() - - ep = torch.export.export( - model, (), kwargs=inputs, dynamic_shapes=data["dynamic_shapes"], strict=False - ) - got = ep.module()(**inputs) - self.assertEqualArrayAny(expected, got) - if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/_unittests/ut_torch_models/test_tiny_llms_bypassed.py b/_unittests/ut_torch_models/test_tiny_llms_bypassed.py new file mode 100644 index 00000000..36de0ed9 --- /dev/null +++ b/_unittests/ut_torch_models/test_tiny_llms_bypassed.py @@ -0,0 +1,58 @@ +import copy +import unittest +import torch +from transformers.cache_utils import DynamicCache +from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings +from onnx_diagnostic.torch_models.llms import get_tiny_llm +from onnx_diagnostic.helpers import string_type +from onnx_diagnostic.torch_export_patches import bypass_export_some_errors +from onnx_diagnostic.torch_export_patches.patches.patch_transformers import ( + patched_DynamicCache, +) + + +class TestTinyLlmBypassed(ExtTestCase): + @ignore_warnings(UserWarning) + def test_export_tiny_llm_2_bypassed(self): + data = get_tiny_llm() + model, inputs = data["model"], data["inputs"] + expected = model(**copy.deepcopy(inputs)) + self.assertEqual( + {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs) + ) + + with bypass_export_some_errors( + patch_torch=False, patch_transformers=True, catch_constraints=False, verbose=10 + ) as modificator: + + for k in patched_DynamicCache._PATCHES_: + self.assertEqual(getattr(patched_DynamicCache, k), getattr(DynamicCache, k)) + + inputs = modificator(copy.deepcopy(inputs)) + + def debug(): + print("***", string_type(inputs, with_shape=True)) + print("***", data["dynamic_shapes"]) + import torch.export._draft_export + + ep, report = torch.export._draft_export.draft_export( + model, + (), + kwargs=inputs, + dynamic_shapes=data["dynamic_shapes"], + strict=False, + ) + print(report) + + if self._debug(): + debug() + + ep = torch.export.export( + model, (), kwargs=inputs, dynamic_shapes=data["dynamic_shapes"], strict=False + ) + got = ep.module()(**inputs) + self.assertEqualArrayAny(expected, got) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 1e0e456072e57c0cecdc4fdde5982b4fef9f481b Mon Sep 17 00:00:00 2001 From: xadupre Date: Sat, 12 Apr 2025 10:42:19 +0200 Subject: [PATCH 5/6] fix ci --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 807d6753..b461325b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -72,7 +72,7 @@ jobs: - name: Generate coverage report run: | pip install pytest pytest-cov - export PYTHONPATH=. UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py + PYTHONPATH=. UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 From 21b7e236d49de030069f8e445547473dcd685632 Mon Sep 17 00:00:00 2001 From: xadupre Date: Sat, 12 Apr 2025 10:53:10 +0200 Subject: [PATCH 6/6] bypass --- _unittests/ut_torch_models/test_llm_phi2.py | 25 +------------------ .../test_tiny_llms_bypassed.py | 17 +++++++++++++ 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/_unittests/ut_torch_models/test_llm_phi2.py b/_unittests/ut_torch_models/test_llm_phi2.py index 4384d846..af4f616a 100644 --- a/_unittests/ut_torch_models/test_llm_phi2.py +++ b/_unittests/ut_torch_models/test_llm_phi2.py @@ -1,14 +1,8 @@ import unittest import torch -from onnx_diagnostic.ext_test_case import ( - ExtTestCase, - ignore_warnings, - requires_transformers, - requires_python, -) +from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings, requires_transformers from onnx_diagnostic.torch_models.llms import get_phi2 from onnx_diagnostic.helpers import string_type -from onnx_diagnostic.torch_export_patches import bypass_export_some_errors class TestLlmPhi(ExtTestCase): @@ -29,23 +23,6 @@ def test_export_phi2_1(self): ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds) assert ep - @ignore_warnings(UserWarning) - @requires_python((3, 12)) - def test_export_phi2_2_bypassed(self): - data = get_phi2(num_hidden_layers=2) - model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"] - self.assertEqual( - {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs) - ) - with bypass_export_some_errors(patch_transformers=True) as modificator: - inputs = modificator(inputs) - ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False) - assert ep - with bypass_export_some_errors(patch_transformers=True) as modificator: - inputs = modificator(inputs) - ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False) - assert ep - if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/_unittests/ut_torch_models/test_tiny_llms_bypassed.py b/_unittests/ut_torch_models/test_tiny_llms_bypassed.py index 36de0ed9..4ec3e943 100644 --- a/_unittests/ut_torch_models/test_tiny_llms_bypassed.py +++ b/_unittests/ut_torch_models/test_tiny_llms_bypassed.py @@ -4,6 +4,7 @@ from transformers.cache_utils import DynamicCache from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings from onnx_diagnostic.torch_models.llms import get_tiny_llm +from onnx_diagnostic.torch_models.llms import get_phi2 from onnx_diagnostic.helpers import string_type from onnx_diagnostic.torch_export_patches import bypass_export_some_errors from onnx_diagnostic.torch_export_patches.patches.patch_transformers import ( @@ -53,6 +54,22 @@ def debug(): got = ep.module()(**inputs) self.assertEqualArrayAny(expected, got) + @ignore_warnings(UserWarning) + def test_export_phi2_2_bypassed(self): + data = get_phi2(num_hidden_layers=2) + model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"] + self.assertEqual( + {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs) + ) + with bypass_export_some_errors(patch_transformers=True) as modificator: + inputs = modificator(inputs) + ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False) + assert ep + with bypass_export_some_errors(patch_transformers=True) as modificator: + inputs = modificator(inputs) + ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False) + assert ep + if __name__ == "__main__": unittest.main(verbosity=2)