diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff6a7916..e9aa96db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,45 +67,30 @@ jobs: run: python -m pip freeze - name: tiny-llm torch.export.export - run: | - export PYTHONPATH=. - python _unittests/ut_torch_models/test_tiny_llms.py + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms.py - name: tiny-llm onnx - run: | - export PYTHONPATH=. - python _unittests/ut_torch_models/test_tiny_llms_onnx.py + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_onnx.py continue-on-error: true # connectivity issues - name: tiny-llm example - run: | - export PYTHONPATH=. - python _doc/examples/plot_export_tiny_llm.py + run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm.py continue-on-error: true # connectivity issues - name: tiny-llm bypass - run: | - export PYTHONPATH=. - python _doc/examples/plot_export_tiny_llm_patched.py + run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm_patched.py continue-on-error: true # connectivity issues + - name: run tests bypassed + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_bypassed.py + - name: run tests run: | pip install pytest - export PYTHONPATH=. - UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py - export PYTHONPATH= + PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py - name: run backend tests python - run: | - pip install pytest - export PYTHONPATH=. - UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py - export PYTHONPATH= + run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py - name: run backend tests onnxruntime - run: | - pip install pytest - export PYTHONPATH=. - UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15 - export PYTHONPATH= + run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 894a214f..b461325b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -54,33 +54,25 @@ jobs: run: python -m pip freeze - name: tiny-llm torch.export.export - run: | - export PYTHONPATH=. - python _unittests/ut_torch_models/test_tiny_llms.py + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms.py - name: tiny-llm onnx - run: | - export PYTHONPATH=. - python _unittests/ut_torch_models/test_tiny_llms_onnx.py + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_onnx.py continue-on-error: true - name: tiny-llm example - run: | - export PYTHONPATH=. - python _doc/examples/plot_export_tiny_llm.py + run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm.py - name: tiny-llm bypass - run: | - export PYTHONPATH=. - python _doc/examples/plot_export_tiny_llm_patched.py + run: PYTHONPATH=. python _doc/examples/plot_export_tiny_llm_patched.py + + - name: run tests bypassed + run: PYTHONPATH=. python _unittests/ut_torch_models/test_tiny_llms_bypassed.py - name: Generate coverage report run: | - pip install pytest - pip install pytest-cov - export PYTHONPATH=. - UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py - export PYTHONPATH= + pip install pytest pytest-cov + PYTHONPATH=. UNITTEST_GOING=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 diff --git a/_unittests/ut_torch_models/test_llm_phi2.py b/_unittests/ut_torch_models/test_llm_phi2.py index 9a00d1e8..af4f616a 100644 --- a/_unittests/ut_torch_models/test_llm_phi2.py +++ b/_unittests/ut_torch_models/test_llm_phi2.py @@ -3,7 +3,6 @@ from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings, requires_transformers from onnx_diagnostic.torch_models.llms import get_phi2 from onnx_diagnostic.helpers import string_type -from onnx_diagnostic.torch_export_patches import bypass_export_some_errors class TestLlmPhi(ExtTestCase): @@ -24,23 +23,6 @@ def test_export_phi2_1(self): ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds) assert ep - @ignore_warnings(UserWarning) - @requires_transformers("4.52") # TODO - def test_export_phi2_2_bypassed(self): - data = get_phi2(num_hidden_layers=2) - model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"] - self.assertEqual( - {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs) - ) - with bypass_export_some_errors(patch_transformers=True) as modificator: - inputs = modificator(inputs) - ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False) - assert ep - with bypass_export_some_errors(patch_transformers=True) as modificator: - inputs = modificator(inputs) - ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False) - assert ep - if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/_unittests/ut_torch_models/test_tiny_llms.py b/_unittests/ut_torch_models/test_tiny_llms.py index 4350bb68..909bdc05 100644 --- a/_unittests/ut_torch_models/test_tiny_llms.py +++ b/_unittests/ut_torch_models/test_tiny_llms.py @@ -1,14 +1,9 @@ import copy import unittest import torch -from transformers.cache_utils import DynamicCache from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings, requires_transformers from onnx_diagnostic.torch_models.llms import get_tiny_llm from onnx_diagnostic.helpers import string_type -from onnx_diagnostic.torch_export_patches import bypass_export_some_errors -from onnx_diagnostic.torch_export_patches.patches.patch_transformers import ( - patched_DynamicCache, -) class TestTinyLlm(ExtTestCase): @@ -33,47 +28,6 @@ def test_export_tiny_llm_1(self): got = ep.module()(**inputs) self.assertEqualArrayAny(expected, got) - @ignore_warnings(UserWarning) - def test_export_tiny_llm_2_bypassed(self): - data = get_tiny_llm() - model, inputs = data["model"], data["inputs"] - expected = model(**copy.deepcopy(inputs)) - self.assertEqual( - {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs) - ) - - with bypass_export_some_errors( - patch_torch=False, patch_transformers=True, catch_constraints=False, verbose=10 - ) as modificator: - - for k in patched_DynamicCache._PATCHES_: - self.assertEqual(getattr(patched_DynamicCache, k), getattr(DynamicCache, k)) - - inputs = modificator(copy.deepcopy(inputs)) - - def debug(): - print("***", string_type(inputs, with_shape=True)) - print("***", data["dynamic_shapes"]) - import torch.export._draft_export - - ep, report = torch.export._draft_export.draft_export( - model, - (), - kwargs=inputs, - dynamic_shapes=data["dynamic_shapes"], - strict=False, - ) - print(report) - - if self._debug(): - debug() - - ep = torch.export.export( - model, (), kwargs=inputs, dynamic_shapes=data["dynamic_shapes"], strict=False - ) - got = ep.module()(**inputs) - self.assertEqualArrayAny(expected, got) - if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/_unittests/ut_torch_models/test_tiny_llms_bypassed.py b/_unittests/ut_torch_models/test_tiny_llms_bypassed.py new file mode 100644 index 00000000..4ec3e943 --- /dev/null +++ b/_unittests/ut_torch_models/test_tiny_llms_bypassed.py @@ -0,0 +1,75 @@ +import copy +import unittest +import torch +from transformers.cache_utils import DynamicCache +from onnx_diagnostic.ext_test_case import ExtTestCase, ignore_warnings +from onnx_diagnostic.torch_models.llms import get_tiny_llm +from onnx_diagnostic.torch_models.llms import get_phi2 +from onnx_diagnostic.helpers import string_type +from onnx_diagnostic.torch_export_patches import bypass_export_some_errors +from onnx_diagnostic.torch_export_patches.patches.patch_transformers import ( + patched_DynamicCache, +) + + +class TestTinyLlmBypassed(ExtTestCase): + @ignore_warnings(UserWarning) + def test_export_tiny_llm_2_bypassed(self): + data = get_tiny_llm() + model, inputs = data["model"], data["inputs"] + expected = model(**copy.deepcopy(inputs)) + self.assertEqual( + {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs) + ) + + with bypass_export_some_errors( + patch_torch=False, patch_transformers=True, catch_constraints=False, verbose=10 + ) as modificator: + + for k in patched_DynamicCache._PATCHES_: + self.assertEqual(getattr(patched_DynamicCache, k), getattr(DynamicCache, k)) + + inputs = modificator(copy.deepcopy(inputs)) + + def debug(): + print("***", string_type(inputs, with_shape=True)) + print("***", data["dynamic_shapes"]) + import torch.export._draft_export + + ep, report = torch.export._draft_export.draft_export( + model, + (), + kwargs=inputs, + dynamic_shapes=data["dynamic_shapes"], + strict=False, + ) + print(report) + + if self._debug(): + debug() + + ep = torch.export.export( + model, (), kwargs=inputs, dynamic_shapes=data["dynamic_shapes"], strict=False + ) + got = ep.module()(**inputs) + self.assertEqualArrayAny(expected, got) + + @ignore_warnings(UserWarning) + def test_export_phi2_2_bypassed(self): + data = get_phi2(num_hidden_layers=2) + model, inputs, ds = data["model"], data["inputs"], data["dynamic_shapes"] + self.assertEqual( + {"attention_mask", "past_key_values", "input_ids", "position_ids"}, set(inputs) + ) + with bypass_export_some_errors(patch_transformers=True) as modificator: + inputs = modificator(inputs) + ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False) + assert ep + with bypass_export_some_errors(patch_transformers=True) as modificator: + inputs = modificator(inputs) + ep = torch.export.export(model, (), kwargs=inputs, dynamic_shapes=ds, strict=False) + assert ep + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/onnx_diagnostic/ext_test_case.py b/onnx_diagnostic/ext_test_case.py index 4e2d1c60..fb786e38 100644 --- a/onnx_diagnostic/ext_test_case.py +++ b/onnx_diagnostic/ext_test_case.py @@ -399,6 +399,18 @@ def has_cuda() -> bool: return torch.cuda.device_count() > 0 +def requires_python(version: Tuple[int, ...], msg: str = ""): + """ + Skips a test if python is too old. + + :param msg: to overwrite the message + :param version: minimum version + """ + if sys.version_info[: len(version)] < version: + return unittest.skip(msg or f"python not recent enough {sys.version_info} < {version}") + return lambda x: x + + def requires_cuda(msg: str = "", version: str = "", memory: int = 0): """ Skips a test if cuda is not available.