diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fa75d6e..b142e67e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,27 +16,35 @@ jobs: matrix: os: [ubuntu-latest] python: ['3.10', '3.11', '3.12', '3.13'] - transformers: ['4.48.3', '4.51.3', '4.52.4', '4.53.3', 'main'] + transformers: ['4.48.3', '4.51.3', '4.52.4', '4.53.3', '4.54.0', 'main'] torch: ['2.7', 'main'] exclude: - python: '3.10' torch: 'main' - - python: '3.11' - torch: 'main' + - python: '3.10' + transformers: 'main' - python: '3.10' transformers: '4.52.4' - python: '3.10' - transformers: 'main' + transformers: '4.53.3' + - python: '3.10' + transformers: '4.54.0' + - python: '3.11' + torch: 'main' - python: '3.11' transformers: '4.53.3' - python: '3.11' transformers: 'main' + - python: '3.11' + transformers: '4.54.0' - python: '3.13' torch: '2.7' - python: '3.13' transformers: '4.48.3' - python: '3.13' transformers: '4.51.3' + - python: '3.13' + transformers: '4.52.4' steps: - uses: actions/checkout@v3 @@ -121,8 +129,8 @@ jobs: pip install pytest PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py --ignore _unittests/ut_tasks/test_tasks_zero_shot_image_classification.py --ignore _unittests/ut_tasks/test_tasks_image_classification.py - - name: run backend tests python - run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py + # - name: run backend tests python + # run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py - - name: run backend tests onnxruntime - run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15 + # - name: run backend tests onnxruntime + # run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 52f24b94..8c7148ff 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -85,6 +85,12 @@ jobs: pip install pytest pytest-cov PYTHONPATH=. UNITTEST_GOING=1 UNITTEST_DOT=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py + - name: run backend tests python + run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py + + - name: run backend tests onnxruntime + run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15 + - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: diff --git a/_unittests/ut_helpers/test_helper.py b/_unittests/ut_helpers/test_helper.py index 70abc638..eab075db 100644 --- a/_unittests/ut_helpers/test_helper.py +++ b/_unittests/ut_helpers/test_helper.py @@ -333,6 +333,7 @@ def test_size_type_onnx(self): "FLOAT8E5M2", "FLOAT8E4M3FN", "FLOAT8E4M3FNUZ", + "FLOAT8E8M0", }: onnx_dtype_to_torch_dtype(i) onnx_dtype_to_np_dtype(i) diff --git a/_unittests/ut_reference/test_backend_extended_reference_evaluator.py b/_unittests/ut_reference/test_backend_extended_reference_evaluator.py index 5410dd9b..084356ba 100644 --- a/_unittests/ut_reference/test_backend_extended_reference_evaluator.py +++ b/_unittests/ut_reference/test_backend_extended_reference_evaluator.py @@ -238,6 +238,9 @@ def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs): # The following tests fail due to a type mismatch. backend_test.exclude("(test_eyelike_without_dtype)") +if onnx_opset_version() <= 24: + backend_test.exclude("(attention_3d|causal_expanded)") + # import all test cases at global scope to make them visible to python.unittest globals().update(backend_test.test_cases) diff --git a/_unittests/ut_reference/test_backend_onnxruntime_evaluator.py b/_unittests/ut_reference/test_backend_onnxruntime_evaluator.py index 521aab6f..07d2e3ec 100644 --- a/_unittests/ut_reference/test_backend_onnxruntime_evaluator.py +++ b/_unittests/ut_reference/test_backend_onnxruntime_evaluator.py @@ -231,6 +231,7 @@ def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs): "|test_sub_uint8" "|test_thresholdedrelu" "|test_top_k_uint64" + "|test_training" ")" ) @@ -242,6 +243,30 @@ def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs): ")" ) +if onnx_opset_version() <= 24: + backend_test.exclude( + "(deform_conv" + "|gru" + "|lstm" + "|l1normalization" + "|l2normalization" + "|lpnormalization" + "|maxunpool" + "|attention_3d" + "|causal_expanded" + "|layer_normalization.*expanded" + "|layer_normalization.*expanded" + "|affine_grid.*expanded" + "|test_rnn_seq" + "|test_roialign_aligned_false" + "|test_roialign_aligned_true" + "|test_roialign_mode_max" + "|test_simple_rnn_batchwise" + "|test_simple_rnn_defaults" + "|test_simple_rnn_with_initial_bias" + ")" + ) + # import all test cases at global scope to make them visible to python.unittest globals().update(backend_test.test_cases) diff --git a/onnx_diagnostic/helpers/helper.py b/onnx_diagnostic/helpers/helper.py index f7feb049..789491f4 100644 --- a/onnx_diagnostic/helpers/helper.py +++ b/onnx_diagnostic/helpers/helper.py @@ -36,11 +36,12 @@ def size_type(dtype: Any) -> int: TensorProto.FLOAT8E4M3FNUZ, TensorProto.FLOAT8E5M2, TensorProto.FLOAT8E5M2FNUZ, + getattr(TensorProto, "FLOAT8E8M0", None), }: return 1 if dtype in {TensorProto.COMPLEX128}: return 16 - from .helpers.onnx_helper import onnx_dtype_name + from .onnx_helper import onnx_dtype_name raise AssertionError( f"Unable to return the element size for type {onnx_dtype_name(dtype)}" diff --git a/onnx_diagnostic/reference/ops/op_cast_like.py b/onnx_diagnostic/reference/ops/op_cast_like.py index 5b5d3d18..d2ddff7d 100644 --- a/onnx_diagnostic/reference/ops/op_cast_like.py +++ b/onnx_diagnostic/reference/ops/op_cast_like.py @@ -11,22 +11,26 @@ float8e5m2fnuz, ) except ImportError: + bfloat16 = None from onnx.reference.ops.op_cast import cast_to from ...helpers.onnx_helper import np_dtype_to_tensor_dtype def _cast_like(x, y, saturate): - if y.dtype == bfloat16 and y.dtype.descr[0][0] == "bfloat16": - # np.uint16 == np.uint16 is True as well as np.uint16 == bfloat16 - to = TensorProto.BFLOAT16 - elif y.dtype == float8e4m3fn and y.dtype.descr[0][0] == "e4m3fn": - to = TensorProto.FLOAT8E4M3FN - elif y.dtype == float8e4m3fnuz and y.dtype.descr[0][0] == "e4m3fnuz": - to = TensorProto.FLOAT8E4M3FNUZ - elif y.dtype == float8e5m2 and y.dtype.descr[0][0] == "e5m2": - to = TensorProto.FLOAT8E5M2 - elif y.dtype == float8e5m2fnuz and y.dtype.descr[0][0] == "e5m2fnuz": - to = TensorProto.FLOAT8E5M2FNUZ + if bfloat16 is not None: + if y.dtype == bfloat16 and y.dtype.descr[0][0] == "bfloat16": + # np.uint16 == np.uint16 is True as well as np.uint16 == bfloat16 + to = TensorProto.BFLOAT16 + elif y.dtype == float8e4m3fn and y.dtype.descr[0][0] == "e4m3fn": + to = TensorProto.FLOAT8E4M3FN + elif y.dtype == float8e4m3fnuz and y.dtype.descr[0][0] == "e4m3fnuz": + to = TensorProto.FLOAT8E4M3FNUZ + elif y.dtype == float8e5m2 and y.dtype.descr[0][0] == "e5m2": + to = TensorProto.FLOAT8E5M2 + elif y.dtype == float8e5m2fnuz and y.dtype.descr[0][0] == "e5m2fnuz": + to = TensorProto.FLOAT8E5M2FNUZ + else: + to = np_dtype_to_tensor_dtype(y.dtype) # type: ignore else: to = np_dtype_to_tensor_dtype(y.dtype) # type: ignore return (cast_to(x, to, saturate),)