Add transformers==4.54.0 to CI (#195)

* Add transformers==4.54.0 to CI

* fix a couple of issues

* issue

* disable

* fix issues

* update ci

* disable tests

Author: xadupre

.github/workflows/ci.yml

@@ -16,27 +16,35 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python: ['3.10', '3.11', '3.12', '3.13']
-        transformers: ['4.48.3', '4.51.3', '4.52.4', '4.53.3', 'main']
+        transformers: ['4.48.3', '4.51.3', '4.52.4', '4.53.3', '4.54.0', 'main']
         torch: ['2.7', 'main']
         exclude:
           - python: '3.10'
             torch: 'main'
-          - python: '3.11'
-            torch: 'main'
+          - python: '3.10'
+            transformers: 'main'
           - python: '3.10'
             transformers: '4.52.4'
           - python: '3.10'
-            transformers: 'main'
+            transformers: '4.53.3'
+          - python: '3.10'
+            transformers: '4.54.0'
+          - python: '3.11'
+            torch: 'main'
           - python: '3.11'
             transformers: '4.53.3'
           - python: '3.11'
             transformers: 'main'
+          - python: '3.11'
+            transformers: '4.54.0'
           - python: '3.13'
             torch: '2.7'
           - python: '3.13'
             transformers: '4.48.3'
           - python: '3.13'
             transformers: '4.51.3'
+          - python: '3.13'
+            transformers: '4.52.4'
     steps:
       - uses: actions/checkout@v3
 
@@ -121,8 +129,8 @@ jobs:
           pip install pytest
           PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py --ignore _unittests/ut_tasks/test_tasks_zero_shot_image_classification.py --ignore _unittests/ut_tasks/test_tasks_image_classification.py
 
-      - name: run backend tests python
-        run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py
+      # - name: run backend tests python
+      #   run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py
 
-      - name: run backend tests onnxruntime
-        run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15
+      # - name: run backend tests onnxruntime
+      #   run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15

.github/workflows/documentation.yml

@@ -85,6 +85,12 @@ jobs:
           pip install pytest pytest-cov          
           PYTHONPATH=. UNITTEST_GOING=1 UNITTEST_DOT=1 pytest --cov=./onnx_diagnostic/ --cov-report=xml --durations=10 _unittests --ignore _unittests/ut_reference/test_backend_extended_reference_evaluator.py  --ignore _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --ignore _unittests/ut_torch_models/test_tiny_llms_bypassed.py
 
+      - name: run backend tests python
+        run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_extended_reference_evaluator.py
+
+      - name: run backend tests onnxruntime
+        run: PYTHONPATH=. UNITTEST_GOING=1 pytest --durations=10 _unittests/ut_reference/test_backend_onnxruntime_evaluator.py --maxfail=15
+
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
         env:

_unittests/ut_helpers/test_helper.py

@@ -333,6 +333,7 @@ def test_size_type_onnx(self):
                     "FLOAT8E5M2",
                     "FLOAT8E4M3FN",
                     "FLOAT8E4M3FNUZ",
+                    "FLOAT8E8M0",
                 }:
                     onnx_dtype_to_torch_dtype(i)
                     onnx_dtype_to_np_dtype(i)

_unittests/ut_reference/test_backend_extended_reference_evaluator.py

@@ -238,6 +238,9 @@ def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs):
     # The following tests fail due to a type mismatch.
     backend_test.exclude("(test_eyelike_without_dtype)")
 
+if onnx_opset_version() <= 24:
+    backend_test.exclude("(attention_3d|causal_expanded)")
+
 
 # import all test cases at global scope to make them visible to python.unittest
 globals().update(backend_test.test_cases)

_unittests/ut_reference/test_backend_onnxruntime_evaluator.py

@@ -231,6 +231,7 @@ def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs):
     "|test_sub_uint8"
     "|test_thresholdedrelu"
     "|test_top_k_uint64"
+    "|test_training"
     ")"
 )
 
@@ -242,6 +243,30 @@ def run_node(cls, node, inputs, device=None, outputs_info=None, **kwargs):
     ")"
 )
 
+if onnx_opset_version() <= 24:
+    backend_test.exclude(
+        "(deform_conv"
+        "|gru"
+        "|lstm"
+        "|l1normalization"
+        "|l2normalization"
+        "|lpnormalization"
+        "|maxunpool"
+        "|attention_3d"
+        "|causal_expanded"
+        "|layer_normalization.*expanded"
+        "|layer_normalization.*expanded"
+        "|affine_grid.*expanded"
+        "|test_rnn_seq"
+        "|test_roialign_aligned_false"
+        "|test_roialign_aligned_true"
+        "|test_roialign_mode_max"
+        "|test_simple_rnn_batchwise"
+        "|test_simple_rnn_defaults"
+        "|test_simple_rnn_with_initial_bias"
+        ")"
+    )
+
 
 # import all test cases at global scope to make them visible to python.unittest
 globals().update(backend_test.test_cases)

onnx_diagnostic/helpers/helper.py

@@ -36,11 +36,12 @@ def size_type(dtype: Any) -> int:
             TensorProto.FLOAT8E4M3FNUZ,
             TensorProto.FLOAT8E5M2,
             TensorProto.FLOAT8E5M2FNUZ,
+            getattr(TensorProto, "FLOAT8E8M0", None),
         }:
             return 1
         if dtype in {TensorProto.COMPLEX128}:
             return 16
-        from .helpers.onnx_helper import onnx_dtype_name
+        from .onnx_helper import onnx_dtype_name
 
         raise AssertionError(
             f"Unable to return the element size for type {onnx_dtype_name(dtype)}"

onnx_diagnostic/reference/ops/op_cast_like.py

@@ -11,22 +11,26 @@
         float8e5m2fnuz,
     )
 except ImportError:
+    bfloat16 = None
     from onnx.reference.ops.op_cast import cast_to
 from ...helpers.onnx_helper import np_dtype_to_tensor_dtype
 
 
 def _cast_like(x, y, saturate):
-    if y.dtype == bfloat16 and y.dtype.descr[0][0] == "bfloat16":
-        # np.uint16 == np.uint16 is True as well as np.uint16 == bfloat16
-        to = TensorProto.BFLOAT16
-    elif y.dtype == float8e4m3fn and y.dtype.descr[0][0] == "e4m3fn":
-        to = TensorProto.FLOAT8E4M3FN
-    elif y.dtype == float8e4m3fnuz and y.dtype.descr[0][0] == "e4m3fnuz":
-        to = TensorProto.FLOAT8E4M3FNUZ
-    elif y.dtype == float8e5m2 and y.dtype.descr[0][0] == "e5m2":
-        to = TensorProto.FLOAT8E5M2
-    elif y.dtype == float8e5m2fnuz and y.dtype.descr[0][0] == "e5m2fnuz":
-        to = TensorProto.FLOAT8E5M2FNUZ
+    if bfloat16 is not None:
+        if y.dtype == bfloat16 and y.dtype.descr[0][0] == "bfloat16":
+            # np.uint16 == np.uint16 is True as well as np.uint16 == bfloat16
+            to = TensorProto.BFLOAT16
+        elif y.dtype == float8e4m3fn and y.dtype.descr[0][0] == "e4m3fn":
+            to = TensorProto.FLOAT8E4M3FN
+        elif y.dtype == float8e4m3fnuz and y.dtype.descr[0][0] == "e4m3fnuz":
+            to = TensorProto.FLOAT8E4M3FNUZ
+        elif y.dtype == float8e5m2 and y.dtype.descr[0][0] == "e5m2":
+            to = TensorProto.FLOAT8E5M2
+        elif y.dtype == float8e5m2fnuz and y.dtype.descr[0][0] == "e5m2fnuz":
+            to = TensorProto.FLOAT8E5M2FNUZ
+        else:
+            to = np_dtype_to_tensor_dtype(y.dtype)  # type: ignore
     else:
         to = np_dtype_to_tensor_dtype(y.dtype)  # type: ignore
     return (cast_to(x, to, saturate),)