Fix Qwen2-VL and Qwen2.5-VL config attribute access for Transformers v5 (#1062)

Fixes #1012
⚠️ Dependency: This PR depends on #1060. Please review and merge #1060
first.

## Summary
- Fix `AttributeError: 'Qwen2VLConfig' object has no attribute
'hidden_size'` for Qwen2-VL and Qwen2.5-VL models
- Update test configurations to use the new `text_config` structure
required by Transformers v5

## Changes
1. **Model code** (`src/liger_kernel/transformers/model/qwen2_vl.py`,
`qwen2_5_vl.py`):
- Changed `self.config.hidden_size` →
`self.config.text_config.hidden_size`
- Changed `self.config.vocab_size` →
`self.config.text_config.vocab_size`

2. **Test configurations** (`test/convergence/bf16/test_mini_models.py`,
`fp32/test_mini_models.py`):
- Restructured `mini_qwen2_vl` and `mini_qwen2_5_vl` configurations to
use `text_config` dictionary for text-related parameters

## Background
In Transformers v5, `Qwen2VLConfig` and `Qwen2_5_VLConfig` moved
text-related parameters (such as `hidden_size`, `vocab_size`) into a
nested `text_config` attribute, following the pattern used by other
multimodal models.

## Test plan
- [x] `python -m pytest test/convergence/bf16/test_mini_models.py -k
"qwen2_vl or qwen2_5_vl"` passes
- [x] `python -m pytest test/convergence/fp32/test_mini_models.py -k
"qwen2_vl or qwen2_5_vl"` passes

Author: yukiu00

src/liger_kernel/transformers/model/qwen2_5_vl.py

@@ -5,12 +5,30 @@
 
 import torch
 
+from packaging import version
+from transformers import __version__ as transformers_version
 from transformers.utils import can_return_tuple
 
 from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
 from liger_kernel.transformers.model.loss_utils import unpack_cross_entropy_result
 from liger_kernel.transformers.model.output_classes import LigerQwen2_5_VLCausalLMOutputWithPast
 
+_TRANSFORMERS_V5_OR_LATER = version.parse(transformers_version) >= version.parse("5.0.0")
+
+
+def _get_hidden_size(config) -> int:
+    """Get hidden_size from Qwen2.5VLConfig in a version-aware manner."""
+    if _TRANSFORMERS_V5_OR_LATER:
+        return config.text_config.hidden_size
+    return config.hidden_size
+
+
+def _get_vocab_size(config) -> int:
+    """Get vocab_size from Qwen2.5VLConfig in a version-aware manner."""
+    if _TRANSFORMERS_V5_OR_LATER:
+        return config.text_config.vocab_size
+    return config.vocab_size
+
 
 @can_return_tuple
 def lce_forward(
@@ -129,7 +147,7 @@ def lce_forward(
             lm_head_weight=self.lm_head.weight,
             labels=labels,
             shift_labels=shift_labels,
-            hidden_size=self.config.hidden_size,
+            hidden_size=_get_hidden_size(self.config),
             **kwargs,
         )
         loss, _, token_accuracy = unpack_cross_entropy_result(result)
@@ -142,7 +160,7 @@ def lce_forward(
                 logits=logits,
                 labels=labels,
                 shift_labels=shift_labels,
-                vocab_size=self.config.vocab_size,
+                vocab_size=_get_vocab_size(self.config),
             )
 
     if not return_dict:

src/liger_kernel/transformers/model/qwen2_vl.py

@@ -5,12 +5,30 @@
 
 import torch
 
+from packaging import version
+from transformers import __version__ as transformers_version
 from transformers.utils import can_return_tuple
 
 from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
 from liger_kernel.transformers.model.loss_utils import unpack_cross_entropy_result
 from liger_kernel.transformers.model.output_classes import LigerQwen2VLCausalLMOutputWithPast
 
+_TRANSFORMERS_V5_OR_LATER = version.parse(transformers_version) >= version.parse("5.0.0")
+
+
+def _get_hidden_size(config) -> int:
+    """Get hidden_size from Qwen2VLConfig in a version-aware manner."""
+    if _TRANSFORMERS_V5_OR_LATER:
+        return config.text_config.hidden_size
+    return config.hidden_size
+
+
+def _get_vocab_size(config) -> int:
+    """Get vocab_size from Qwen2VLConfig in a version-aware manner."""
+    if _TRANSFORMERS_V5_OR_LATER:
+        return config.text_config.vocab_size
+    return config.vocab_size
+
 
 @can_return_tuple
 def lce_forward(
@@ -125,7 +143,7 @@ def lce_forward(
             lm_head_weight=self.lm_head.weight,
             labels=labels,
             shift_labels=shift_labels,
-            hidden_size=self.config.hidden_size,
+            hidden_size=_get_hidden_size(self.config),
             **kwargs,
         )
         loss, _, token_accuracy = unpack_cross_entropy_result(result)
@@ -138,7 +156,7 @@ def lce_forward(
                 logits=logits,
                 labels=labels,
                 shift_labels=shift_labels,
-                vocab_size=self.config.vocab_size,
+                vocab_size=_get_vocab_size(self.config),
             )
 
     if not return_dict:

test/convergence/bf16/test_mini_models.py

@@ -711,34 +711,36 @@
         liger_kernel_patch_revert_func=revert_liger_kernel_to_qwen2_vl,
         model_class=Qwen2VLForConditionalGeneration,
         mini_model_config=Qwen2VLConfig(
-            attention_dropout=0.0,
-            # bos and eos set to match the Mistral-7B tokenizer used to create the test dataset
-            # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json
-            bos_token_id=1,  # 151643
-            eos_token_id=2,  # 151645
+            # In transformers v5, text-related parameters must be in text_config
+            text_config={
+                "attention_dropout": 0.0,
+                # bos and eos set to match the Mistral-7B tokenizer used to create the test dataset
+                # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json
+                "bos_token_id": 1,  # 151643
+                "eos_token_id": 2,  # 151645
+                "hidden_act": "silu",
+                "hidden_size": 1536,  # 8192
+                "initializer_range": 0.02,
+                "intermediate_size": 4864,  # 29568
+                "max_position_embeddings": 32768,
+                "max_window_layers": 4,  # 80
+                "num_attention_heads": 12,  # 64
+                "num_hidden_layers": 4,  # 80
+                "num_key_value_heads": 2,  # 8
+                "rms_norm_eps": 1e-6,  # 1e-5
+                "rope_parameters": {
+                    "mrope_section": [16, 24, 24],  # (temporal, height, width)
+                },
+                "sliding_window": 4096,
+                "tie_word_embeddings": False,
+                "use_cache": True,
+                "vocab_size": 32768,  # 152064  # >32k, Mistral-7B tokenizer vocab size
+                "use_sliding_window": False,
+            },
             vision_start_token_id=32765,  # vocab_size - 5
             vision_end_token_id=32766,  # vocab_size - 4
-            vision_token_id=32767,  # vocab_size - 3
             image_token_id=32768,  # vocab_size - 2
             video_token_id=32769,  # vocab_size - 1
-            hidden_act="silu",
-            hidden_size=1536,  # 8192
-            initializer_range=0.02,
-            intermediate_size=4864,  # 29568
-            max_position_embeddings=32768,
-            max_window_layers=4,  # 80
-            num_attention_heads=12,  # 64
-            num_hidden_layers=4,  # 80
-            num_key_value_heads=2,  # 8
-            rms_norm_eps=1e-6,  # 1e-5
-            rope_parameters=dict(
-                mrope_section=[16, 24, 24],  # (temporal, height, width)
-            ),
-            sliding_window=4096,
-            tie_word_embeddings=False,
-            use_cache=True,
-            vocab_size=32768,  # 152064  # >32k, Mistral-7B tokenizer vocab size
-            use_sliding_window=False,
             vision_config={
                 "depth": 4,  # 32
                 "embed_dim": 1280,
@@ -751,7 +753,6 @@
                 "spatial_patch_size": 14,
                 "temporal_patch_size": 2,
             },
-            attn_implementation="sdpa",
         ),
     )
 
@@ -761,34 +762,36 @@
         liger_kernel_patch_revert_func=revert_liger_kernel_to_qwen2_5_vl,
         model_class=Qwen2_5_VLForConditionalGeneration,
         mini_model_config=Qwen2_5_VLConfig(
-            attention_dropout=0.0,
-            # bos and eos set to match the Mistral-7B tokenizer used to create the test dataset
-            # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json
-            bos_token_id=1,  # 151643
-            eos_token_id=2,  # 151645
+            # In transformers v5, text-related parameters must be in text_config
+            text_config={
+                "attention_dropout": 0.0,
+                # bos and eos set to match the Mistral-7B tokenizer used to create the test dataset
+                # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json
+                "bos_token_id": 1,  # 151643
+                "eos_token_id": 2,  # 151645
+                "hidden_act": "silu",
+                "hidden_size": 1536,  # 8192
+                "initializer_range": 0.02,
+                "intermediate_size": 4864,  # 29568
+                "max_position_embeddings": 32768,
+                "max_window_layers": 4,  # 80
+                "num_attention_heads": 12,  # 64
+                "num_hidden_layers": 4,  # 80
+                "num_key_value_heads": 2,  # 8
+                "rms_norm_eps": 1e-6,  # 1e-5
+                "rope_parameters": {
+                    "mrope_section": [16, 24, 24],  # (temporal, height, width)
+                },
+                "sliding_window": 4096,
+                "tie_word_embeddings": False,
+                "use_cache": True,
+                "vocab_size": 32768,  # 152064  # >32k, Mistral-7B tokenizer vocab size
+                "use_sliding_window": False,
+            },
             vision_start_token_id=32765,  # vocab_size - 5
             vision_end_token_id=32766,  # vocab_size - 4
-            vision_token_id=32767,  # vocab_size - 3
             image_token_id=32768,  # vocab_size - 2
             video_token_id=32769,  # vocab_size - 1
-            hidden_act="silu",
-            hidden_size=1536,  # 8192
-            initializer_range=0.02,
-            intermediate_size=4864,  # 29568
-            max_position_embeddings=32768,
-            max_window_layers=4,  # 80
-            num_attention_heads=12,  # 64
-            num_hidden_layers=4,  # 80
-            num_key_value_heads=2,  # 8
-            rms_norm_eps=1e-6,  # 1e-5
-            rope_parameters=dict(
-                mrope_section=[16, 24, 24],  # (temporal, height, width)
-            ),
-            sliding_window=4096,
-            tie_word_embeddings=False,
-            use_cache=True,
-            vocab_size=32768,  # 152064  # >32k, Mistral-7B tokenizer vocab size
-            use_sliding_window=False,
             vision_config={
                 "depth": 4,  # 32
                 "hidden_act": "silu",
@@ -805,7 +808,6 @@
                 "tokens_per_second": 2,
                 "temporal_patch_size": 2,
             },
-            attn_implementation="sdpa",
         ),
     )
 

test/convergence/fp32/test_mini_models.py

@@ -708,34 +708,36 @@
         liger_kernel_patch_revert_func=revert_liger_kernel_to_qwen2_vl,
         model_class=Qwen2VLForConditionalGeneration,
         mini_model_config=Qwen2VLConfig(
-            attention_dropout=0.0,
-            # bos and eos set to match the Mistral-7B tokenizer used to create the test dataset
-            # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json
-            bos_token_id=1,  # 151643
-            eos_token_id=2,  # 151645
+            # In transformers v5, text-related parameters must be in text_config
+            text_config={
+                "attention_dropout": 0.0,
+                # bos and eos set to match the Mistral-7B tokenizer used to create the test dataset
+                # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json
+                "bos_token_id": 1,  # 151643
+                "eos_token_id": 2,  # 151645
+                "hidden_act": "silu",
+                "hidden_size": 1536,  # 8192
+                "initializer_range": 0.02,
+                "intermediate_size": 4864,  # 29568
+                "max_position_embeddings": 32768,
+                "max_window_layers": 4,  # 80
+                "num_attention_heads": 12,  # 64
+                "num_hidden_layers": 4,  # 80
+                "num_key_value_heads": 2,  # 8
+                "rms_norm_eps": 1e-6,  # 1e-5
+                "rope_parameters": {
+                    "mrope_section": [16, 24, 24],  # (temporal, height, width)
+                },
+                "sliding_window": 4096,
+                "tie_word_embeddings": False,
+                "use_cache": True,
+                "vocab_size": 32768,  # 152064  # >32k, Mistral-7B tokenizer vocab size
+                "use_sliding_window": False,
+            },
             vision_start_token_id=32765,  # vocab_size - 5
             vision_end_token_id=32766,  # vocab_size - 4
-            vision_token_id=32767,  # vocab_size - 3
             image_token_id=32768,  # vocab_size - 2
             video_token_id=32769,  # vocab_size - 1
-            hidden_act="silu",
-            hidden_size=1536,  # 8192
-            initializer_range=0.02,
-            intermediate_size=4864,  # 29568
-            max_position_embeddings=32768,
-            max_window_layers=4,  # 80
-            num_attention_heads=12,  # 64
-            num_hidden_layers=4,  # 80
-            num_key_value_heads=2,  # 8
-            rms_norm_eps=1e-6,  # 1e-5
-            rope_parameters=dict(
-                mrope_section=[16, 24, 24],  # (temporal, height, width)
-            ),
-            sliding_window=4096,
-            tie_word_embeddings=False,
-            use_cache=True,
-            vocab_size=32768,  # 152064  # >32k, Mistral-7B tokenizer vocab size
-            use_sliding_window=False,
             vision_config={
                 "depth": 4,  # 32
                 "embed_dim": 1280,
@@ -748,7 +750,6 @@
                 "spatial_patch_size": 14,
                 "temporal_patch_size": 2,
             },
-            attn_implementation="sdpa",
         ),
     )
 
@@ -758,34 +759,36 @@
         liger_kernel_patch_revert_func=revert_liger_kernel_to_qwen2_5_vl,
         model_class=Qwen2_5_VLForConditionalGeneration,
         mini_model_config=Qwen2_5_VLConfig(
-            attention_dropout=0.0,
-            # bos and eos set to match the Mistral-7B tokenizer used to create the test dataset
-            # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json
-            bos_token_id=1,  # 151643
-            eos_token_id=2,  # 151645
+            # In transformers v5, text-related parameters must be in text_config
+            text_config={
+                "attention_dropout": 0.0,
+                # bos and eos set to match the Mistral-7B tokenizer used to create the test dataset
+                # https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json
+                "bos_token_id": 1,  # 151643
+                "eos_token_id": 2,  # 151645
+                "hidden_act": "silu",
+                "hidden_size": 1536,  # 8192
+                "initializer_range": 0.02,
+                "intermediate_size": 4864,  # 29568
+                "max_position_embeddings": 32768,
+                "max_window_layers": 4,  # 80
+                "num_attention_heads": 12,  # 64
+                "num_hidden_layers": 4,  # 80
+                "num_key_value_heads": 2,  # 8
+                "rms_norm_eps": 1e-6,  # 1e-5
+                "rope_parameters": {
+                    "mrope_section": [16, 24, 24],  # (temporal, height, width)
+                },
+                "sliding_window": 4096,
+                "tie_word_embeddings": False,
+                "use_cache": True,
+                "vocab_size": 32768,  # 152064  # >32k, Mistral-7B tokenizer vocab size
+                "use_sliding_window": False,
+            },
             vision_start_token_id=32765,  # vocab_size - 5
             vision_end_token_id=32766,  # vocab_size - 4
-            vision_token_id=32767,  # vocab_size - 3
             image_token_id=32768,  # vocab_size - 2
             video_token_id=32769,  # vocab_size - 1
-            hidden_act="silu",
-            hidden_size=1536,  # 8192
-            initializer_range=0.02,
-            intermediate_size=4864,  # 29568
-            max_position_embeddings=32768,
-            max_window_layers=4,  # 80
-            num_attention_heads=12,  # 64
-            num_hidden_layers=4,  # 80
-            num_key_value_heads=2,  # 8
-            rms_norm_eps=1e-6,  # 1e-5
-            rope_parameters=dict(
-                mrope_section=[16, 24, 24],  # (temporal, height, width)
-            ),
-            sliding_window=4096,
-            tie_word_embeddings=False,
-            use_cache=True,
-            vocab_size=32768,  # 152064  # >32k, Mistral-7B tokenizer vocab size
-            use_sliding_window=False,
             vision_config={
                 "depth": 4,  # 32
                 "hidden_act": "silu",
@@ -802,7 +805,6 @@
                 "tokens_per_second": 2,
                 "temporal_patch_size": 2,
             },
-            attn_implementation="sdpa",
         ),
     )