GLM-4.5V Model Support (#39805)

* init

* update

* uupdate

* ruff

* t patch is 2 defalut not 1

* draft

* back

* back1

* update

* config update

* update using glm-41 format

* add self.rope_scaling = config.rope_scaling

* update config

* update

* remove the processor

* update

* fix tests

* update

* for test

* update

* update 2126

* self.rope_scaling is missing in GLM4MOE lets add it

* update

* update

* Update modular_glm4v_moe.py

* change config

* update apply_multimodal_rotary_pos_emb

* format

* update

* Delete 3-rollout_qas_thinking_answers.py

* use right name

* update with place holder

* update

* use right rotary

* Update image_processing_glm4v_fast.py

* rope_config_validation needs to rewrite the entire config file in modular

* update

* changed name

* update

* Update modeling_glm4v_moe.py

* _init_weights shoud be add in Glm4vMoePreTrainedModel

* remove use_qk_norm

* Update modular_glm4v_moe.py

* remove use_qk_norm as it is not use

* fix style

* deprecations are not needed on new models

* fix merge issues

---------

Co-authored-by: raushan <[email protected]>
Co-authored-by: Arthur <[email protected]>
Co-authored-by: Arthur <[email protected]>

Author: 3 people

docs/source/en/_toctree.yml

@@ -1009,6 +1009,8 @@
         title: GIT
       - local: model_doc/glm4v
         title: glm4v
+      - local: model_doc/glm4v_moe
+        title: glm4v_moe
       - local: model_doc/got_ocr2
         title: GOT-OCR2
       - local: model_doc/granitevision

docs/source/en/model_doc/glm4v_moe.md

@@ -0,0 +1,64 @@
+<!--Copyright 2025 The ZhipuAI Inc. and The HuggingFace Inc. team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+
+-->
+
+<div style="float: right;">
+    <div class="flex flex-wrap space-x-1">
+<img alt="PyTorch" src="https://img.shields.io/badge/PyTorch-DE3412?style=flat&logo=pytorch&logoColor=white">
+<img alt="FlashAttention" src="https://img.shields.io/badge/%E2%9A%A1%EF%B8%8E%20FlashAttention-eae0c8?style=flat">
+<img alt="SDPA" src="https://img.shields.io/badge/SDPA-DE3412?style=flat&logo=pytorch&logoColor=white">    </div>
+</div>
+
+# Glm4vMoe
+
+## Overview
+
+The Glm4vMoe model was proposed in [<INSERT PAPER NAME HERE>](<INSERT PAPER LINK HERE>) by <INSERT AUTHORS HERE>.
+<INSERT SHORT SUMMARY HERE>
+
+The abstract from the paper is the following:
+
+*<INSERT PAPER ABSTRACT HERE>*
+
+Tips:
+
+<INSERT TIPS ABOUT MODEL HERE>
+
+This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/<INSERT YOUR HF USERNAME HERE>).
+The original code can be found [here](<INSERT LINK TO GITHUB REPO HERE>).
+
+
+## Glm4vMoeConfig
+
+[[autodoc]] Glm4vMoeConfig
+
+## Glm4vMoeTextConfig
+
+[[autodoc]] Glm4vMoeTextConfig
+
+## Glm4vMoeTextModel
+
+[[autodoc]] Glm4vMoeTextModel
+    - forward
+
+## Glm4vMoeModel
+
+[[autodoc]] Glm4vMoeModel
+    - forward
+
+## Glm4vMoeForConditionalGeneration
+
+[[autodoc]] Glm4vMoeForConditionalGeneration
+    - forward

src/transformers/models/auto/configuration_auto.py

@@ -163,6 +163,8 @@
         ("glm4", "Glm4Config"),
         ("glm4_moe", "Glm4MoeConfig"),
         ("glm4v", "Glm4vConfig"),
+        ("glm4v_moe", "Glm4vMoeConfig"),
+        ("glm4v_moe_text", "Glm4vMoeTextConfig"),
         ("glm4v_text", "Glm4vTextConfig"),
         ("glpn", "GLPNConfig"),
         ("got_ocr2", "GotOcr2Config"),
@@ -569,6 +571,8 @@
         ("glm4", "GLM4"),
         ("glm4_moe", "Glm4MoE"),
         ("glm4v", "GLM4V"),
+        ("glm4v_moe", "GLM4VMOE"),
+        ("glm4v_moe_text", "GLM4VMOE"),
         ("glm4v_text", "GLM4V"),
         ("glpn", "GLPN"),
         ("got_ocr2", "GOT-OCR2"),
@@ -900,6 +904,7 @@
         ("gemma3n_text", "gemma3n"),
         ("gemma3n_vision", "gemma3n"),
         ("glm4v_text", "glm4v"),
+        ("glm4v_moe_text", "glm4v_moe"),
         ("idefics3_vision", "idefics3"),
         ("siglip_vision_model", "siglip"),
         ("aimv2_vision_model", "aimv2"),

src/transformers/models/auto/modeling_auto.py

@@ -165,6 +165,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("glm4", "Glm4Model"),
         ("glm4_moe", "Glm4MoeModel"),
         ("glm4v", "Glm4vModel"),
+        ("glm4v_moe", "Glm4vMoeModel"),
+        ("glm4v_moe_text", "Glm4vMoeTextModel"),
         ("glm4v_text", "Glm4vTextModel"),
         ("glpn", "GLPNModel"),
         ("got_ocr2", "GotOcr2Model"),
@@ -970,6 +972,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("gemma3n", "Gemma3nForConditionalGeneration"),
         ("git", "GitForCausalLM"),
         ("glm4v", "Glm4vForConditionalGeneration"),
+        ("glm4v_moe", "Glm4vMoeForConditionalGeneration"),
         ("got_ocr2", "GotOcr2ForConditionalGeneration"),
         ("idefics", "IdeficsForVisionText2Text"),
         ("idefics2", "Idefics2ForConditionalGeneration"),

src/transformers/models/auto/processing_auto.py

@@ -74,6 +74,7 @@
         ("gemma3n", "Gemma3nProcessor"),
         ("git", "GitProcessor"),
         ("glm4v", "Glm4vProcessor"),
+        ("glm4v_moe", "Glm4vProcessor"),
         ("got_ocr2", "GotOcr2Processor"),
         ("granite_speech", "GraniteSpeechProcessor"),
         ("grounding-dino", "GroundingDinoProcessor"),

src/transformers/models/auto/tokenization_auto.py

@@ -294,6 +294,7 @@
         ("glm4", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),
         ("glm4_moe", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),
         ("glm4v", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),
+        ("glm4v_moe", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),
         ("gpt-sw3", ("GPTSw3Tokenizer" if is_sentencepiece_available() else None, None)),
         ("gpt2", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
         ("gpt_bigcode", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),

src/transformers/models/glm4_moe/modeling_glm4_moe.py

@@ -135,6 +135,7 @@ def __init__(self, config: Glm4MoeConfig, layer_idx: Optional[int] = None):
         self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
         self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
         self.scaling = self.head_dim**-0.5
+        self.rope_scaling = config.rope_scaling
         self.attention_dropout = config.attention_dropout
         self.is_causal = True
 

src/transformers/models/glm4_moe/modular_glm4_moe.py

@@ -263,6 +263,7 @@ def __init__(self, config: Glm4MoeConfig, layer_idx: Optional[int] = None):
         self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
         self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
         self.scaling = self.head_dim**-0.5
+        self.rope_scaling = config.rope_scaling
         self.attention_dropout = config.attention_dropout
         self.is_causal = True
 

src/transformers/models/glm4v/configuration_glm4v.py

@@ -94,7 +94,7 @@ def __init__(
         patch_size=14,
         rms_norm_eps=1e-05,
         spatial_merge_size=2,
-        temporal_patch_size=1,
+        temporal_patch_size=2,
         out_hidden_size=4096,
         intermediate_size=13696,
         initializer_range=0.02,

src/transformers/models/glm4v/image_processing_glm4v_fast.py

@@ -22,8 +22,6 @@
 from ...image_processing_utils_fast import (
     BaseImageProcessorFast,
     DefaultFastImageProcessorKwargs,
-    group_images_by_shape,
-    reorder_images,
 )
 from ...image_utils import (
     OPENAI_CLIP_MEAN,
@@ -47,7 +45,6 @@
 if is_torch_available():
     import torch
 
-
 if is_torchvision_available():
     if is_torchvision_v2_available():
         from torchvision.transforms.v2 import functional as F
@@ -112,48 +109,44 @@ def _preprocess(
         Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
         """
 
-        # Group images by size for batched resizing
-        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
-        resized_images_grouped = {}
-        for shape, stacked_images in grouped_images.items():
-            height, width = stacked_images.shape[-2:]
+        processed_images = []
+        processed_grids = []
+
+        all_target_sizes = []
+        for image in images:
+            height, width = image.shape[-2:]
+            resized_height, resized_width = smart_resize(
+                num_frames=temporal_patch_size,
+                height=height,
+                width=width,
+                temporal_factor=temporal_patch_size,
+                factor=patch_size * merge_size,
+            )
+            all_target_sizes.append((resized_height, resized_width))
+
+        target_height = max([s[0] for s in all_target_sizes])
+        target_width = max([s[1] for s in all_target_sizes])
+
+        for image in images:
             if do_resize:
-                resized_height, resized_width = smart_resize(
-                    num_frames=temporal_patch_size,
-                    height=height,
-                    width=width,
-                    temporal_factor=temporal_patch_size,
-                    factor=patch_size * merge_size,
-                )
-                stacked_images = self.resize(
-                    stacked_images,
-                    size=SizeDict(height=resized_height, width=resized_width),
+                image = self.resize(
+                    image,
+                    size=SizeDict(height=target_height, width=target_width),
                     interpolation=interpolation,
                 )
-            resized_images_grouped[shape] = stacked_images
-        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
-        # Group images by size for further processing
-        # Needed in case do_resize is False, or resize returns images with different sizes
-        grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
-        processed_images_grouped = {}
-        processed_grids = {}
-        for shape, stacked_images in grouped_images.items():
-            resized_height, resized_width = stacked_images.shape[-2:]
-            # Fused rescale and normalize
-            stacked_images = self.rescale_and_normalize(
-                stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
-            )
-            # add a temporal dimension
-            patches = stacked_images.unsqueeze(1)
-            if patches.shape[1] % temporal_patch_size != 0:
-                repeats = patches[:, -1:].repeat(1, temporal_patch_size - 1, 1, 1, 1)
-                patches = torch.cat([patches, repeats], dim=1)
-            batch_size, grid_t, channel = patches.shape[:3]
-            grid_t = grid_t // temporal_patch_size
-            grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
 
+            image = self.rescale_and_normalize(
+                image.unsqueeze(0), do_rescale, rescale_factor, do_normalize, image_mean, image_std
+            ).squeeze(0)
+
+            patches = image.unsqueeze(0)
+            if patches.shape[0] % temporal_patch_size != 0:
+                repeats = patches[-1:].repeat(temporal_patch_size - (patches.shape[0] % temporal_patch_size), 1, 1, 1)
+                patches = torch.cat([patches, repeats], dim=0)
+            channel = patches.shape[1]
+            grid_t = patches.shape[0] // temporal_patch_size
+            grid_h, grid_w = target_height // patch_size, target_width // patch_size
             patches = patches.view(
-                batch_size,
                 grid_t,
                 temporal_patch_size,
                 channel,
@@ -164,18 +157,14 @@ def _preprocess(
                 merge_size,
                 patch_size,
             )
-            patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
+            patches = patches.permute(0, 3, 6, 4, 7, 2, 1, 5, 8)
             flatten_patches = patches.reshape(
-                batch_size,
                 grid_t * grid_h * grid_w,
                 channel * temporal_patch_size * patch_size * patch_size,
             )
+            processed_images.append(flatten_patches)
+            processed_grids.append([grid_t, grid_h, grid_w])
 
-            processed_images_grouped[shape] = flatten_patches
-            processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
-
-        processed_images = reorder_images(processed_images_grouped, grouped_images_index)
-        processed_grids = reorder_images(processed_grids, grouped_images_index)
         pixel_values = torch.stack(processed_images, dim=0)
         image_grid_thw = torch.tensor(processed_grids)