Merge pull request #4 from Guo-Chenxu/minicpm_o_2_6

resolve comments

Author: tc-mb

src/transformers/models/minicpm_o_2_6/configuration_minicpm_o_2_6.py

@@ -16,12 +16,15 @@
 import os
 from typing import Union
 
-from ...configuration_utils import PretrainedConfig
+from ...configuration_utils import PretrainedConfig, layer_type_validation
+from ...modeling_rope_utils import rope_config_validation
 from transformers.models.siglip.configuration_siglip import SiglipVisionConfig
 from transformers import Qwen2Config, WhisperConfig
 from ...utils import logging
 
 logger = logging.get_logger(__name__)
+
+
 class MiniCPMVSliceConfig(PretrainedConfig):
     model_type = "minicpmv"
 
@@ -39,9 +42,8 @@ def __init__(
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
-        cls._set_token_in_kwargs(kwargs)
-
-        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+        config_dict, kwargs = cls.get_config_dict(
+            pretrained_model_name_or_path, **kwargs)
 
         if config_dict.get("model_type") == "minicpmv":
             config_dict = config_dict["slice_config"]
@@ -84,10 +86,6 @@ def __init__(
         attn_implementation: str = "sdpa",
         use_mlp: bool = True,
         aug_loss_weight: bool = True,
-        do_sample: bool = True,
-        top_p: float = 0.7,
-        top_k: int = 20,
-        repetition_penalty: float = 1.0,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -116,13 +114,9 @@ def __init__(
         self.attn_implementation = attn_implementation
         self.use_mlp = use_mlp
         self.aug_loss_weight = aug_loss_weight
-        self.do_sample = do_sample
-        self.top_p = top_p
-        self.top_k = top_k
-        self.repetition_penalty = repetition_penalty
 
 
-class MiniCPM_o_2_6Config(Qwen2Config):
+class MiniCPM_o_2_6Config(PretrainedConfig):
     model_type = "minicpmo"
     keys_to_ignore_at_inference = ["past_key_values"]
 
@@ -136,6 +130,21 @@ class MiniCPM_o_2_6Config(Qwen2Config):
         "patch_size": 14,
     }
 
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
     def __init__(
         self,
         use_cache=True,
@@ -155,6 +164,24 @@ def __init__(
         init_vision=True,
         init_audio=True,
         init_tts=True,
+        vocab_size=151936,
+        hidden_size=4096,
+        intermediate_size=22016,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        hidden_act="silu",
+        max_position_embeddings=32768,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        use_sliding_window=False,
+        sliding_window=4096,
+        max_window_layers=28,
+        layer_types=None,
+        attention_dropout=0.0,
         **kwargs,
     ):
         self.use_cache = use_cache
@@ -179,7 +206,8 @@ def __init__(
 
         # same as HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit add tgt_sizes
         if vision_config is None:
-            self.vision_config = SiglipVisionConfig(**self.default_vision_config)
+            self.vision_config = SiglipVisionConfig(
+                **self.default_vision_config)
             logger.info("vision_config is None, using default vision config")
         elif isinstance(vision_config, dict):
             self.vision_config = SiglipVisionConfig(**vision_config)
@@ -203,7 +231,47 @@ def __init__(
 
         self.patch_size = self.vision_config.patch_size
 
-        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.use_sliding_window = use_sliding_window
+        self.sliding_window = sliding_window if self.use_sliding_window else None
+        self.max_window_layers = max_window_layers
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_dropout = attention_dropout
+        # Validate the correctness of rotary position embeddings parameters
+        # BC: if there is a 'type' field, move it to 'rope_type'.
+        if self.rope_scaling is not None and "type" in self.rope_scaling:
+            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+        rope_config_validation(self)
+
+        self.layer_types = layer_types
+        if self.layer_types is None:
+            self.layer_types = [
+                "sliding_attention"
+                if self.sliding_window is not None and i >= self.max_window_layers
+                else "full_attention"
+                for i in range(self.num_hidden_layers)
+            ]
+        layer_type_validation(self.layer_types)
+
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
 
 
 __all__ = ["MiniCPM_o_2_6Config"]

src/transformers/models/minicpm_o_2_6/feature_extractor_minicpm_o_2_6.py

@@ -0,0 +1,161 @@
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from typing import List, Optional, Union
+
+from transformers import WhisperFeatureExtractor, AutoFeatureExtractor, AutoTokenizer
+import numpy as np
+import torch
+
+
+class MiniCPM_o_2_6FeatureExtractor(WhisperFeatureExtractor):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def __call__(
+        self,
+        tokenizer: None,
+        audios: Union[np.ndarray, List[np.ndarray], List[List[np.ndarray]]],
+        audio_parts: Optional[list] = None,
+        chunk_input: Optional[bool] = False,
+        sampling_rate: Optional[int] = None,
+        chunk_length: Optional[int] = 1,
+        **kwargs,
+    ):
+        if isinstance(audios, np.ndarray):
+            audios_list = [[audios]]
+        elif isinstance(audios[0], np.ndarray):
+            audios_list = [audios]
+        else:
+            audios_list = audios
+
+        if audio_parts is not None:
+            assert len(audio_parts) == len(audios_list)
+            for parts, audios in zip(audio_parts, audios_list):
+                assert len(parts) == len(audios)
+
+        audio_feature_lens_list = []
+        audio_ph_list = []
+
+        audio_features_all = []
+
+        # audio placeholder not dependent on audio_parts
+        for audios in audios_list:
+            if audios:
+                audio_ph_list.append([self.get_audio_placeholder(tokenizer,
+                    len(a), chunk_input, chunk_length) for a in audios])
+            else:
+                audio_ph_list.append([])
+
+        for idx, audios in enumerate(audios_list):
+            if audio_parts is not None:
+                # same audio part merge
+                audio_part = audio_parts[idx]
+                merge_audio = []
+                cur_audio = []
+                for aid, (part, audio) in enumerate(zip(audio_part, audios)):
+                    if aid == 0 or audio_part[aid] == audio_part[aid - 1]:
+                        cur_audio.append(audio)
+                    else:
+                        merge_audio.append(np.hstack(cur_audio))
+                        cur_audio = [audio]
+                if cur_audio:
+                    merge_audio.append(np.hstack(cur_audio))
+
+            else:
+                merge_audio = audios
+
+            audio_feature_lens = []
+
+            # If the audio exceeds 30 seconds, split it into chunks every 30 seconds.
+            final_merge_audio = []
+            max_audio_inp_len = 30 * sampling_rate
+            for audio in merge_audio:
+                if len(audio) <= max_audio_inp_len:
+                    final_merge_audio.append(audio)
+                else:
+                    for i in range(math.ceil(len(audio) / max_audio_inp_len)):
+                        final_merge_audio.append(
+                            audio[i * max_audio_inp_len: (i + 1) * max_audio_inp_len])
+
+            if audios:
+                audio_inputs = super().__call__(
+                    final_merge_audio,
+                    sampling_rate=sampling_rate,
+                    return_attention_mask=True,
+                    padding="max_length",
+                    return_tensors="pt",
+                    **kwargs,
+                )
+                audio_feature = audio_inputs["input_features"]
+                actual_lens = audio_inputs["attention_mask"].sum(dim=1)
+
+                for feat, lens in zip(audio_feature, actual_lens):
+                    audio_features_all.append(feat[:, :lens])
+                    audio_feature_lens.append(lens)
+
+                audio_feature_lens = torch.hstack(audio_feature_lens)
+                audio_feature_lens_list.append(audio_feature_lens)
+            else:
+                audio_feature_lens_list.append([])
+
+        if audio_features_all:
+            audio_features = [i.permute(1, 0) for i in audio_features_all]
+            audio_features = torch.nn.utils.rnn.pad_sequence(
+                audio_features, batch_first=True, padding_value=0.0
+            ).permute(0, 2, 1)
+        else:
+            audio_features = []
+
+        return audio_features, audio_feature_lens_list, audio_ph_list
+
+    def get_audio_placeholder(self, tokenizer, audio_lens, chunk_input, chunk_length):
+        pool_step = 2
+        feature_lens = math.ceil(
+            audio_lens / self.hop_length)
+
+        feature_lens = (feature_lens - 1) // 2 + 1
+        output_lens = (feature_lens - pool_step) // pool_step + 1
+
+        if chunk_input:
+            fbank_feat_in_chunk = int(chunk_length * 100)
+            cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
+            audio_embeds_in_chunk = (
+                cnn_feat_in_chunk - pool_step) // pool_step + 1
+            num_audio_chunks = (
+                output_lens + audio_embeds_in_chunk - 1) // audio_embeds_in_chunk
+
+            place_holders = ""
+            total_unk_len = 0
+            for _ in range(num_audio_chunks):
+                unk_len = min(audio_embeds_in_chunk,
+                              output_lens - total_unk_len)
+                place_holders += tokenizer.audio_start + \
+                    tokenizer.unk_token * unk_len + tokenizer.audio_end
+                total_unk_len += unk_len
+            audio_placeholder = place_holders
+        else:
+            audio_placeholder = tokenizer.audio_start + \
+                tokenizer.unk_token * output_lens + tokenizer.audio_end
+
+        return audio_placeholder
+
+
+AutoFeatureExtractor.register(
+    "MiniCPM_o_2_6FeatureExtractor", MiniCPM_o_2_6FeatureExtractor)
+
+__all__ = ["MiniCPM_o_2_6FeatureExtractor"]