[model] support ovis2.5 (#5426)

Author: Jintao-Huang

docs/source/Instruction/命令行参数.md

@@ -722,6 +722,13 @@ qwen2_5_omni除了包含qwen2_5_vl和qwen2_audio的模型特定参数外，还
 ### ovis1_6, ovis2
 - MAX_PARTITION: 默认为9，参考[这里](https://github.com/AIDC-AI/Ovis/blob/d248e34d755a95d24315c40e2489750a869c5dbc/ovis/model/modeling_ovis.py#L312)。
 
+### ovis2_5
+以下参数含义可以在[这里](https://modelscope.cn/models/AIDC-AI/Ovis2.5-2B)的示例代码中找到。
+- MIX_PIXELS: int类型，默认为`448 * 448`。
+- MAX_PIXELS: int类型，默认为`1344 * 1792`。若出现OOM，可以调小该值。
+- VIDEO_MAX_PIXELS: int类型，默认为`896 * 896`。
+- NUM_FRAMES: 默认为8。用于视频抽帧。
+
 ### mplug_owl3, mplug_owl3_241101
 - MAX_NUM_FRAMES: 默认为16，参考[这里](https://modelscope.cn/models/iic/mPLUG-Owl3-7B-240728)。
 

docs/source/Instruction/支持的模型和数据集.md

@@ -691,6 +691,8 @@
 |[AIDC-AI/Ovis2-8B](https://modelscope.cn/models/AIDC-AI/Ovis2-8B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2-8B](https://huggingface.co/AIDC-AI/Ovis2-8B)|
 |[AIDC-AI/Ovis2-16B](https://modelscope.cn/models/AIDC-AI/Ovis2-16B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2-16B](https://huggingface.co/AIDC-AI/Ovis2-16B)|
 |[AIDC-AI/Ovis2-34B](https://modelscope.cn/models/AIDC-AI/Ovis2-34B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2-34B](https://huggingface.co/AIDC-AI/Ovis2-34B)|
+|[AIDC-AI/Ovis2.5-2B](https://modelscope.cn/models/AIDC-AI/Ovis2.5-2B)|ovis2_5|ovis2_5|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2.5-2B](https://huggingface.co/AIDC-AI/Ovis2.5-2B)|
+|[AIDC-AI/Ovis2.5-9B](https://modelscope.cn/models/AIDC-AI/Ovis2.5-9B)|ovis2_5|ovis2_5|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2.5-9B](https://huggingface.co/AIDC-AI/Ovis2.5-9B)|
 |[XiaomiMiMo/MiMo-VL-7B-SFT](https://modelscope.cn/models/XiaomiMiMo/MiMo-VL-7B-SFT)|mimo_vl|mimo_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|&#x2718;|vision, video|[XiaomiMiMo/MiMo-VL-7B-SFT](https://huggingface.co/XiaomiMiMo/MiMo-VL-7B-SFT)|
 |[XiaomiMiMo/MiMo-VL-7B-RL](https://modelscope.cn/models/XiaomiMiMo/MiMo-VL-7B-RL)|mimo_vl|mimo_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|&#x2718;|vision, video|[XiaomiMiMo/MiMo-VL-7B-RL](https://huggingface.co/XiaomiMiMo/MiMo-VL-7B-RL)|
 |[mispeech/midashenglm-7b](https://modelscope.cn/models/mispeech/midashenglm-7b)|midashenglm|midashenglm|transformers>=4.52, soundfile|&#x2718;|audio|[mispeech/midashenglm-7b](https://huggingface.co/mispeech/midashenglm-7b)|

docs/source_en/Instruction/Command-line-parameters.md

@@ -738,6 +738,15 @@ For the meaning of the arguments, please refer to [here](https://modelscope.cn/m
 ### ovis1_6, ovis2
 - MAX_PARTITION: Default is 9, refer to [here](https://github.com/AIDC-AI/Ovis/blob/d248e34d755a95d24315c40e2489750a869c5dbc/ovis/model/modeling_ovis.py#L312)
 
+### ovis2_5
+
+The meanings of the following parameters can be found in the example code [here](https://modelscope.cn/models/AIDC-AI/Ovis2.5-2B).
+
+- MIX_PIXELS: int type, default is `448 * 448`.
+- MAX_PIXELS: int type, default is `1344 * 1792`. If OOM (out of memory) occurs, you can reduce this value.
+- VIDEO_MAX_PIXELS: int type, default is `896 * 896`.
+- NUM_FRAMES: default is 8. Used for video frame sampling.
+
 ### mplug_owl3, mplug_owl3_241101
 - MAX_NUM_FRAMES: Default is 16, refer to [here](https://modelscope.cn/models/iic/mPLUG-Owl3-7B-240728)
 

docs/source_en/Instruction/Supported-models-and-datasets.md

@@ -691,6 +691,8 @@ The table below introduces the models integrated with ms-swift:
 |[AIDC-AI/Ovis2-8B](https://modelscope.cn/models/AIDC-AI/Ovis2-8B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2-8B](https://huggingface.co/AIDC-AI/Ovis2-8B)|
 |[AIDC-AI/Ovis2-16B](https://modelscope.cn/models/AIDC-AI/Ovis2-16B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2-16B](https://huggingface.co/AIDC-AI/Ovis2-16B)|
 |[AIDC-AI/Ovis2-34B](https://modelscope.cn/models/AIDC-AI/Ovis2-34B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2-34B](https://huggingface.co/AIDC-AI/Ovis2-34B)|
+|[AIDC-AI/Ovis2.5-2B](https://modelscope.cn/models/AIDC-AI/Ovis2.5-2B)|ovis2_5|ovis2_5|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2.5-2B](https://huggingface.co/AIDC-AI/Ovis2.5-2B)|
+|[AIDC-AI/Ovis2.5-9B](https://modelscope.cn/models/AIDC-AI/Ovis2.5-9B)|ovis2_5|ovis2_5|transformers>=4.46.2, moviepy<2|&#x2718;|vision|[AIDC-AI/Ovis2.5-9B](https://huggingface.co/AIDC-AI/Ovis2.5-9B)|
 |[XiaomiMiMo/MiMo-VL-7B-SFT](https://modelscope.cn/models/XiaomiMiMo/MiMo-VL-7B-SFT)|mimo_vl|mimo_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|&#x2718;|vision, video|[XiaomiMiMo/MiMo-VL-7B-SFT](https://huggingface.co/XiaomiMiMo/MiMo-VL-7B-SFT)|
 |[XiaomiMiMo/MiMo-VL-7B-RL](https://modelscope.cn/models/XiaomiMiMo/MiMo-VL-7B-RL)|mimo_vl|mimo_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|&#x2718;|vision, video|[XiaomiMiMo/MiMo-VL-7B-RL](https://huggingface.co/XiaomiMiMo/MiMo-VL-7B-RL)|
 |[mispeech/midashenglm-7b](https://modelscope.cn/models/mispeech/midashenglm-7b)|midashenglm|midashenglm|transformers>=4.52, soundfile|&#x2718;|audio|[mispeech/midashenglm-7b](https://huggingface.co/mispeech/midashenglm-7b)|

swift/llm/model/constant.py

@@ -157,6 +157,7 @@ class MLLMModelType:
     ovis1_6 = 'ovis1_6'
     ovis1_6_llama3 = 'ovis1_6_llama3'
     ovis2 = 'ovis2'
+    ovis2_5 = 'ovis2_5'
     mimo_vl = 'mimo_vl'
     midashenglm = 'midashenglm'
 

swift/llm/model/model/qwen.py

@@ -926,7 +926,7 @@ def update(self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx
         ],
         TemplateType.ovis1_6,
         get_model_tokenizer_ovis,
-        model_arch=ModelArch.ovis1_6,
+        model_arch=ModelArch.ovis,
         architectures=['Ovis'],
         tags=['vision'],
         requires=['transformers>=4.42'],
@@ -942,7 +942,7 @@ def update(self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx
         ],
         TemplateType.ovis1_6_llama3,
         get_model_tokenizer_ovis,
-        model_arch=ModelArch.ovis1_6,
+        model_arch=ModelArch.ovis,
         architectures=['Ovis'],
         tags=['vision'],
     ))
@@ -962,7 +962,40 @@ def update(self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx
         ],
         TemplateType.ovis2,
         get_model_tokenizer_ovis,
-        model_arch=ModelArch.ovis1_6,
+        model_arch=ModelArch.ovis,
+        architectures=['Ovis'],
+        tags=['vision'],
+        requires=['transformers>=4.46.2', 'moviepy<2'],
+    ))
+
+
+def get_model_tokenizer_ovis2_5(*args, **kwargs):
+    model, tokenizer = get_model_tokenizer_with_flash_attn(*args, **kwargs)
+    if model is not None:
+        model.visual_tokenizer.to(model.dtype)
+        model.vte.to(model.dtype)
+
+        func_list = ['generate', 'forward', 'get_input_embeddings']
+        use_submodel_func(model, 'llm', func_list)
+        embedding = model.get_input_embeddings()
+        patch_output_clone(embedding)
+        patch_get_input_embeddings(model.visual_tokenizer, 'vit.vision_model.embeddings.patch_embedding')
+
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.ovis2_5,
+        [
+            ModelGroup([
+                Model('AIDC-AI/Ovis2.5-2B', 'AIDC-AI/Ovis2.5-2B'),
+                Model('AIDC-AI/Ovis2.5-9B', 'AIDC-AI/Ovis2.5-9B'),
+            ]),
+        ],
+        TemplateType.ovis2_5,
+        get_model_tokenizer_ovis2_5,
+        model_arch=ModelArch.ovis,
         architectures=['Ovis'],
         tags=['vision'],
         requires=['transformers>=4.46.2', 'moviepy<2'],

swift/llm/model/model_arch.py

@@ -68,7 +68,7 @@ class MLLMModelArch:
     got_ocr2 = 'got_ocr2'
     dots_ocr = 'dots_ocr'
 
-    ovis1_6 = 'ovis1_6'
+    ovis = 'ovis'
     molmo = 'molmo'
     emu3_chat = 'emu3_chat'
     megrez_omni = 'megrez_omni'
@@ -593,11 +593,12 @@ def register_model_arch(model_arch: ModelKeys, *, exist_ok: bool = False) -> Non
         vision_tower='vision_model',
     ))
 
-register_model_arch(MultiModelKeys(
-    MLLMModelArch.ovis1_6,
-    language_model='llm',
-    vision_tower='visual_tokenizer',
-))
+register_model_arch(
+    MultiModelKeys(
+        MLLMModelArch.ovis,
+        language_model='llm',
+        vision_tower=['visual_tokenizer', 'vte'],
+    ))
 
 register_model_arch(
     MultiModelKeys(

swift/llm/template/constant.py

@@ -116,6 +116,7 @@ class MLLMTemplateType:
     ovis1_6 = 'ovis1_6'
     ovis1_6_llama3 = 'ovis1_6_llama3'
     ovis2 = 'ovis2'
+    ovis2_5 = 'ovis2_5'
     mimo_vl = 'mimo_vl'
     midashenglm = 'midashenglm'
 

swift/llm/template/template/qwen.py

@@ -8,6 +8,7 @@
 import torch.nn.functional as F
 import transformers
 from packaging import version
+from torch import nn
 
 from swift.llm import get_packed_seq_params, to_device, to_float_dtype
 from swift.utils import get_env_args, is_deepspeed_enabled
@@ -17,7 +18,7 @@
 from ..template_inputs import StdTemplateInputs
 from ..template_meta import TemplateMeta
 from ..utils import Context, Word, findall
-from ..vision_utils import load_audio, load_batch, load_video_ovis2
+from ..vision_utils import load_audio, load_batch, load_video_ovis2, load_video_ovis2_5
 from .llama import Llama3TemplateMeta
 from .utils import DEFAULT_SYSTEM, ChatmlTemplateMeta, ThinkingTemplate
 
@@ -736,6 +737,86 @@ def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int
 ))
 
 
+class Ovis2_5Template(ThinkingTemplate):
+    num_frames = 8
+    use_model = True
+    skip_prompt = False
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return [[-200], '\n']
+        elif media_type == 'video':
+            num_frames = get_env_args('num_frames', int, self.num_frames)
+            inputs.images = load_video_ovis2_5(inputs.videos[index], num_frames)
+            return [[-200], '\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        min_pixels = get_env_args('min_pixels', int, 448 * 448)
+        max_pixels = get_env_args('max_pixels', int, 1344 * 1792)
+        video_max_pixels = get_env_args('video_max_pixels', int, 896 * 896)
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        visual_tokenizer = self.model.visual_tokenizer
+        idx_list = findall(input_ids, [-200])
+        if inputs.videos:
+            assert len(inputs.videos) == 1, 'only support single video'
+            encoded['pixel_values'], encoded['grid_thws'] = visual_tokenizer.preprocess(
+                video=inputs.images, min_pixels=min_pixels, max_pixels=video_max_pixels)
+            num_video_tokens = encoded['grid_thws'].prod(dim=-1)
+            num_video_tokens //= visual_tokenizer.vit.config.hidden_stride**2
+            num_video_tokens //= visual_tokenizer.vit.config.temporal_patch_size
+
+            def _get_new_tokens(i):
+                token_len = num_video_tokens[i].item()
+                return [-303] + [-300] * token_len + [-304]
+
+            input_ids, encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                input_ids, encoded['labels'], encoded['loss_scale'], idx_list, _get_new_tokens)
+        elif images:
+            pixel_values, grid_thws = zip(
+                *(visual_tokenizer.preprocess(image=image, min_pixels=min_pixels, max_pixels=max_pixels)
+                  for image in images))
+            encoded['pixel_values'] = torch.cat(pixel_values, dim=0)
+            encoded['grid_thws'] = torch.cat(grid_thws, dim=0)
+
+            num_image_atoms = encoded['grid_thws'].prod(dim=-1)
+            num_image_atoms //= visual_tokenizer.vit.config.hidden_stride**2
+            num_image_atoms //= visual_tokenizer.vit.config.temporal_patch_size
+
+            def _get_new_tokens(i):
+                token_len = num_image_atoms[i].item()
+                return [-301] + [-300] * token_len + [-302]
+
+            input_ids, encoded['labels'], encoded['loss_scale'] = self._extend_tokens(
+                input_ids, encoded['labels'], encoded['loss_scale'], idx_list, _get_new_tokens)
+
+        encoded['input_ids'] = input_ids
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        inputs_embeds = model.merge_multimodal(
+            input_ids=inputs['input_ids'],
+            pixel_values=inputs.pop('pixel_values', None),
+            grid_thws=inputs.pop('grid_thws', None))
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        grid_thws = self.concat_tensor(batch, 'grid_thws', 0)
+        if grid_thws is not None:
+            res['grid_thws'] = grid_thws
+        return res
+
+
+register_template(QwenTemplateMeta(
+    MLLMTemplateType.ovis2_5,
+    template_cls=Ovis2_5Template,
+    default_system=None,
+))
+
+
 @dataclass
 class MarcoO1TemplateMeta(QwenTemplateMeta):
     default_system: Optional[str] = """

swift/llm/template/vision_utils.py

@@ -273,3 +273,12 @@ def load_video_ovis2(video_path, num_frames):
         frames = [clip.get_frame(index / clip.fps) for index in sampled_indices]
         frames = [Image.fromarray(frame, mode='RGB') for frame in frames]
     return frames
+
+
+def load_video_ovis2_5(video_path, num_frames):
+    from moviepy.editor import VideoFileClip
+    with VideoFileClip(video_path) as clip:
+        total_frames = int(clip.fps * clip.duration)
+        indices = [int(i * total_frames / num_frames) for i in range(num_frames)]
+        frames = [Image.fromarray(clip.get_frame(t)) for t in (idx / clip.fps for idx in indices)]
+    return frames