fix minicpm-v (#1562)

Author: Jintao-Huang

swift/llm/utils/model.py

@@ -5572,6 +5572,7 @@ def _new_forward(*args, **kwargs) -> Tensor:
     LoRATM.llama,
     TemplateType.minicpm_v,
     support_flash_attn=True,
+    requires=['timm', 'transformers<4.42'],
     tags=['multi-modal', 'vision'],
     hf_model_id='openbmb/MiniCPM-V')
 @register_model(
@@ -5580,44 +5581,53 @@ def _new_forward(*args, **kwargs) -> Tensor:
     LoRATM.llama,
     TemplateType.minicpm_v,
     support_flash_attn=True,
-    requires=['timm'],
+    requires=['timm', 'transformers<4.42'],
     tags=['multi-modal', 'vision'],
     hf_model_id='openbmb/MiniCPM-V-2')
-@register_model(
-    ModelType.minicpm_v_v2_5_chat,
-    'OpenBMB/MiniCPM-Llama3-V-2_5',
-    LoRATM.minicpm_llama,
-    TemplateType.minicpm_v_v2_5,
-    support_flash_attn=True,
-    support_lmdeploy=True,
-    requires=['timm'],
-    placeholder_tokens=['<unk>'],
-    function_kwargs={'patching_embedding': True},
-    tags=['multi-modal', 'vision'],
-    hf_model_id='openbmb/MiniCPM-Llama3-V-2_5')
 def get_model_tokenizer_minicpm_v(model_dir: str,
                                   torch_dtype: Dtype,
                                   model_kwargs: Dict[str, Any],
                                   load_model: bool = True,
                                   **kwargs):
-    patching_embedding = kwargs.pop('patching_embedding', False)
     model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, torch_dtype, model_kwargs, load_model, **kwargs)
     if load_model:
         model.resampler.to(torch_dtype)  # fix float32
         _patch_minicpm_v_device_map(model)
         func_list = ['generate', 'get_input_embeddings', 'forward']
         _use_submodel_func(model, 'llm', func_list)
-        if patching_embedding:
-            embedding = model.get_input_embeddings()
-            if not hasattr(embedding, '__old_forward'):  # Avoid double patching
-                old_forward = embedding.forward
+    return model, tokenizer
+
+
+@register_model(
+    ModelType.minicpm_v_v2_5_chat,
+    'OpenBMB/MiniCPM-Llama3-V-2_5',
+    LoRATM.minicpm_llama,
+    TemplateType.minicpm_v_v2_5,
+    support_flash_attn=True,
+    requires=['timm', 'transformers>=4.36'],
+    placeholder_tokens=['<unk>'],
+    tags=['multi-modal', 'vision'],
+    hf_model_id='openbmb/MiniCPM-Llama3-V-2_5')
+def get_model_tokenizer_minicpm_v_2_5(model_dir: str,
+                                      torch_dtype: Dtype,
+                                      model_kwargs: Dict[str, Any],
+                                      load_model: bool = True,
+                                      **kwargs):
+    from transformers import AutoProcessor
+    processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+    model, tokenizer = get_model_tokenizer_minicpm_v(model_dir, torch_dtype, model_kwargs, load_model, **kwargs)
+    tokenizer.processor = processor
+    if load_model:
+        embedding = model.get_input_embeddings()
+        if not hasattr(embedding, '__old_forward'):  # Avoid double patching
+            old_forward = embedding.forward
 
-                @wraps(old_forward)
-                def _new_forward(*args, **kwargs):
-                    return old_forward(*args, **kwargs).requires_grad_(True).clone()
+            @wraps(old_forward)
+            def _new_forward(*args, **kwargs):
+                return old_forward(*args, **kwargs).requires_grad_(True).clone()
 
-                embedding.__old_forward = old_forward
-                embedding.forward = _new_forward
+            embedding.__old_forward = old_forward
+            embedding.forward = _new_forward
     return model, tokenizer
 
 

swift/llm/utils/template.py

@@ -1878,10 +1878,10 @@ def _encode(self, example: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, An
         if len(inputs) == 0:
             return inputs, {}
         images = example.get('images', [])
-        image_processor = self.tokenizer.processor.image_processor
         if self._is_vllm:
             images = self._prepare_vllm_images(images)
         if images:
+            image_processor = self.tokenizer.processor.image_processor
             image_inputs = image_processor(images, return_tensors='pt').to(self.model.dtype)
             inputs['pixel_values'] = image_inputs['pixel_values']
             if 'image_sizes' in image_inputs:
@@ -2470,7 +2470,16 @@ def _encode(self, example: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, An
         tgt_sizes = None
         slice_mode = getattr(config, 'slice_mode', False)
         if slice_mode:
-            images, placeholder = self.model.get_slice_image_placeholder(image, self.tokenizer)
+            if self.is_v2_5:
+                from .utils import to_device
+                image_processor = self.tokenizer.processor.image_processor
+                image_inputs = image_processor(images, return_tensors='pt').to(self.model.dtype)
+                placeholder = image_processor.get_slice_image_placeholder(image_inputs.image_sizes[0][0])
+                pixel_values = to_device(image_inputs['pixel_values'], self.model.device)
+                tgt_sizes = image_inputs['tgt_sizes']
+            else:
+                images, placeholder = self.model.get_slice_image_placeholder(image, self.tokenizer)
+                pixel_values = [[self.model.transform(img).to(device=self.model.device) for img in images]]
             placeholder += '\n'
             placeholder_id = self.tokenizer.encode(placeholder, add_special_tokens=False)
             input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:])
@@ -2485,33 +2494,21 @@ def _encode(self, example: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, An
                 torch.hstack(
                     [image_start_idx[:valid_image_nums].unsqueeze(-1), image_end_idx[:valid_image_nums].unsqueeze(-1)])
             ]
-            if self.is_v2_5:
-                pixel_values = []
-                tgt_sizes = []
-                config = self.model.config
-                for image in images:
-                    image = self.model.transform(image).to(device=self.model.device)
-                    H, W = image.shape[1:]
-                    pixel_values.append(self.model.reshape_by_patch(image))
-                    tgt_sizes.append(torch.Tensor([H // config.patch_size, W // config.patch_size]).type(torch.int32))
-                tgt_sizes = torch.vstack(tgt_sizes)
-            else:
-                pixel_values = [self.model.transform(img).to(device=self.model.device) for img in images]
         else:
             placeholder = '<image>' + '<unk>' * config.query_num + '</image>\n'
             placeholder_id = self.tokenizer.encode(placeholder, add_special_tokens=False)
             input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:])
             if labels is not None:
                 labels = (labels[:idx] + [-100] * len(placeholder_id) + labels[idx + 1:])
             image_bound = [torch.tensor([[idx, idx + config.query_num]])]
-            pixel_values = [self.model.transform(image).to(device=self.model.device)]
+            pixel_values = [[self.model.transform(image).to(device=self.model.device)]]
         data = {
             'input_ids': torch.tensor(input_ids)[None].to(device=self.model.device),
             'image_bound': image_bound,
-            'pixel_values': [pixel_values]
+            'pixel_values': pixel_values
         }
-        if tgt_sizes is not None:
-            data['tgt_sizes'] = [tgt_sizes]
+        if tgt_sizes is not None:  # v2.5
+            data['tgt_sizes'] = tgt_sizes
         inputs_embeds, _ = self.model.get_vllm_embedding(data)
         inputs_embeds = inputs_embeds.detach()
         inputs['input_ids'] = input_ids