[bugfix] fix ovis2_5 (#5803)

Author: Jintao-Huang

swift/llm/template/template/qwen.py

@@ -816,11 +816,11 @@ def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, An
             visual_embeds = model.vte(visual_tokens).to(dtype=inputs_embeds.dtype, device=inputs_embeds.device)
             inputs_embeds[input_ids == VISUAL_ATOM_ID] = visual_embeds
         elif is_deepspeed_enabled():
-            media_inputs = model.visual_tokenizer.preprocess(
+            pixel_values, grid_thws = model.visual_tokenizer.preprocess(
                 Image.new('RGB', (32, 32), (0, 0, 0)), min_pixels=self.min_pixels, max_pixels=self.max_pixels)
-            media_inputs = to_device(media_inputs, input_ids.device)
-            pixel_values = media_inputs['pixel_values'].type(inputs_embeds.dtype)
-            visual_tokens = model.visual_tokenizer(pixel_values, media_inputs['grid_thws'])
+            pixel_values = pixel_values.to(device=inputs_embeds.device)
+            grid_thws = grid_thws.to(device=inputs_embeds.device)
+            visual_tokens = model.visual_tokenizer(pixel_values, grid_thws)
             visual_embeds = model.vte(visual_tokens).to(dtype=inputs_embeds.dtype, device=inputs_embeds.device)
             inputs_embeds = inputs_embeds + visual_embeds.mean() * 0.
 

swift/megatron/model/mm_gpt/qwen.py

@@ -207,11 +207,11 @@ def get_inputs_embeds(self, inputs_embeds, **kwargs):
         for i, indicator_id in enumerate(INDICATOR_IDS):
             inputs_embeds[input_ids == indicator_id] = visual_indicator_embeds[i]
         if pixel_values is None:
-            media_inputs = self.visual_tokenizer.preprocess(
+            pixel_values, grid_thws = self.visual_tokenizer.preprocess(
                 Image.new('RGB', (32, 32), (0, 0, 0)), min_pixels=self.min_pixels, max_pixels=self.max_pixels)
-            media_inputs = to_device(media_inputs, input_ids.device)
-            pixel_values = media_inputs['pixel_values'].type(inputs_embeds.dtype)
-            visual_tokens = self.visual_tokenizer(pixel_values, media_inputs['grid_thws'])
+            pixel_values = pixel_values.to(device=inputs_embeds.device)
+            grid_thws = grid_thws.to(device=inputs_embeds.device)
+            visual_tokens = self.visual_tokenizer(pixel_values, grid_thws)
             visual_embeds = self.vte(visual_tokens).to(dtype=inputs_embeds.dtype, device=inputs_embeds.device)
             inputs_embeds += visual_embeds.mean() * 0.
         else: