[bugfix] fix vllm qwen2_5_vl (#5473)

Author: Jintao-Huang

swift/llm/infer/infer_engine/vllm_engine.py

@@ -280,6 +280,9 @@ def _add_request(self,
                         mm_data = {key.rstrip('s'): media_data[0]}
             if mm_data:
                 llm_inputs['multi_modal_data'] = mm_data
+            mm_processor_kwargs = inputs.get('mm_processor_kwargs')
+            if mm_processor_kwargs:
+                llm_inputs['mm_processor_kwargs'] = mm_processor_kwargs
             if self.task_type == 'embedding':
                 from vllm.pooling_params import PoolingParams
                 if 'task' in inspect.signature(PoolingParams).parameters:

swift/llm/template/base.py

@@ -1181,7 +1181,10 @@ def _encode_truncated(self, inputs: StdTemplateInputs):
 
         if self.mode in {'vllm', 'lmdeploy', 'sglang'}:
             encoded = Template._encode(self, inputs)
-            for key in ['images', 'audios', 'videos']:
+            keys = ['images', 'audios', 'videos']
+            if self.mode == 'vllm':
+                keys.append('mm_processor_kwargs')
+            for key in keys:
                 value = getattr(inputs, key)
                 if value:
                     encoded[key] = value

swift/llm/template/template/kwai.py

@@ -44,7 +44,8 @@ def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int
             video, video_kwargs = fetch_video({'video': video}, return_video_sample_fps=True)
             if isinstance(video, torch.Tensor):
                 video = video.to(torch.uint8)
-            inputs.videos[index] = (video, video_kwargs)
+            inputs.videos[index] = video
+            inputs.mm_processor_kwargs.setdefault('fps', []).append(video_kwargs)
             return ['<|vision_start|><|video_pad|><|vision_end|>']
 
     def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
@@ -53,15 +54,12 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
         input_ids = encoded['input_ids']
         labels = encoded['labels']
         loss_scale = encoded.get('loss_scale', None)
-
-        images = inputs.images
-        videos = [video[0] for video in inputs.videos]
-        fps = [video[1] for video in inputs.videos]
         for media_type in ['images', 'videos']:
-            if locals()[media_type]:
+            mm_data = getattr(inputs, media_type)
+            if mm_data:
                 if media_type == 'images':
                     media_token = self.image_token_id
-                    media_inputs = processor.image_processor(images=images, return_tensors='pt', do_resize=False)
+                    media_inputs = processor.image_processor(images=mm_data, return_tensors='pt', do_resize=False)
                     media_grid_thw = media_inputs['image_grid_thw']
                 else:
                     kwargs = {}
@@ -70,9 +68,10 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
                     else:
                         processor_func = processor.image_processor
                         kwargs['images'] = None
-                    media_inputs = processor_func(videos=videos, return_tensors='pt', do_resize=False, **kwargs)
+                    media_inputs = processor_func(videos=mm_data, return_tensors='pt', do_resize=False, **kwargs)
                     media_grid_thw = media_inputs['video_grid_thw']
                     media_token = self.video_token_id
+                    fps = inputs.mm_processor_kwargs['fps']
                     media_inputs['second_per_grid_ts'] = [
                         processor.image_processor.temporal_patch_size / tmp for tmp in fps
                     ]

swift/llm/template/template/qwen.py

@@ -244,7 +244,9 @@ def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int
             video, video_kwargs = fetch_video({'video': video}, return_video_sample_fps=True)
             if isinstance(video, torch.Tensor):
                 video = video.to(torch.uint8)
-            inputs.videos[index] = (video, video_kwargs)
+            inputs.videos[index] = video
+            if self.version == 'v2_5':
+                inputs.mm_processor_kwargs.setdefault('fps', []).append(video_kwargs)
             return ['<|vision_start|><|video_pad|><|vision_end|>']
 
     def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
@@ -259,14 +261,12 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
         input_ids = encoded['input_ids']
         labels = encoded['labels']
         loss_scale = encoded.get('loss_scale', None)
-        images = inputs.images
-        videos = [video[0] for video in inputs.videos]
-        fps = [video[1] for video in inputs.videos]
         for media_type in ['images', 'videos']:
-            if locals()[media_type]:
+            mm_data = getattr(inputs, media_type)
+            if mm_data:
                 if media_type == 'images':
                     media_token = self.image_token_id
-                    media_inputs = processor.image_processor(images=images, return_tensors='pt', do_resize=False)
+                    media_inputs = processor.image_processor(images=mm_data, return_tensors='pt', do_resize=False)
                     media_grid_thw = media_inputs['image_grid_thw']
                 else:
                     kwargs = {}
@@ -275,10 +275,11 @@ def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
                     else:
                         processor_func = processor.image_processor
                         kwargs['images'] = None
-                    media_inputs = processor_func(videos=videos, return_tensors='pt', do_resize=False, **kwargs)
+                    media_inputs = processor_func(videos=mm_data, return_tensors='pt', do_resize=False, **kwargs)
                     media_grid_thw = media_inputs['video_grid_thw']
                     media_token = self.video_token_id
                     if self.version == 'v2_5':
+                        fps = inputs.mm_processor_kwargs['fps']
                         media_inputs['second_per_grid_ts'] = [
                             processor.image_processor.temporal_patch_size / tmp for tmp in fps
                         ]

swift/llm/template/template_inputs.py

@@ -112,6 +112,7 @@ class StdTemplateInputs:
     rejected_images: List[Union[str, Image.Image]] = field(default_factory=list)
 
     margin: Optional[float] = None  # for reward modeling
+    mm_processor_kwargs: Dict[str, Any] = field(default_factory=dict)
 
     def __post_init__(self):
         self.image_idx = 0