[Misc] add mm_processor_kwargs to extra_body for Qwen2.5-VL (#13533)

Author: wulipc

vllm/entrypoints/openai/protocol.py

@@ -312,6 +312,10 @@ class ChatCompletionRequest(OpenAIBaseModel):
         description=("Additional kwargs to pass to the template renderer. "
                      "Will be accessible by the chat template."),
     )
+    mm_processor_kwargs: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description=("Additional kwargs to pass to the HF processor."),
+    )
     guided_json: Optional[Union[str, dict, BaseModel]] = Field(
         default=None,
         description=("If specified, the output will follow the JSON schema."),

vllm/entrypoints/openai/serving_engine.py

@@ -451,6 +451,8 @@ async def _preprocess_chat(
             prompt_token_ids=prompt_inputs["prompt_token_ids"])
         if mm_data is not None:
             engine_prompt["multi_modal_data"] = mm_data
+        if request.mm_processor_kwargs is not None:
+            engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
 
         return conversation, [request_prompt], [engine_prompt]
 

vllm/model_executor/models/qwen2_5_vl.py

@@ -689,7 +689,7 @@ def get_hf_processor(
         min_pixels: Optional[int] = None,
         max_pixels: Optional[int] = None,
         size: Optional[dict[str, int]] = None,
-        fps: Optional[float] = None,
+        fps: Optional[Union[float, List[float]]] = None,
         **kwargs: object,
     ) -> Qwen2_5_VLProcessor:
         if fps is not None:

vllm/transformers_utils/processor.py

@@ -23,6 +23,15 @@ def __hash__(self) -> int:  # type: ignore[override]
         return hash(frozenset(self.items()))
 
 
+class HashableList(list):
+    """
+    A list that can be hashed by lru_cache.
+    """
+
+    def __hash__(self) -> int:  # type: ignore[override]
+        return hash(tuple(self))
+
+
 def _merge_mm_kwargs(model_config: "ModelConfig", **kwargs):
     base_kwargs = model_config.mm_processor_kwargs
     if base_kwargs is None:
@@ -36,7 +45,8 @@ def _merge_mm_kwargs(model_config: "ModelConfig", **kwargs):
     for key, value in merged_kwargs.items():
         if isinstance(value, dict):
             merged_kwargs[key] = HashableDict(value)
-
+        if isinstance(value, list):
+            merged_kwargs[key] = HashableList(value)
     return merged_kwargs