Merge branch 'main' into release/3.3

Author: Jintao-Huang

swift/llm/infer/deploy.py

@@ -3,7 +3,7 @@
 import inspect
 import multiprocessing
 import time
-from contextlib import contextmanager, nullcontext
+from contextlib import contextmanager
 from dataclasses import asdict
 from http import HTTPStatus
 from threading import Thread

swift/llm/infer/infer.py

@@ -1,9 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from contextlib import nullcontext
 from typing import Any, Dict, List, Union
 
 import numpy as np
-import torch.distributed as dist
 from datasets import Dataset as HfDataset
 
 from swift.llm import InferArguments, InferRequest, SwiftPipeline, load_dataset, prepare_model_template, sample_dataset

swift/llm/infer/infer_engine/vllm_engine.py

@@ -187,8 +187,7 @@ def _prepare_engine_kwargs(
     def _fix_vllm_bug(self) -> None:
         # fix vllm==0.4 bug (very slow)
         tokenizer = self.tokenizer
-        if version.parse(
-                vllm.__version__) >= version.parse('0.4') and not tokenizer.__class__.__name__.startswith('Cached'):
+        if self._version_ge('0.4') and not tokenizer.__class__.__name__.startswith('Cached'):
             _tokenizer_len = len(tokenizer)
             __old_len__ = tokenizer.__class__.__len__
 
@@ -224,6 +223,13 @@ def _add_stop_words(self, generation_config: SamplingParams, request_config: Req
         stop_words = (request_config.stop or []) + (self.generation_config.stop or []) + template_meta.stop_words
         generation_config.stop = self._get_stop_words(stop_words)
 
+    @staticmethod
+    def _version_ge(base_version: str):
+        vllm_version = vllm.__version__
+        if vllm_version is None or 'dev' in vllm_version:
+            return True
+        return version.parse(vllm_version) >= version.parse(base_version)
+
     def _add_request(self,
                      inputs: Dict[str, Any],
                      generation_config: SamplingParams,
@@ -241,18 +247,18 @@ def _add_request(self,
                     lora_name=adapter_name, lora_path=adapter_path, lora_int_id=len(self._adapters_pool) + 1)
                 self._adapters_pool[adapter_name] = kwargs['lora_request']
         input_ids = inputs['input_ids']
-        if version.parse(vllm.__version__) >= version.parse('0.4.3'):
+        if self._version_ge('0.4.3'):
             llm_inputs = {'prompt_token_ids': input_ids}
             mm_data = {}
             for key in ['images', 'audios', 'videos']:
                 media_data = inputs.get(key) or []
                 if media_data:
-                    if version.parse(vllm.__version__) < version.parse('0.6'):
+                    if self._version_ge('0.6'):
+                        mm_data = {key.rstrip('s'): media_data[0] if len(media_data) == 1 else media_data}
+                    else:
                         assert len(media_data) == 1, (
                             f'The current version of vllm only supports single {key}. Please upgrade to vllm >= 0.6.0')
                         mm_data = {key.rstrip('s'): media_data[0]}
-                    else:
-                        mm_data = {key.rstrip('s'): media_data[0] if len(media_data) == 1 else media_data}
             if mm_data:
                 llm_inputs['multi_modal_data'] = mm_data
             if self.use_async_engine:

tests/test_align/test_vllm_vlm.py

@@ -31,8 +31,9 @@ def _infer_image(model, use_chat_template: bool = True, max_model_len=8192, syst
     return resp_list[0].choices[0].message.content
 
 
-def _infer_video(model, use_chat_template: bool = True, max_model_len=8192, system=None):
-    engine = VllmEngine(model, max_model_len=max_model_len, limit_mm_per_prompt={'image': 16, 'video': 2})
+def _infer_video(model, use_chat_template: bool = True, max_model_len=8192, system=None, limit_mm_per_prompt=None):
+    limit_mm_per_prompt = limit_mm_per_prompt or {'image': 16, 'video': 2}
+    engine = VllmEngine(model, max_model_len=max_model_len, limit_mm_per_prompt=limit_mm_per_prompt)
     if not use_chat_template:
         engine.default_template.use_chat_template = False
     videos = ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']
@@ -116,6 +117,12 @@ def test_qwen2_5_vl_video():
                         'on a white blanket. The baby is looking at the book and is smiling. The baby')
 
 
+def test_qwen2_5_omni():
+    limit_mm_per_prompt = {'image': 1, 'video': 1, 'audio': 1}
+    response = _infer_video('Qwen/Qwen2.5-Omni-7B', limit_mm_per_prompt=limit_mm_per_prompt)
+    assert response
+
+
 if __name__ == '__main__':
     from swift.llm import VllmEngine, InferRequest, RequestConfig
     # test_qwen2_vl()
@@ -125,5 +132,6 @@ def test_qwen2_5_vl_video():
     # test_qwen2_audio()
     # test_minicpmv_2_5()
     # test_minicpmv_2_6()
-    test_minicpmo_2_6_video()
+    # test_minicpmo_2_6_video()
     # test_qwen2_5_vl_video()
+    test_qwen2_5_omni()