fix vllm==0.4.3 (#1055)

Author: Jintao-Huang

swift/llm/deploy.py

@@ -11,6 +11,7 @@
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse, StreamingResponse
 from modelscope import GenerationConfig
+from packaging import version
 from peft import PeftModel
 
 from swift.utils import get_logger, get_main, seed_everything
@@ -162,7 +163,13 @@ async def inference_vllm_async(request: Union[ChatCompletionRequest, CompletionR
                 break
         assert lora_request is not None
         generate_kwargs['lora_request'] = lora_request
-    result_generator = llm_engine.generate(None, generation_config, request_id, input_ids, **generate_kwargs)
+
+    import vllm
+    if version.parse(vllm.__version__) >= version.parse('0.4.3'):
+        result_generator = llm_engine.generate({'prompt_token_ids': input_ids}, generation_config, request_id,
+                                               **generate_kwargs)
+    else:
+        result_generator = llm_engine.generate(None, generation_config, request_id, input_ids, **generate_kwargs)
 
     async def _generate_full():
         result = None

swift/llm/utils/preprocess.py

@@ -15,17 +15,19 @@ class SwiftPreprocessor:
     def __call__(self, dataset: HfDataset) -> HfDataset:
         if 'history' in dataset.features:
             old_history = dataset['history']
-
+            has_history = False
             history: List[History] = []
-            for old_h in tqdm(old_history):
-                if isinstance(old_h, list):
-                    break
-                h = None
-                if old_h is not None:
+            for h in tqdm(old_history):
+                if isinstance(h, str):
                     h = ast.literal_eval(old_h)
+                elif h is None:
+                    h = []
+                if len(h) > 0:
+                    has_history = True
                 history.append(h)
-            else:
-                dataset = dataset.remove_columns(['history']).add_column('history', history)
+            dataset = dataset.remove_columns(['history'])
+            if has_history:
+                dataset = dataset.add_column('history', history)
         return dataset
 
 

swift/llm/utils/vllm_utils.py

@@ -266,7 +266,10 @@ def inference_stream_vllm(llm_engine: LLMEngine,
             resp_list[i] = {'response': '', 'history': history}
             continue
         input_ids = inputs['input_ids']
-        llm_engine.add_request(str(i), None, generation_config, input_ids, **add_request_kwargs)
+        if version.parse(vllm.__version__) >= version.parse('0.4.3'):
+            llm_engine.add_request(str(i), {'prompt_token_ids': input_ids}, generation_config, **add_request_kwargs)
+        else:
+            llm_engine.add_request(str(i), None, generation_config, input_ids, **add_request_kwargs)
 
     print_idx_list = [[0] for _ in range(len(request_list))]
     prog_bar = tqdm(total=len(request_list), dynamic_ncols=True, disable=not use_tqdm)
@@ -353,7 +356,10 @@ def inference_vllm(llm_engine: LLMEngine,
             resp_list[i] = {'response': '', 'history': history}
             continue
         input_ids = inputs['input_ids']
-        llm_engine.add_request(str(i), None, generation_config, input_ids, **add_request_kwargs)
+        if version.parse(vllm.__version__) >= version.parse('0.4.3'):
+            llm_engine.add_request(str(i), {'prompt_token_ids': input_ids}, generation_config, **add_request_kwargs)
+        else:
+            llm_engine.add_request(str(i), None, generation_config, input_ids, **add_request_kwargs)
 
     if use_tqdm is True:
         assert verbose is False