support lmdeploy & app-ui (#1546)

Author: Jintao-Huang

requirements/framework.txt

@@ -1,6 +1,7 @@
 accelerate
 addict
 aiohttp
+attrdict
 binpacking
 dacite
 datasets<2.19

requirements/llm.txt

@@ -1,4 +1,3 @@
-attrdict
 charset_normalizer
 cpm_kernels
 fastapi

swift/llm/app_ui.py

@@ -15,14 +15,17 @@ def clear_session() -> History:
 def gradio_generation_demo(args: AppUIArguments) -> None:
     import gradio as gr
     if args.infer_backend == 'vllm':
-        from swift.llm import prepare_vllm_engine_template, inference_stream_vllm
+        from swift.llm import prepare_vllm_engine_template, inference_stream_vllm as inference_stream_x
         llm_engine, template = prepare_vllm_engine_template(args)
+    elif args.infer_backend == 'lmdeploy':
+        from swift.llm import prepare_lmdeploy_engine_template, inference_stream_lmdeploy as inference_stream_x
+        llm_engine, template = prepare_lmdeploy_engine_template(args)
     else:
         model, template = prepare_model_template(args)
 
     def model_generation(query: str) -> Iterator[str]:
-        if args.infer_backend == 'vllm':
-            gen = inference_stream_vllm(llm_engine, template, [{'query': query}])
+        if args.infer_backend in {'vllm', 'lmdeploy'}:
+            gen = inference_stream_x(llm_engine, template, [{'query': query}])
             for resp_list in gen:
                 response = resp_list[0]['response']
                 yield response
@@ -52,15 +55,18 @@ def model_generation(query: str) -> Iterator[str]:
 def gradio_chat_demo(args: AppUIArguments) -> None:
     import gradio as gr
     if args.infer_backend == 'vllm':
-        from swift.llm import prepare_vllm_engine_template, inference_stream_vllm
+        from swift.llm import prepare_vllm_engine_template, inference_stream_vllm as inference_stream_x
         llm_engine, template = prepare_vllm_engine_template(args)
+    elif args.infer_backend == 'lmdeploy':
+        from swift.llm import prepare_lmdeploy_engine_template, inference_stream_lmdeploy as inference_stream_x
+        llm_engine, template = prepare_lmdeploy_engine_template(args)
     else:
         model, template = prepare_model_template(args)
 
     def model_chat(query: str, history: History) -> Iterator[Tuple[str, History]]:
         old_history, history = limit_history_length(template, query, history, args.max_length)
-        if args.infer_backend == 'vllm':
-            gen = inference_stream_vllm(llm_engine, template, [{'query': query, 'history': history}])
+        if args.infer_backend in {'vllm', 'lmdeploy'}:
+            gen = inference_stream_x(llm_engine, template, [{'query': query, 'history': history}])
             for resp_list in gen:
                 history = resp_list[0]['history']
                 total_history = old_history + history

swift/llm/deploy.py

@@ -4,7 +4,6 @@
 import logging
 import time
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import nullcontext
 from dataclasses import asdict
 from http import HTTPStatus
 from typing import List, Optional, Union
@@ -101,17 +100,10 @@ async def _prepare_request(request: Union[ChatCompletionRequest, CompletionReque
         if not is_valid:
             return create_error_response(HTTPStatus.BAD_REQUEST, 'API key error')
 
-    if _args.infer_backend == 'vllm':
-        from .utils import vllm_context
-        model_or_engine = llm_engine
-        context = vllm_context(template)
-    elif _args.infer_backend == 'lmdeploy':
-        from .utils import lmdeploy_context
+    if _args.infer_backend in {'vllm', 'lmdeploy'}:
         model_or_engine = llm_engine
-        context = lmdeploy_context(template)
     else:
         model_or_engine = model
-        context = nullcontext(template)
 
     error_msg = await check_model(request)
     if error_msg is not None:
@@ -147,10 +139,9 @@ async def _prepare_request(request: Union[ChatCompletionRequest, CompletionReque
                 example['tools'] = [tool]
             elif request.tool_choice == 'auto':
                 example['tools'] = request.tools
-        with context:
-            executor = ThreadPoolExecutor(max_workers=1)
-            loop = asyncio.get_running_loop()
-            inputs = (await loop.run_in_executor(executor, template.encode, example))[0]
+        executor = ThreadPoolExecutor(max_workers=1)
+        loop = asyncio.get_running_loop()
+        inputs = (await loop.run_in_executor(executor, template.encode, example))[0]
         request_id = f'chatcmpl-{random_uuid()}'
         _request['messages'] = messages
     else:
@@ -167,10 +158,9 @@ async def _prepare_request(request: Union[ChatCompletionRequest, CompletionReque
         example = {'query': prompt}
         if len(images) > 0:
             example['images'] = images
-        with context:
-            executor = ThreadPoolExecutor(max_workers=1)
-            loop = asyncio.get_running_loop()
-            inputs = (await loop.run_in_executor(executor, template.encode, example))[0]
+        executor = ThreadPoolExecutor(max_workers=1)
+        loop = asyncio.get_running_loop()
+        inputs = (await loop.run_in_executor(executor, template.encode, example))[0]
         request_id = f'cmpl-{random_uuid()}'
         _request['prompt'] = prompt
 
@@ -709,9 +699,11 @@ def llm_deploy(args: DeployArguments) -> None:
     if args.infer_backend == 'vllm':
         from .utils import prepare_vllm_engine_template
         llm_engine, template = prepare_vllm_engine_template(args, use_async=True)
+        template._is_vllm = True
     elif args.infer_backend == 'lmdeploy':
         from .utils import prepare_lmdeploy_engine_template
         llm_engine, template = prepare_lmdeploy_engine_template(args)
+        template._is_lmdeploy = True
     else:
         model, template = prepare_model_template(args)
     uvicorn.run(app, host=args.host, port=args.port, ssl_keyfile=args.ssl_keyfile, ssl_certfile=args.ssl_certfile)

swift/llm/utils/lmdeploy_utils.py

@@ -204,6 +204,8 @@ def inference_stream_lmdeploy(lmdeploy_engine: Union[AsyncEngine, VLAsyncEngine]
                               generation_info: Optional[Dict[str, Any]] = None,
                               use_tqdm: bool = False,
                               **kwargs) -> List[Dict[str, Any]]:
+    if len(request_list) == 0:
+        return []
     start_runtime = time.perf_counter()
     if generation_config is None:
         generation_config = getattr(lmdeploy_engine, 'generation_config', LmdeployGenerationConfig())
@@ -292,6 +294,8 @@ def inference_lmdeploy(lmdeploy_engine: Union[AsyncEngine, VLAsyncEngine],
                        prompt_prefix: str = '[PROMPT]',
                        output_prefix: str = '[OUTPUT]',
                        **kwargs) -> List[Dict[str, Any]]:
+    if len(request_list) == 0:
+        return []
     runtime = time.perf_counter()
     if generation_config is None:
         generation_config = getattr(lmdeploy_engine, 'generation_config', LmdeployGenerationConfig())

swift/llm/utils/vllm_utils.py

@@ -376,6 +376,8 @@ def inference_stream_vllm(
     return: e.g. [{'response': 'hi!', 'history': [('hello!', 'hi!')]}].
         The keys to be included will be: 'response', 'history'.
     """
+    if len(request_list) == 0:
+        return []
     start_runtime = time.perf_counter()
     if generation_config is None:
         generation_config = getattr(llm_engine, 'generation_config', VllmGenerationConfig())
@@ -468,6 +470,8 @@ def inference_vllm(llm_engine: LLMEngine,
     return: e.g. [{'response': 'hi!', 'history': [('hello!', 'hi!')]}].
         The keys to be included will be: 'response', 'history'.
     """
+    if len(request_list) == 0:
+        return []
     runtime = time.perf_counter()
     if generation_config is None:
         generation_config = getattr(llm_engine, 'generation_config', VllmGenerationConfig())