fix lmdeploy bug

Author: shell-nlp

docker-compose.yml

@@ -25,4 +25,4 @@ services:
               count: all
               # count: 2  # 两种方式
               capabilities: [ gpu ]
-    command: uv run gpt_server/serving/main.py
+    command: python gpt_server/serving/main.py

gpt_server/model_backend/lmdeploy_backend.py

@@ -7,6 +7,7 @@
 )
 from typing import Any, Dict, AsyncGenerator
 from lmdeploy.archs import get_task
+from lmdeploy.serve.async_engine import get_names_from_model
 from loguru import logger
 from gpt_server.model_backend.base import ModelBackend
 
@@ -73,6 +74,10 @@ def __init__(self, model_path) -> None:
             backend=backend,
             backend_config=backend_config,
         )
+        model_type = get_names_from_model(model_path=model_path)[1]
+        self.messages_type_select = (
+            model_type[1] == "base"
+        )  # 如果为True 则使用 prompt:str 否则： messages：list
 
     async def stream_chat(self, params: Dict[str, Any]) -> AsyncGenerator:
         prompt = params.get("prompt", "")
@@ -112,6 +117,8 @@ async def stream_chat(self, params: Dict[str, Any]) -> AsyncGenerator:
         logger.info(f"request_id {int(request_id)}")
         if params.get("tools", None):
             messages = prompt or messages  # 解决lmdeploy 的提示模板不支持 tools
+        if self.messages_type_select:
+            messages = prompt or messages
         results_generator = self.async_engine.generate(
             messages=messages, session_id=int(request_id), gen_config=gen_config
         )

gpt_server/utils.py

@@ -316,19 +316,10 @@ def is_port_in_use(port):
 }
 
 
-def infer_model_type(model_path: str) -> str:
-    """自动推测模型类型，未使用此代码"""
-    from lmdeploy.model import best_match_model
-    from transformers import AutoConfig
-
-    match_model_type = best_match_model(model_path)
-    model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
-    config_model_type = model_config.get("model_type", False)
-    return model_type_mapping[match_model_type]
-
-
 if __name__ == "__main__":
-    # ckpt = "deepseek-ai/deepseek-moe-16b-base"  # internlm2
-    # model_type = best_match_model(ckpt)
-    # print(model_type)
-    pass
+    # /home/dev/model/KirillR/QwQ-32B-Preview-AWQ
+    from lmdeploy.serve.async_engine import get_names_from_model
+
+    ckpt = "/home/dev/model/KirillR/QwQ-32B-Preview-AWQ"  # internlm2
+    model_type = get_names_from_model(ckpt)
+    print(model_type[1] == "base")