支持了 phi-4

Author: shell-nlp

README.md

@@ -42,6 +42,7 @@
 ## 更新信息
 
 ```plaintext
+2024-12-14 支持了 phi-4
 2024-12-7  支持了 /v1/rerank 接口
 2024-12-1  支持了 QWQ-32B-Preview
 2024-10-15 支持了 Qwen2-VL
@@ -263,7 +264,7 @@ streamlit run server_ui.py
 |        Llama-3        |llama| √ |  √  |         √         |        √        |
 |        Baichuan-2        |baichuan| √ |  √  |         √         |        √        |
 |        QWQ-32B-Preview |qwen| √ |  √  |         √         |        √        |
-
+|        Phi-4 |phi| √ |  √  |         ×         |        ×        |
 ### **VLM** (视觉大模型榜单 https://rank.opencompass.org.cn/leaderboard-multimodal)
 
 | Models / BackEnd |model_type| HF | vllm | LMDeploy TurboMind | LMDeploy PyTorch |

gpt_server/model_backend/lmdeploy_backend.py

@@ -107,7 +107,8 @@ async def stream_chat(self, params: Dict[str, Any]) -> AsyncGenerator:
             response_format=params["response_format"],
         )
         logger.info(f"request_id {int(request_id)}")
-        messages = prompt or messages  # TODO 可能影响推理性能
+        if params.get("tools", None):
+            messages = prompt or messages  # 解决lmdeploy 的提示模板不支持 tools
         results_generator = self.async_engine.generate(
             messages=messages, session_id=int(request_id), gen_config=gen_config
         )

gpt_server/model_worker/phi.py

@@ -0,0 +1,92 @@
+import json
+from typing import List
+from fastchat.constants import ErrorCode, SERVER_ERROR_MSG
+from loguru import logger
+import torch
+
+from gpt_server.model_worker.base.model_worker_base import ModelWorkerBase
+
+
+class PhiWorker(ModelWorkerBase):
+    def __init__(
+        self,
+        controller_addr: str,
+        worker_addr: str,
+        worker_id: str,
+        model_path: str,
+        model_names: List[str],
+        limit_worker_concurrency: int,
+        conv_template: str = None,  # type: ignore
+    ):
+        super().__init__(
+            controller_addr,
+            worker_addr,
+            worker_id,
+            model_path,
+            model_names,
+            limit_worker_concurrency,
+            conv_template,
+            model_type="AutoModelForCausalLM",
+        )
+        # from tokenizer_config.json
+        self.stop_words_ids = [
+            100257,  # eos
+            100265,  # eos
+        ]
+
+        self.stop = [
+            self.tokenizer.decode(skip_word) for skip_word in self.stop_words_ids
+        ]
+        logger.info(f"{model_names[0]} 停用词: {self.stop}")
+
+    async def generate_stream_gate(self, params):
+        self.call_ct += 1
+        logger.info(f"params {params}")
+        logger.info(f"worker_id: {self.worker_id}")
+        try:
+            messages = params["messages"]
+            if isinstance(messages, list):
+                task = "chat"
+            elif isinstance(messages, str):
+                task = "completion"
+            if task == "chat":
+                # 暂时保留，用于特殊情况的处理
+                text = self.tokenizer.apply_chat_template(
+                    conversation=messages,
+                    tokenize=False,
+                    add_generation_prompt=True,
+                )
+            elif task == "completion":
+                text = messages
+
+            input_ids = self.tokenizer([text], return_tensors="pt").input_ids
+            # ---------------添加额外的参数------------------------
+            params["messages"] = messages
+            params["prompt"] = text
+            params["stop"].extend(self.stop)
+            params["stop_words_ids"] = self.stop_words_ids
+            params["input_ids"] = input_ids
+            # ---------------添加额外的参数------------------------
+            async for ret in self.backend.stream_chat(params=params):
+                yield json.dumps(ret).encode() + b"\0"
+
+        except torch.cuda.OutOfMemoryError as e:
+            ret = {
+                "text": f"{SERVER_ERROR_MSG}\n\n({e})",
+                "error_code": ErrorCode.CUDA_OUT_OF_MEMORY,
+            }
+            yield json.dumps(ret).encode() + b"\0"
+        except (ValueError, RuntimeError) as e:
+            logger.info(e)
+            ret = {
+                "text": f"{SERVER_ERROR_MSG}\n\n({e})",
+                "error_code": ErrorCode.INTERNAL_ERROR,
+            }
+            yield json.dumps(ret).encode() + b"\0"
+
+    def get_embeddings(self, params):
+        return super().get_embeddings(params)
+
+
+if __name__ == "__main__":
+    PhiWorker.run()