升级版本

Author: shell-nlp

.dockerignore

@@ -6,4 +6,5 @@ __pycache__/
 *.egg-info
 logs/
 outputs/
-data/
+data/
+.env

gpt_server/model_backend/vllm_backend.py

@@ -87,7 +87,7 @@ async def stream_chat(self, params: Dict[str, Any]) -> AsyncGenerator:
         if multimodal:  # 多模态模型
             # ----------------------------------------------------------------
             model_config = await self.engine.get_model_config()
-            conversation, mm_data_future = parse_chat_messages_futures(
+            conversation, mm_data_future, _ = parse_chat_messages_futures(
                 messages, model_config, tokenizer, content_format="string"
             )
 

gpt_server/serving/main.py

@@ -25,9 +25,37 @@
     delete_log,
 )
 
+
+def delete_flash_attn():
+    "删除 flash_attn，避免报错"
+    import shutil
+    import os
+    from pathlib import Path
+    from loguru import logger
+
+    root_path = Path(__file__).parent.parent.parent
+    flash_attn_path = root_path.joinpath(
+        ".venv/lib/python3.11/site-packages/flash_attn"
+    )
+
+    try:
+        # 检查路径是否存在
+        if os.path.exists(flash_attn_path):
+            # 删除整个目录树
+            shutil.rmtree(flash_attn_path)
+            logger.info(f"成功删除: {flash_attn_path}")
+
+    except PermissionError:
+        logger.error("权限不足，无法删除 flash_attn")
+    except Exception as e:
+        logger.error(f"删除 flash_attn 失败: {e}")
+
+
 # 删除日志
 delete_log()
 
+delete_flash_attn()
+
 config_path = os.path.join(root_dir, "gpt_server/script/config.yaml")
 env = os.getenv("ENV")
 if env == "test":

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gpt_server"
-version = "0.6.3"
+version = "0.6.4"
 description = "gpt_server是一个用于生产级部署LLMs、Embedding、Reranker、ASR和TTS的开源框架。"
 readme = "README.md"
 license = { text = "Apache 2.0" }
@@ -11,21 +11,21 @@ dependencies = [
     "fastapi==0.115.0",
     "ffmpy",
     "fschat==0.2.36",
-    "infinity-emb[all]==0.0.76",
+    "infinity-emb[all]==0.0.77",
     "lmdeploy==0.10.0",
     "loguru>=0.7.2",
     "openai==1.99.1",
     "setuptools==75.2.0",
     "streamlit==1.39.0",
-    "torch==2.6.0",
-    "torchvision==0.20.1",
-    "vllm",
+    "torch==2.8.0",
+    "torchvision==0.23.0",
+    "vllm==0.10.2",
     "qwen_vl_utils",
     "evalscope[perf,rag]==0.16.1",
     "modelscope==1.26.0",
     "edge-tts>=7.0.0",
     "funasr>=1.2.6",
-    "sglang[all]>=0.5.1.post3",
+    "sglang[all]>=0.5.2",
     "flashinfer-python",
     "flashtts>=0.1.7",
     "diffusers>=0.35.1",
@@ -37,31 +37,26 @@ dependencies = [
 default-groups = [] # 默认只安装dependencies中的库
 override-dependencies = [
     "setuptools==77.0.3",
-    "torchvision==0.23.0",
-    "torchaudio==2.8.0",
-    "torch==2.8.0",
-    "triton==3.4.0",
-    "transformers==4.56.1",      #  infinity-emb
-    "soundfile==0.13.1",         # infinity
-    "xgrammar==0.1.23",          #  sglang[all]==0.4.5 depends on xgrammar==0.1.17
-    "flashinfer-python==0.2.10",
-    "outlines-core==0.2.10",     # sglang 和 vllm 的冲突
-    "peft>=0.17.0",              # 和 lmdeloy 冲突
+    "transformers==4.56.1",  #  infinity-emb
+    "soundfile==0.13.1",     # infinity
+    "xgrammar==0.1.24",      #  sglang[all]==0.4.5 depends on xgrammar==0.1.17
+    "outlines-core==0.2.11", # sglang 和 vllm 的冲突
+    "peft>=0.17.0",          # 和 lmdeloy 冲突
 ]
 
 [project.scripts]
 gpt_server = "gpt_server.cli:main"
 
-[tool.uv.sources]
-vllm = { index = "vllm-custom" }
+# [tool.uv.sources]
+# vllm = { index = "vllm-custom" }
 
 [[tool.uv.index]]
 url = "https://pypi.tuna.tsinghua.edu.cn/simple"
 default = true
 
-[[tool.uv.index]]
-name = "vllm-custom"
-url = "https://wheels.vllm.ai/006e7a34aeb3e905ca4131a3251fe079f0511e2f"
+# [[tool.uv.index]]
+# name = "vllm-custom"
+# url = "https://wheels.vllm.ai/006e7a34aeb3e905ca4131a3251fe079f0511e2f"
 
 [build-system]
 requires = ["setuptools", "wheel"]