升级 vllm sglang版本

Author: shell-nlp

gpt_server/model_worker/base/model_worker_base.py

@@ -309,6 +309,8 @@ def run(cls):
         parser.add_argument("--limit_worker_concurrency", type=int, default=1024)
         # port
         parser.add_argument("--port", type=int, default=None)
+        # model_type
+        parser.add_argument("--model_type", type=str, default="auto")
         args = parser.parse_args()
         os.environ["num_gpus"] = str(args.num_gpus)
         if args.backend == "vllm":
@@ -331,6 +333,7 @@ def run(cls):
         if args.punc_model:
             os.environ["punc_model"] = args.punc_model
 
+        os.environ["model_type"] = args.model_type
         os.environ["enable_prefix_caching"] = args.enable_prefix_caching
         os.environ["gpu_memory_utilization"] = args.gpu_memory_utilization
         os.environ["kv_cache_quant_policy"] = args.kv_cache_quant_policy

gpt_server/serving/main.py

@@ -23,38 +23,11 @@
     start_api_server,
     start_model_worker,
     delete_log,
+    pre_processing,
 )
 
 
-def delete_flash_attn():
-    "删除 flash_attn，避免报错"
-    import shutil
-    import os
-    from pathlib import Path
-    from loguru import logger
-
-    root_path = Path(__file__).parent.parent.parent
-    flash_attn_path = root_path.joinpath(
-        ".venv/lib/python3.11/site-packages/flash_attn"
-    )
-
-    try:
-        # 检查路径是否存在
-        if os.path.exists(flash_attn_path):
-            # 删除整个目录树
-            shutil.rmtree(flash_attn_path)
-            logger.info(f"成功删除: {flash_attn_path}")
-
-    except PermissionError:
-        logger.error("权限不足，无法删除 flash_attn")
-    except Exception as e:
-        logger.error(f"删除 flash_attn 失败: {e}")
-
-
-# 删除日志
-delete_log()
-
-delete_flash_attn()
+pre_processing()
 
 config_path = os.path.join(root_dir, "gpt_server/script/config.yaml")
 env = os.getenv("ENV")

gpt_server/utils.py

@@ -18,6 +18,45 @@
 STATIC_DIR = root_dir / "static"
 
 
+def clear_flashinfer_cache():
+    os.system("flashinfer clear-cache")
+
+
+def delete_flash_attn():
+    "删除 flash_attn，避免报错"
+    import shutil
+    import os
+    from pathlib import Path
+    from loguru import logger
+
+    root_path = Path(__file__).parent.parent
+    flash_attn_path = root_path.joinpath(
+        ".venv/lib/python3.11/site-packages/flash_attn"
+    )
+
+    try:
+        # 检查路径是否存在
+        if os.path.exists(flash_attn_path):
+            # 删除整个目录树
+            shutil.rmtree(flash_attn_path)
+            logger.info(f"成功删除: {flash_attn_path}")
+
+    except PermissionError:
+        logger.error("权限不足，无法删除 flash_attn")
+    except Exception as e:
+        logger.error(f"删除 flash_attn 失败: {e}")
+
+
+def pre_processing():
+    "前置处理"
+    # 删除日志
+    delete_log()
+    # 删除 垃圾flash attn
+    delete_flash_attn()
+    # 清理 flashinfer 缓存
+    clear_flashinfer_cache()
+
+
 def kill_child_processes(parent_pid, including_parent=False):
     "杀死子进程/僵尸进程"
     try:
@@ -263,6 +302,7 @@ def start_model_worker(config: dict):
                         + f" --log_level {log_level}"  # 日志水平
                         + f" --task_type {task_type}"  # 日志水平
                         + f" --limit_worker_concurrency {limit_worker_concurrency}"  # 限制worker并发数
+                        + f" --model_type {model_type}"  # 默认类型
                     )
                     # 处理为 None的情况
                     if port:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gpt_server"
-version = "0.6.4"
+version = "0.6.5"
 description = "gpt_server是一个用于生产级部署LLMs、Embedding、Reranker、ASR和TTS的开源框架。"
 readme = "README.md"
 license = { text = "Apache 2.0" }
@@ -16,16 +16,16 @@ dependencies = [
     "loguru>=0.7.2",
     "openai==1.99.1",
     "setuptools==75.2.0",
-    "streamlit==1.39.0",
+    "streamlit>=1.50.0",
     "torch==2.8.0",
     "torchvision==0.23.0",
-    "vllm==0.10.2",
+    "vllm==0.11.0",
     "qwen_vl_utils",
     "evalscope[perf,rag]==0.16.1",
     "modelscope==1.26.0",
     "edge-tts>=7.0.0",
     "funasr>=1.2.6",
-    "sglang[all]>=0.5.2",
+    "sglang[all]>=0.5.3.post1",
     "flashinfer-python",
     "flashtts>=0.1.7",
     "diffusers>=0.35.1",
@@ -35,9 +35,10 @@ dependencies = [
 
 [tool.uv]
 default-groups = [] # 默认只安装dependencies中的库
+prerelease = "allow"
 override-dependencies = [
     "setuptools==77.0.3",
-    "transformers==4.56.1",  #  infinity-emb
+    "transformers==4.57.0",  #  infinity-emb
     "soundfile==0.13.1",     # infinity
     "xgrammar==0.1.24",      #  sglang[all]==0.4.5 depends on xgrammar==0.1.17
     "outlines-core==0.2.11", # sglang 和 vllm 的冲突