From b2ba68b3d9abd555d032eea3fe203472caa2c386 Mon Sep 17 00:00:00 2001 From: lao Date: Mon, 14 Apr 2025 17:32:11 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E5=AF=B9Xinferen?= =?UTF-8?q?ce=E6=A8=A1=E5=9E=8B=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在.env.example中添加了XINFERENCE_OPENAI_ENDPOINT和XINFERENCE_MODEL环境变量，并在src/utils/utils.py中实现了对Xinference模型的支持。新增的模型名称列表包括qwen2.5和deepseek系列模型。 --- .env.example | 3 +++ src/utils/utils.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.env.example b/.env.example index d99f3580..d123dbef 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,9 @@ UNBOUND_API_KEY= SiliconFLOW_ENDPOINT=https://api.siliconflow.cn/v1/ SiliconFLOW_API_KEY= +XINFERENCE_OPENAI_ENDPOINT=https://api.xinference.com/v1 +XINFERENCE_MODEL= + # Set to false to disable anonymized telemetry ANONYMIZED_TELEMETRY=false diff --git a/src/utils/utils.py b/src/utils/utils.py index 62fc8a8e..3f5806e3 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -183,6 +183,18 @@ def get_llm_model(provider: str, **kwargs): model_name=kwargs.get("model_name", "Qwen/QwQ-32B"), temperature=kwargs.get("temperature", 0.0), ) + elif provider == "xinference": + if not kwargs.get("base_url", ""): + base_url = os.getenv("XINFERENCE_OPENAI_ENDPOINT", "https://api.xinference.com/v1") + else: + base_url = kwargs.get("base_url") + + return ChatOpenAI( + model=kwargs.get("model_name", "gpt-4o"), + temperature=kwargs.get("temperature", 0.0), + base_url=base_url, + api_key=api_key, + ) else: raise ValueError(f"Unsupported provider: {provider}") @@ -234,6 +246,10 @@ def get_llm_model(provider: str, **kwargs): "Pro/THUDM/chatglm3-6b", "Pro/THUDM/glm-4-9b-chat", ], + "xinference": ["qwen2.5-instruct", "qwen2.5", "qwen2.5-coder", "qwen2.5-coder-instruct", "qwen2.5-instruct-1m", + "qwen2.5-vl-instruct", "deepseek", "deepseek-chat", "deepseek-coder", "deepseek-coder-instruct", + "deepseek-r1", "deepseek-v2", "deepseek-v2-chat", "deepseek-v2-chat-0628", "deepseek-v2.5", + "deepseek-v3", "deepseek-vl-chat", "deepseek-vl2"] } From 4f5ff3846a42b8ad6904f7f6708f4dfdbfa7b6e7 Mon Sep 17 00:00:00 2001 From: lao Date: Mon, 14 Apr 2025 18:10:45 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat(utils):=20=E6=B7=BB=E5=8A=A0XInference?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=B9=B6=E6=9B=B4=E6=96=B0=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在utils.py中添加对XInference的支持，并在.env.example中添加XINFERENCE_API_KEY环境变量。这些更改使得系统能够更好地集成XInference服务。 --- .env.example | 1 + src/utils/utils.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index d123dbef..7989a1f0 100644 --- a/.env.example +++ b/.env.example @@ -31,6 +31,7 @@ SiliconFLOW_ENDPOINT=https://api.siliconflow.cn/v1/ SiliconFLOW_API_KEY= XINFERENCE_OPENAI_ENDPOINT=https://api.xinference.com/v1 +XINFERENCE_API_KEY= XINFERENCE_MODEL= # Set to false to disable anonymized telemetry diff --git a/src/utils/utils.py b/src/utils/utils.py index 3f5806e3..f353247a 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -24,7 +24,8 @@ "google": "Google", "alibaba": "Alibaba", "moonshot": "MoonShot", - "unbound": "Unbound AI" + "unbound": "Unbound AI", + "xinference": "XInference", } @@ -189,8 +190,17 @@ def get_llm_model(provider: str, **kwargs): else: base_url = kwargs.get("base_url") + if not kwargs.get("api_key", ""): + api_key = os.getenv("XINFERENCE_API_KEY", "") + else: + api_key = kwargs.get("api_key") + + if not kwargs.get("model_name", ""): + model_name = os.getenv("XINFERENCE_MODEL", "qwen2.5-instruct") + else: + model_name = kwargs.get("model_name") return ChatOpenAI( - model=kwargs.get("model_name", "gpt-4o"), + model=model_name, temperature=kwargs.get("temperature", 0.0), base_url=base_url, api_key=api_key,