-
Notifications
You must be signed in to change notification settings - Fork 1.1k
Feature/modelscope support #898
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from 5 commits
67f544c
964b9d4
bd043bd
8434abd
32dcb31
7e650a7
68504b5
5f91ca5
2c0ab23
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -26,17 +26,41 @@ def check_llm_connection( | |||||
| # 创建OpenAI客户端并发送请求到API | ||||||
| base_url = normalize_base_url(base_url) | ||||||
| api_key = api_key.strip() | ||||||
| response = openai.OpenAI( | ||||||
| base_url=base_url, api_key=api_key, timeout=60 | ||||||
| ).chat.completions.create( | ||||||
| model=model, | ||||||
| messages=[ | ||||||
| {"role": "system", "content": "You are a helpful assistant."}, | ||||||
| {"role": "user", "content": 'Just respond with "Hello"!'}, | ||||||
| ], | ||||||
| timeout=30, | ||||||
| ) | ||||||
| return True, response.choices[0].message.content | ||||||
| # Check whether it is the ModelScope platform, as some models require the stream and enable_thinking parameter | ||||||
| if "modelscope" in base_url: | ||||||
|
||||||
| extra_body = {"enable_thinking": True} | ||||||
| response_stream = openai.OpenAI( | ||||||
| base_url=base_url, api_key=api_key, timeout=60 | ||||||
| ).chat.completions.create( | ||||||
| model=model, | ||||||
| messages=[ | ||||||
| {"role": "system", "content": "You are a helpful assistant."}, | ||||||
| {"role": "user", 'content': 'Just respond with "Hello"!'}, | ||||||
|
||||||
| {"role": "user", 'content': 'Just respond with "Hello"!'}, | |
| {"role": "user", "content": "Just respond with \"Hello\"!"}, |
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -3,6 +3,7 @@ | |||||||||||||||||||||||||||
| import os | ||||||||||||||||||||||||||||
| import threading | ||||||||||||||||||||||||||||
| from typing import Any, List, Optional | ||||||||||||||||||||||||||||
| from types import SimpleNamespace | ||||||||||||||||||||||||||||
| from urllib.parse import urlparse, urlunparse | ||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||
| import openai | ||||||||||||||||||||||||||||
|
|
@@ -135,13 +136,37 @@ def call_llm( | |||||||||||||||||||||||||||
| ValueError: If response is invalid (empty choices or content) | ||||||||||||||||||||||||||||
| """ | ||||||||||||||||||||||||||||
| client = get_llm_client() | ||||||||||||||||||||||||||||
| # Check whether it is the ModelScope platform, as some models require the stream and enable_thinking parameters | ||||||||||||||||||||||||||||
| if "modelscope" in str(client.base_url): | ||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||
| logger.info("Detected ModelScope API, using stream mode with enable_thinking=True.") | ||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||
| extra_body = {"enable_thinking": True} | ||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||
| response_stream = client.chat.completions.create( | ||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||
| model=model, | ||||||||||||||||||||||||||||
| messages=messages, # pyright: ignore[reportArgumentType] | ||||||||||||||||||||||||||||
| temperature=temperature, | ||||||||||||||||||||||||||||
| stream=True, | ||||||||||||||||||||||||||||
| extra_body=extra_body, | ||||||||||||||||||||||||||||
| **kwargs, | ||||||||||||||||||||||||||||
| ) | ||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||
| response = client.chat.completions.create( | ||||||||||||||||||||||||||||
| model=model, | ||||||||||||||||||||||||||||
| messages=messages, # pyright: ignore[reportArgumentType] | ||||||||||||||||||||||||||||
| temperature=temperature, | ||||||||||||||||||||||||||||
| **kwargs, | ||||||||||||||||||||||||||||
| ) | ||||||||||||||||||||||||||||
| full_content = "" | ||||||||||||||||||||||||||||
| for chunk in response_stream: | ||||||||||||||||||||||||||||
| if chunk.choices and chunk.choices[0].delta.content: | ||||||||||||||||||||||||||||
| full_content += chunk.choices[0].delta.content | ||||||||||||||||||||||||||||
| if not full_content: | ||||||||||||||||||||||||||||
| raise ValueError("ModelScope streaming response yielded no content") | ||||||||||||||||||||||||||||
| fake_message = SimpleNamespace(content=full_content) | ||||||||||||||||||||||||||||
| fake_choice = SimpleNamespace(message=fake_message) | ||||||||||||||||||||||||||||
| response = SimpleNamespace(choices=[fake_choice]) | ||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||
| fake_message = SimpleNamespace(content=full_content) | |
| fake_choice = SimpleNamespace(message=fake_message) | |
| response = SimpleNamespace(choices=[fake_choice]) | |
| import time | |
| fake_message = SimpleNamespace(content=full_content) | |
| fake_choice = SimpleNamespace(message=fake_message, finish_reason="stop", index=0) | |
| response = SimpleNamespace( | |
| id="chatcmpl-fake-modelscope", | |
| object="chat.completion", | |
| created=int(time.time()), | |
| model=model, | |
| choices=[fake_choice], | |
| ) |
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -353,6 +353,14 @@ def __createLLMServiceCards(self): | |||||||||
| "default_base": "https://open.bigmodel.cn/api/paas/v4", | ||||||||||
| "default_models": ["glm-4-plus", "glm-4-air-250414", "glm-4-flash"], | ||||||||||
| }, | ||||||||||
| LLMServiceEnum.MODELSCOPE: { | ||||||||||
| "prefix": "modelscope", | ||||||||||
| "api_key_cfg": cfg.modelscope_api_key, | ||||||||||
| "api_base_cfg": cfg.modelscope_api_base, | ||||||||||
| "model_cfg": cfg.modelscope_model, | ||||||||||
| "default_base": "https://api-inference.modelscope.cn/v1", | ||||||||||
| "default_models": ["Qwen/Qwen3-8B", "Qwen/Qwen3-30B-A3B-Instruct-2507", "deepseek-ai/DeepSeek-V3.1"], | ||||||||||
|
||||||||||
| "default_models": ["Qwen/Qwen3-8B", "Qwen/Qwen3-30B-A3B-Instruct-2507", "deepseek-ai/DeepSeek-V3.1"], | |
| "default_models": ["Qwen/Qwen3-8B", "Qwen/Qwen2.5-32B-Instruct", "deepseek-ai/DeepSeek-V3.1"], |
Copilot
AI
Nov 28, 2025
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The model name "deepseek-ai/DeepSeek-V3.1" may be incorrect. Based on DeepSeek's release history and the documentation in this repository (which references "deepseek-ai/DeepSeek-V3"), the current latest version is V3, not V3.1.
Please verify this model exists in ModelScope's registry. If it doesn't exist, consider using "deepseek-ai/DeepSeek-V3" instead, which is the verified model name used in the documentation.
| "default_models": ["Qwen/Qwen3-8B", "Qwen/Qwen3-30B-A3B-Instruct-2507", "deepseek-ai/DeepSeek-V3.1"], | |
| "default_models": ["Qwen/Qwen3-8B", "Qwen/Qwen3-30B-A3B-Instruct-2507", "deepseek-ai/DeepSeek-V3"], |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The ModelScope detection logic relies on a simple substring check of "modelscope" in the base_url. This approach is fragile and could match unintended URLs (e.g., "example.com/modelscope-test"). Consider using a more robust detection mechanism, such as checking against the configured ModelScope base URL from the config, or using the LLMServiceEnum to properly identify the service type.