Skip to content

Commit fe00bf3

Browse files
shell-nlpshell-nlp
committed
patch
1 parent 7a19c79 commit fe00bf3

File tree

11 files changed

+457
-22
lines changed

11 files changed

+457
-22
lines changed

‎gpt_server/model_backend/lmdeploy_backend.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ def __init__(self, model_path) -> None:
4545
enable_prefix_caching = bool(os.getenv("enable_prefix_caching", False))
4646
max_model_len = os.getenv("max_model_len", None)
4747
gpu_memory_utilization = float(os.getenv("gpu_memory_utilization", 0.8))
48+
kv_cache_quant_policy = int(os.getenv("kv_cache_quant_policy", 0))
4849
dtype = os.getenv("dtype", "auto")
4950
logger.info(f"后端 {backend}")
5051
if backend == "pytorch":
@@ -54,6 +55,7 @@ def __init__(self, model_path) -> None:
5455
session_len=int(max_model_len) if max_model_len else None,
5556
enable_prefix_caching=enable_prefix_caching,
5657
cache_max_entry_count=gpu_memory_utilization,
58+
quant_policy=kv_cache_quant_policy,
5759
)
5860
if backend == "turbomind":
5961
backend_config = TurbomindEngineConfig(
@@ -62,6 +64,7 @@ def __init__(self, model_path) -> None:
6264
session_len=int(max_model_len) if max_model_len else None,
6365
dtype=dtype,
6466
cache_max_entry_count=gpu_memory_utilization,
67+
quant_policy=kv_cache_quant_policy, # 默认为：0
6568
)
6669
pipeline_type, pipeline_class = get_task(model_path)
6770
logger.info(f"模型架构：{pipeline_type}")
@@ -118,7 +121,7 @@ async def stream_chat(self, params: Dict[str, Any]) -> AsyncGenerator:
118121
# Abort the request if the client disconnects.
119122
await self.async_engine.stop_session(session_id=request_id)
120123
text_outputs += request_output.response
121-
124+
122125
usage = {
123126
"prompt_tokens": request_output.input_token_len,
124127
"completion_tokens": request_output.generate_token_len,

‎gpt_server/model_handler/react/v1/chatglm_react.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
from typing import Any, Dict, List, Tuple, Union, Optional
2+
import json
3+
import uuid
4+
5+
from gpt_server.model_handler.react.v0.prompt import (
6+
GLM4_TOOL_PROMPT,
7+
TOOL_SUFFIX_PROMPT,
8+
)
9+
10+
11+
def glm4_tool_formatter(
12+
tools: List[Dict[str, Any]], tool_choice_info: Optional[dict] = None
13+
) -> str:
14+
tool_text = "\n"
15+
tool_names = []
16+
for tool in tools:
17+
tool = tool["function"]
18+
tool_name = tool["name"]
19+
tool_text += f"## {tool_name}\n\n{json.dumps(tool, ensure_ascii=False, indent=4)}\n{TOOL_SUFFIX_PROMPT}\n\n"
20+
tool_names.append(tool_name)
21+
return GLM4_TOOL_PROMPT.format(
22+
tool_text=tool_text, tool_names=", ".join(tool_names)
23+
).strip()
24+
25+
26+
def glm4_tool_extractor(content: str) -> Union[str, List[Tuple[str, str]]]:
27+
i = content.rfind("Action:")
28+
j = content.rfind("Action Input:")
29+
tool_name = content[i + len("Action:") : j].strip().strip(".")
30+
tool_input = content[j + len("Action Input:") :].strip()
31+
try:
32+
tool_input_obj = json.loads(tool_input)
33+
except json.JSONDecodeError:
34+
return content
35+
tool_calls = []
36+
tool_call = {
37+
"index": 0,
38+
"id": "call_{}".format(uuid.uuid4().hex),
39+
"function": {"name": tool_name, "arguments": tool_input},
40+
}
41+
tool_calls.append(tool_call)
42+
43+
return tool_calls
44+
45+
46+
if __name__ == "__main__":
47+
import json
48+
49+
tools_str = """[{'type': 'function', 'function': {'name': 'track', 'description': '追踪指定股票的实时价格', 'parameters': {'type': 'object', 'properties': {'symbol': {'description': '需要追踪的股票代码', 'type': 'integer'}}, 'required': ['symbol']}}}, {'type': 'function', 'function': {'name': 'text-to-speech', 'description': '将文本转换为语音', 'parameters': {'type': 'object', 'properties': {'text': {'description': '需要转换成语音的文本', 'type': 'string'}, 'voice': {'description': '要使用的语音类型（男声、女声等', 'default': '男声', 'type': 'string'}, 'speed': {'description': '语音的速度（快、中等、慢等', 'default': '中等', 'type': 'string'}}, 'required': ['text']}}}]"""
50+
tools_str = tools_str.replace("'", '"')
51+
tools = json.loads(tools_str)
52+
53+
res = glm4_tool_formatter(tools=tools)
54+
print(res)
55+
print()
56+
out = 'multiply\n{"first_int": 8, "second_int": 9}'
57+
r = glm4_tool_extractor(out)
58+
print(r)

‎gpt_server/model_handler/react/v1/prompt.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
TOOL_SUFFIX_PROMPT = (
2+
"在调用上述工具时，Action Input的值必须使用 Json 格式来表示调用的参数。"
3+
)
4+
5+
TOOL_CHOICE_SUFFIX_PROMPT = "\n注意: 上述工具必须被调用！"
6+
# default
7+
TOOL_SYSTEM_PROMPT = """Answer the following questions as best you can. You have access to the following tools:
8+
9+
{tool_text}
10+
11+
Use the following format:
12+
13+
Question: the input question you must answer
14+
Thought: you should always think about what to do
15+
Action: the action to take, should be one of [{tool_names}]
16+
Action Input: the input to the action
17+
Observation: the result of the action
18+
... (this Thought/Action/Action Input/Observation can be repeated zero or more times)
19+
Thought: I now know the final answer
20+
Final Answer: the final answer to the original input question
21+
22+
Begin!
23+
24+
Question:"""
25+
TOOL_SYSTEM_PROMPT_CN = """尽可能回答用户问题，你有权使用以下工具：
26+
27+
{tool_text}
28+
29+
如果使用工具请遵循以下格式回复：
30+
31+
Thought: 思考你当前步骤需要解决什么问题，是否需要使用工具
32+
Action: 工具名称，你的工具必须从 [{tool_names}] 选择
33+
Action Input: 工具输入参数, Action Input的值必须使用 Json 格式来表示调用的参数。
34+
Observation: 调用工具后的结果
35+
... (Thought/Action/Action Input/Observation 可以重复零次或多次)
36+
Thought: 我现在知道了最终答案
37+
Final Answer: 原始输入问题的最终答案
38+
39+
开始!"""
40+
41+
TOOl_CHOICE_SYSTEM_PROMPT_CN = """你是一个工具的执行助手，提供的工具可能是用于将用户的输入格式化为符合工具描述的json模式或者是其它功能。你需要自己判断，你必须强制使用以下工具:
42+
43+
{tool_text}
44+
45+
遵循以下格式：
46+
47+
Thought: 我必须强制执行 {tool_names} 工具
48+
Action: 工具名称必须是 {tool_names}
49+
Action Input: 工具输入参数, Action Input的值必须使用 Json 格式来表示调用的参数。
50+
Observation: 调用工具后的结果
51+
Thought: 我现在知道了最终答案
52+
Final Answer: 原始输入问题的最终答案
53+
54+
开始!"""
55+
TOOl_CHOICE_SYSTEM_PROMPT = """You must use the following tools:
56+
57+
{tool_text}
58+
59+
Use the following format:
60+
61+
Question: the input question you must answer
62+
Thought: I have to execute tool {tool_names}
63+
Action: the action to take, should be one of [{tool_names}]
64+
Action Input: the input to the action
65+
Observation: the result of the action
66+
... (this Thought/Action/Action Input/Observation can be repeated zero or more times)
67+
Thought: I now know the final answer
68+
Final Answer: the final answer to the original input question
69+
70+
Begin!
71+
72+
Question:"""
73+
74+
# 你的任务是针对用户的问题和要求提供适当的答复和支持
75+
GLM4_TOOL_PROMPT = """"你可以使用以下工具提供适当的答复和支持。
76+
77+
# 可用工具
78+
{tool_text}
79+
Use the following format:
80+
81+
Question: the input question you must answer
82+
Thought: you should always think about what to do
83+
Action: the action to take, should be one of [{tool_names}]
84+
Action Input: the input to the action
85+
Observation: the result of the action
86+
... (this Thought/Action/Action Input/Observation can be repeated zero or more times)
87+
Thought: I now know the final answer
88+
Final Answer: the final answer to the original input question
89+
90+
Begin!
91+
92+
Question:
93+
"""

‎gpt_server/model_handler/react/v1/prompts/__init__.py‎

Whitespace-only changes.

‎gpt_server/model_handler/react/v1/prompts/qwen_prompt.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
TOOL_SUFFIX_PROMPT = (
2+
"在调用上述工具时，action_input的值必须使用 Json 格式来表示调用的参数。"
3+
)
4+
5+
TOOL_CHOICE_SUFFIX_PROMPT = "\n\n## 注意: \n上述工具必须被调用!"
6+
# default
7+
8+
TOOL_SYSTEM_PROMPT_CN = """# 工具
9+
## 你拥有如下工具：
10+
11+
{tool_text}
12+
13+
## 如果使用工具，你可以回复零次、一次或多次以下json格式内容，以调用工具,调用工具后,Observation 表示调用工具后的结果,json格式如下:
14+
{{
15+
"thought":"你应该时刻思考自己该做什么",
16+
"reason":{{
17+
"action":"工具名称，必须是 [{tool_names}] 之一",
18+
"action_input":"工具输入, 值必须使用 json 格式"
19+
}}
20+
}}
21+
或
22+
{{
23+
"thought":"你应该时刻思考自己该做什么",
24+
"reason":{{
25+
"final_answer":"根据工具结果进行回复，如果工具返回值存在图片url,需将图片用![](url)渲染出来"
26+
}}
27+
}}
28+
"""
29+
30+
TOOl_CHOICE_SYSTEM_PROMPT_CN = """# 提供的工具是用于将用户的输入或回复格式化为符合工具描述的json模式,你必须强制使用以下工具:
31+
## 工具
32+
## #你拥有如下工具：
33+
34+
{tool_text}
35+
36+
### 你可以在回复中插入零次、一次或多次以下json格式内容，以调用工具,调用工具后,Observation 表示调用工具后的结果,json格式如下:
37+
{{
38+
"thought":"你应该时刻思考自己该做什么",
39+
"reason":{{
40+
"action":"工具名称，必须是 [{tool_names}] 之一",
41+
"action_input":"工具输入, 值必须使用 json 格式"
42+
}}
43+
}}
44+
或
45+
{{
46+
"thought":"你应该时刻思考自己该做什么",
47+
"reason":{{
48+
"final_answer":"根据工具结果进行回复，如果工具返回值存在图片url,需将图片用![](url)渲染出来"
49+
}}
50+
}}"""

‎gpt_server/model_handler/react/v1/qwen_react.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
from loguru import logger
2+
from typing import Any, Dict, List, Tuple, Union, Optional
3+
import json
4+
import uuid
5+
6+
from gpt_server.model_handler.react.v1.prompts.qwen_prompt import (
7+
TOOL_SYSTEM_PROMPT_CN,
8+
TOOl_CHOICE_SYSTEM_PROMPT_CN,
9+
TOOL_CHOICE_SUFFIX_PROMPT,
10+
TOOL_SUFFIX_PROMPT,
11+
)
12+
13+
14+
def qwen_tool_formatter(
15+
tools: List[Dict[str, Any]], tool_choice_info: Optional[dict] = None
16+
) -> str:
17+
tool_chooce_suffix_prompt = ""
18+
logger.info(f"tool_choice_info: {tool_choice_info}")
19+
tool_system_prompt = TOOL_SYSTEM_PROMPT_CN
20+
if tool_choice_info:
21+
tool_chooce_suffix_prompt = TOOL_CHOICE_SUFFIX_PROMPT
22+
tools = [tools[tool_choice_info["tool_choice_idx"]]]
23+
logger.info(f"tools 已被替换为tool_choic: {tools}")
24+
tool_system_prompt = TOOl_CHOICE_SYSTEM_PROMPT_CN
25+
26+
tool_names = []
27+
param_text_list = []
28+
for tool in tools:
29+
tool = tool["function"]
30+
tool_name = tool["name"]
31+
description = tool["description"]
32+
parameters = tool["parameters"]
33+
param_text = (
34+
"""### {tool_name}\n\n{tool_name}: {description} 输入参数： {parameters} \n"""
35+
+ TOOL_SUFFIX_PROMPT
36+
+ tool_chooce_suffix_prompt
37+
)
38+
param_text_str = param_text.format(
39+
tool_name=tool_name,
40+
description=description,
41+
parameters=parameters,
42+
)
43+
param_text_list.append(param_text_str)
44+
45+
tool_names.append(tool_name)
46+
47+
tool_text = "\n\n".join(param_text_list).strip()
48+
return tool_system_prompt.format(
49+
tool_text=tool_text,
50+
tool_names=", ".join(tool_names),
51+
)
52+
53+
54+
def qwen_tool_extractor(content: str) -> Union[str, List[Tuple[str, str]]]:
55+
output = json.loads(content)
56+
reason = output["reason"]
57+
final_answer = reason.get("final_answer", None)
58+
if final_answer: # 最终回答
59+
return output
60+
else: # 工具
61+
tool_name = reason["action"]
62+
tool_input = reason["action_input"]
63+
tool_calls = []
64+
tool_call = {
65+
"index": 0,
66+
"id": "call_{}".format(uuid.uuid4().hex),
67+
"function": {"name": tool_name, "arguments": tool_input},
68+
}
69+
tool_calls.append(tool_call)
70+
71+
return tool_calls
72+
73+
74+
if __name__ == "__main__":
75+
import json
76+
77+
tools_str = """[{'type': 'function', 'function': {'name': 'track', 'description': '追踪指定股票的实时价格', 'parameters': {'type': 'object', 'properties': {'symbol': {'description': '需要追踪的股票代码', 'type': 'integer'}}, 'required': ['symbol']}}}, {'type': 'function', 'function': {'name': 'text-to-speech', 'description': '将文本转换为语音', 'parameters': {'type': 'object', 'properties': {'text': {'description': '需要转换成语音的文本', 'type': 'string'}, 'voice': {'description': '要使用的语音类型（男声、女声等', 'default': '男声', 'type': 'string'}, 'speed': {'description': '语音的速度（快、中等、慢等', 'default': '中等', 'type': 'string'}}, 'required': ['text']}}}]"""
78+
tools_str = tools_str.replace("'", '"')
79+
tools = json.loads(tools_str)
80+
res = qwen_tool_formatter(tools=tools)
81+
print(res)
82+
83+
out = """{
84+
"thought":"你应该时刻思考自己该做什么",
85+
"reason":{
86+
"action":"track",
87+
"action_input":{"a":"1"}
88+
}
89+
}"""
90+
r = qwen_tool_extractor(out)
91+
print("\n\n")
92+
print(r)

‎gpt_server/model_handler/react/v1/schema.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@
33

44

55
class Action(BaseModel):
6-
Action: str = Field(description="工具名称，必须是 [{tool_names}] 之一")
7-
Action_Input: str = Field(description="工具输入, 值必须使用 json 格式")
6+
action: str = Field(description="工具名称，必须是 [{tool_names}] 之一")
7+
action_input: str = Field(description="工具输入, 值必须使用 json 格式")
88

99

1010
class Answer(BaseModel):
11-
Final_Answer: str = Field(description="问题的最终回答")
11+
final_answer: str = Field(description="问题的最终回答")
1212

1313

1414
class React(BaseModel):
15-
Thought: str = Field(description="你应该时刻思考自己该做什么")
16-
Reason: Union[Action, Answer]
15+
thought: str = Field(description="你应该时刻思考自己该做什么")
16+
reason: Union[Action, Answer]

0 commit comments

Comments
 (0)