Skip to content

Commit 2fbe3b7

Browse files
aldehiraldehir
authored
common : add parser for ministral/mistral large 3/devstral 2 (#17713)
1 parent 6339185 commit 2fbe3b7

File tree

4 files changed

+415
-0
lines changed

4 files changed

+415
-0
lines changed

‎common/chat.cpp‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include "chat.h"
22
#include "chat-parser.h"
3+
#include "chat-peg-parser.h"
34
#include "common.h"
45
#include "json-partial.h"
56
#include "json-schema-to-grammar.h"
@@ -150,6 +151,7 @@ struct templates_params {
150151
common_chat_tool_choice tool_choice;
151152
json json_schema;
152153
bool parallel_tool_calls;
154+
common_reasoning_format reasoning_format;
153155
bool stream;
154156
std::string grammar;
155157
bool add_generation_prompt = true;
@@ -589,6 +591,16 @@ common_chat_templates_ptr common_chat_templates_init(
589591
"{%- if false %}");
590592
}
591593

594+
// TODO @aldehir : this is a temporary fix, pending Minja changes
595+
// Ref: https://github.com/ggml-org/llama.cpp/pull/17713#issuecomment-3631342664
596+
if (default_template_src.find("[TOOL_CALLS]") != std::string::npos
597+
// search for the error message and patch it
598+
&& default_template_src.find("if (message['content'] is none or") != std::string::npos) {
599+
string_replace_all(default_template_src,
600+
"{%- if (message['content'] is none or message['content'] == '' or message['content']|length == 0) and (message['tool_calls'] is not defined or message['tool_calls'] is none or message['tool_calls']|length == 0) %}",
601+
"{%- if false %}");
602+
}
603+
592604
std::string token_bos = bos_token_override;
593605
std::string token_eos = eos_token_override;
594606
bool add_bos = false;
@@ -987,6 +999,118 @@ static common_chat_params common_chat_params_init_lfm2(const common_chat_templat
987999
return data;
9881000
}
9891001

1002+
static common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const struct templates_params & inputs) {
1003+
common_chat_params data;
1004+
1005+
// Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja
1006+
auto adjusted_messages = json::array();
1007+
for (const auto & msg : inputs.messages) {
1008+
auto role = msg.value("role", "");
1009+
if (role != "system" && role != "assistant") {
1010+
// Only adjust system and assistant messages. Interestingly, the system message may contain thinking.
1011+
adjusted_messages.push_back(msg);
1012+
continue;
1013+
}
1014+
1015+
auto content = json::array();
1016+
1017+
// If message contains `reasoning_content`, add it as a block of type `thinking`
1018+
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
1019+
content.push_back({
1020+
{"type", "thinking"},
1021+
{"thinking", msg.at("reasoning_content").get<std::string>()},
1022+
});
1023+
}
1024+
1025+
// If message contains `content`, add it as a block of type `text`
1026+
if (msg.contains("content")) {
1027+
if (msg.at("content").is_string()) {
1028+
content.push_back({
1029+
{"type", "text"},
1030+
{"text", msg.at("content").get<std::string>()},
1031+
});
1032+
} else if (msg.at("content").is_array()) {
1033+
auto blocks = msg.at("content");
1034+
content.insert(content.end(), blocks.begin(), blocks.end());
1035+
}
1036+
}
1037+
1038+
auto adjusted = msg;
1039+
adjusted["content"] = content;
1040+
adjusted.erase("reasoning_content");
1041+
adjusted_messages.push_back(adjusted);
1042+
}
1043+
1044+
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
1045+
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
1046+
auto include_grammar = true;
1047+
1048+
data.prompt = apply(tmpl, inputs, /* messages_override = */ adjusted_messages);
1049+
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
1050+
data.preserved_tokens = {
1051+
"[THINK]",
1052+
"[/THINK]",
1053+
"[TOOL_CALLS]",
1054+
"[ARGS]",
1055+
};
1056+
1057+
auto parser = build_chat_peg_native_parser([&](common_chat_peg_native_builder & p) {
1058+
auto reasoning = extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
1059+
1060+
// Response format parser
1061+
if (inputs.json_schema.is_object() && !inputs.json_schema.empty()) {
1062+
// Ministral wants to emit json surrounded by code fences
1063+
return reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```";
1064+
}
1065+
1066+
// Tool call parser
1067+
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
1068+
auto tool_choice = p.choice();
1069+
foreach_function(inputs.tools, [&](const json & tool) {
1070+
const auto & function = tool.at("function");
1071+
std::string name = function.at("name");
1072+
const auto & schema = function.at("parameters");
1073+
1074+
tool_choice |= p.rule("tool-" + name,
1075+
p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]")
1076+
+ p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))
1077+
);
1078+
});
1079+
1080+
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
1081+
auto max_calls = inputs.parallel_tool_calls ? -1 : 1;
1082+
auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls));
1083+
1084+
return reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls;
1085+
}
1086+
1087+
// Content only parser
1088+
include_grammar = false;
1089+
return reasoning << p.content(p.rest());
1090+
});
1091+
1092+
data.parser = parser.save();
1093+
1094+
if (include_grammar) {
1095+
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
1096+
1097+
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
1098+
foreach_function(inputs.tools, [&](const json & tool) {
1099+
const auto & function = tool.at("function");
1100+
auto schema = function.at("parameters");
1101+
builder.resolve_refs(schema);
1102+
});
1103+
parser.build_grammar(builder, data.grammar_lazy);
1104+
});
1105+
1106+
data.grammar_triggers = {
1107+
{COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]"}
1108+
};
1109+
}
1110+
1111+
return data;
1112+
}
1113+
9901114
static common_chat_params common_chat_params_init_magistral(const common_chat_template & tmpl, const struct templates_params & inputs) {
9911115
common_chat_params data;
9921116
data.prompt = apply(tmpl, inputs);
@@ -2341,6 +2465,7 @@ static common_chat_params common_chat_templates_apply_jinja(
23412465
params.messages = common_chat_msgs_to_json_oaicompat<json>(inputs.messages, /* concat_text= */ !tmpl.original_caps().requires_typed_content);
23422466
params.add_generation_prompt = inputs.add_generation_prompt;
23432467
params.tool_choice = inputs.tool_choice;
2468+
params.reasoning_format = inputs.reasoning_format;
23442469
params.enable_thinking = inputs.enable_thinking;
23452470
params.grammar = inputs.grammar;
23462471
params.now = inputs.now;
@@ -2504,6 +2629,13 @@ static common_chat_params common_chat_templates_apply_jinja(
25042629
return common_chat_params_init_llama_3_x(tmpl, params, allow_python_tag_builtin_tools);
25052630
}
25062631

2632+
// Ministral/Mistral Large 3
2633+
if (src.find("[SYSTEM_PROMPT]") != std::string::npos &&
2634+
src.find("[TOOL_CALLS]") != std::string::npos &&
2635+
src.find("[ARGS]") != std::string::npos) {
2636+
return common_chat_params_init_ministral_3(tmpl, params);
2637+
}
2638+
25072639
if (src.find("[THINK]") != std::string::npos && src.find("[/THINK]") != std::string::npos) {
25082640
return common_chat_params_init_magistral(tmpl, params);
25092641
}

‎models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
{#- Default system message if no system prompt is passed. #}
2+
{%- set default_system_message = '# HOW YOU SHOULD THINK AND ANSWER\n\nFirst draft your thinking process (inner monologue) until you arrive at a response. Format your response using Markdown, and use LaTeX for any mathematical equations. Write both your thoughts and the response in the same language as the input.\n\nYour thinking process must follow the template below:[THINK]Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate the response to the user.[/THINK]Here, provide a self-contained response.' %}
3+
4+
{#- Begin of sequence token. #}
5+
{{- bos_token }}
6+
7+
{#- Handle system prompt if it exists. #}
8+
{#- System prompt supports text content or text and thinking chunks. #}
9+
{%- if messages[0]['role'] == 'system' %}
10+
{{- '[SYSTEM_PROMPT]' -}}
11+
{%- if messages[0]['content'] is string %}
12+
{{- messages[0]['content'] -}}
13+
{%- else %}
14+
{%- for block in messages[0]['content'] %}
15+
{%- if block['type'] == 'text' %}
16+
{{- block['text'] }}
17+
{%- elif block['type'] == 'thinking' %}
18+
{{- '[THINK]' + block['thinking'] + '[/THINK]' }}
19+
{%- else %}
20+
{{- raise_exception('Only text and thinking chunks are supported in system message contents.') }}
21+
{%- endif %}
22+
{%- endfor %}
23+
{%- endif %}
24+
{{- '[/SYSTEM_PROMPT]' -}}
25+
{%- set loop_messages = messages[1:] %}
26+
{%- else %}
27+
{%- set loop_messages = messages %}
28+
{%- if default_system_message != '' %}
29+
{{- '[SYSTEM_PROMPT]' + default_system_message + '[/SYSTEM_PROMPT]' }}
30+
{%- endif %}
31+
{%- endif %}
32+
33+
34+
{#- Tools definition #}
35+
{%- set tools_definition = '' %}
36+
{%- set has_tools = false %}
37+
{%- if tools is defined and tools is not none and tools|length > 0 %}
38+
{%- set has_tools = true %}
39+
{%- set tools_definition = '[AVAILABLE_TOOLS]' + (tools| tojson) + '[/AVAILABLE_TOOLS]' %}
40+
{{- tools_definition }}
41+
{%- endif %}
42+
43+
{#- Checks for alternating user/assistant messages. #}
44+
{%- set ns = namespace(index=0) %}
45+
{%- for message in loop_messages %}
46+
{%- if message.role == 'user' or (message.role == 'assistant' and (message.tool_calls is not defined or message.tool_calls is none or message.tool_calls | length == 0)) %}
47+
{%- if (message['role'] == 'user') != (ns.index % 2 == 0) %}
48+
{{- raise_exception('After the optional system message, conversation roles must alternate user and assistant roles except for tool calls and results.') }}
49+
{%- endif %}
50+
{%- set ns.index = ns.index + 1 %}
51+
{%- endif %}
52+
{%- endfor %}
53+
54+
{#- Handle conversation messages. #}
55+
{%- for message in loop_messages %}
56+
57+
{#- User messages supports text content or text and image chunks. #}
58+
{%- if message['role'] == 'user' %}
59+
{%- if message['content'] is string %}
60+
{{- '[INST]' + message['content'] + '[/INST]' }}
61+
{%- elif message['content'] | length > 0 %}
62+
{{- '[INST]' }}
63+
{%- if message['content'] | length == 2 %}
64+
{%- set blocks = message['content'] | sort(attribute='type') %}
65+
{%- else %}
66+
{%- set blocks = message['content'] %}
67+
{%- endif %}
68+
{%- for block in blocks %}
69+
{%- if block['type'] == 'text' %}
70+
{{- block['text'] }}
71+
{%- elif block['type'] in ['image', 'image_url'] %}
72+
{{- '[IMG]' }}
73+
{%- else %}
74+
{{- raise_exception('Only text, image and image_url chunks are supported in user message content.') }}
75+
{%- endif %}
76+
{%- endfor %}
77+
{{- '[/INST]' }}
78+
{%- else %}
79+
{{- raise_exception('User message must have a string or a list of chunks in content') }}
80+
{%- endif %}
81+
82+
{#- Assistant messages supports text content or text, image and thinking chunks. #}
83+
{%- elif message['role'] == 'assistant' %}
84+
{%- if (message['content'] is none or message['content'] == '' or message['content']|length == 0) and (message['tool_calls'] is not defined or message['tool_calls'] is none or message['tool_calls']|length == 0) %}
85+
{{- raise_exception('Assistant message must have a string or a list of chunks in content or a list of tool calls.') }}
86+
{%- endif %}
87+
88+
{%- if message['content'] is string and message['content'] != '' %}
89+
{{- message['content'] }}
90+
{%- elif message['content'] | length > 0 %}
91+
{%- for block in message['content'] %}
92+
{%- if block['type'] == 'text' %}
93+
{{- block['text'] }}
94+
{%- elif block['type'] == 'thinking' %}
95+
{{- '[THINK]' + block['thinking'] + '[/THINK]' }}
96+
{%- else %}
97+
{{- raise_exception('Only text and thinking chunks are supported in assistant message contents.') }}
98+
{%- endif %}
99+
{%- endfor %}
100+
{%- endif %}
101+
102+
{%- if message['tool_calls'] is defined and message['tool_calls'] is not none and message['tool_calls']|length > 0 %}
103+
{%- for tool in message['tool_calls'] %}
104+
{{- '[TOOL_CALLS]' }}
105+
{%- set name = tool['function']['name'] %}
106+
{%- set arguments = tool['function']['arguments'] %}
107+
{%- if arguments is not string %}
108+
{%- set arguments = arguments|tojson|safe %}
109+
{%- elif arguments == '' %}
110+
{%- set arguments = '{}' %}
111+
{%- endif %}
112+
{{- name + '[ARGS]' + arguments }}
113+
{%- endfor %}
114+
{%- endif %}
115+
116+
{{- eos_token }}
117+
118+
{#- Tool messages only supports text content. #}
119+
{%- elif message['role'] == 'tool' %}
120+
{{- '[TOOL_RESULTS]' + message['content']|string + '[/TOOL_RESULTS]' }}
121+
122+
{#- Raise exception for unsupported roles. #}
123+
{%- else %}
124+
{{- raise_exception('Only user, assistant and tool roles are supported, got ' + message['role'] + '.') }}
125+
{%- endif %}
126+
{%- endfor %}

0 commit comments

Comments
 (0)