Skip to content

Commit 0ef6e55

Browse files
authored
chat : add new template for DeepSeek V4 Flash 0731 (ggml-org#26398)
* common/chat: update DeepSeek V4 templates Align the DeepSeek V4 templates with the official encoders while keeping parser behavior out of this change. - Default drop_thinking for DeepSeek V4 history so prior thinking is omitted unless preserve_reasoning is requested or tools are present. - Add structured output response-format instructions to the V4 templates and pass the schema into template rendering. - Add a separate Flash 0731 template for the updated high and max reasoning effort mapping. - Cover reasoning effort, drop_thinking, structured output prompts, preserved reasoning, continuations, and empty tool arguments in template rendering tests. Official references: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/encoding/encoding_dsv4.py https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731/blob/main/encoding/encoding_dsv4.py Assisted-by: Codex * Fix deepseek v4 0731 template selection * remove unneeded lower normalization * Fix DSML parser to consume the tool call separator * address aldehir requests * address aldehir comment
1 parent 94bc47f commit 0ef6e55

6 files changed

Lines changed: 363 additions & 12 deletions

File tree

common/chat.cpp

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2114,6 +2114,11 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
21142114
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
21152115
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
21162116

2117+
std::optional<json> additional_context;
2118+
if (is_v4 && has_response_format) {
2119+
additional_context = json{ { "response_format", inputs.json_schema } };
2120+
}
2121+
21172122
const std::string DSML = "|DSML|";
21182123
const std::string THINK_START = "<think>";
21192124
const std::string THINK_END = "</think>";
@@ -2125,9 +2130,12 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
21252130
const std::string PARAM_START = "<" + DSML + "parameter";
21262131
const std::string PARAM_END = "</" + DSML + "parameter>";
21272132
const std::string GEN_PROMPT = "<|Assistant|>";
2133+
const std::string TC_SEPARATOR = "\n\n";
21282134

2129-
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
2130-
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);
2135+
data.prompt = common_chat_template_direct_apply_impl(
2136+
tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
2137+
data.generation_prompt = common_chat_template_generation_prompt_impl(
2138+
tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
21312139
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
21322140
data.supports_thinking = true;
21332141
data.thinking_start_tag = THINK_START;
@@ -2141,9 +2149,16 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
21412149
if (inputs.has_continuation()) {
21422150
const auto & msg = inputs.continue_msg;
21432151

2144-
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
2145-
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
2146-
data.generation_prompt += THINK_END + msg.render_content();
2152+
if (is_v4 && msg.reasoning_content.empty()) {
2153+
data.generation_prompt = GEN_PROMPT + THINK_END;
2154+
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
2155+
data.generation_prompt += msg.render_content();
2156+
}
2157+
} else {
2158+
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
2159+
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
2160+
data.generation_prompt += THINK_END + msg.render_content();
2161+
}
21472162
}
21482163

21492164
data.prompt += data.generation_prompt;
@@ -2242,7 +2257,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
22422257

22432258
if (extract_reasoning && inputs.enable_thinking) {
22442259
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
2245-
reasoning_with_tc = THINK_START + p.reasoning(p.until_one_of({ FC_START, THINK_END })) + obligatory_tool_calls;
2260+
reasoning_with_tc = THINK_START +
2261+
p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
2262+
p.space() + obligatory_tool_calls;
22462263
allow_reasoning_with_tc = true;
22472264
} else if (extract_reasoning) {
22482265
// Thinking disabled but reasoning extraction requested: the generation prompt
@@ -2265,7 +2282,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
22652282
return generation_prompt + reasoning + p.content(p.rest()) + end;
22662283
}
22672284

2268-
auto content_before_tools = p.negate(p.literal(THINK_START)) + p.content(p.until(FC_START));
2285+
auto content_before_tools = p.negate(p.literal(THINK_START)) +
2286+
p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) +
2287+
p.space();
22692288
return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :
22702289
generation_prompt + reasoning + content_before_tools + tool_calls + end;
22712290
});

common/jinja/caps.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,7 @@ caps caps_get(jinja::program & prog) {
482482
});
483483
},
484484
[&](context & ctx) {
485+
ctx.set_val("enable_thinking", mk_val<value_bool>(true));
485486
caps_apply_preserve_reasoning(ctx, true);
486487
},
487488
nullptr, // tools_fn

conversion/deepseek.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -535,7 +535,10 @@ def __init__(self, *args, **kwargs):
535535
logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors)
536536

537537
# add a default chat template; if the model has a built-in template, it will be overridden later
538-
template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja"
538+
model_id_hint = self.remote_hf_model_id or self.dir_model.name
539+
is_0731 = "0731" in model_id_hint
540+
template_name = "deepseek-ai-DeepSeek-V4-Flash-0731.jinja" if is_0731 else "deepseek-ai-DeepSeek-V4.jinja"
541+
template_path = Path(__file__).parent.parent / "models" / "templates" / template_name
539542
if template_path.is_file():
540543
with open(template_path, "r", encoding="utf-8") as f:
541544
self.gguf_writer.add_chat_template(f.read())
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
{%- if not add_generation_prompt is defined -%}
2+
{%- set add_generation_prompt = false -%}
3+
{%- endif -%}
4+
{%- if not thinking is defined -%}
5+
{%- if enable_thinking is defined -%}
6+
{%- set thinking = enable_thinking -%}
7+
{%- else -%}
8+
{%- set thinking = false -%}
9+
{%- endif -%}
10+
{%- endif -%}
11+
{%- if not drop_thinking is defined -%}
12+
{%- set drop_thinking = true -%}
13+
{%- endif -%}
14+
{%- set dsml_token = '|DSML|' -%}
15+
{%- set thinking_start_token = '<think>' -%}
16+
{%- set thinking_end_token = '</think>' -%}
17+
{%- set reasoning_effort_high = 'Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n' -%}
18+
{%- set reasoning_effort_max = 'Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\nYou MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\nDo not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n' -%}
19+
{%- set response_format_template = '## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n' -%}
20+
{%- set has_tools = false -%}
21+
{%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</' + dsml_token + 'parameter>\n...\n</' + dsml_token + 'invoke>\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n</' + dsml_token + 'invoke>\n</' + dsml_token + 'tool_calls>\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%}
22+
{%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%}
23+
{%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%}
24+
{%- for message in messages -%}
25+
{%- if message['role'] == 'system' -%}
26+
{%- if ns.is_first_sp -%}
27+
{%- set ns.system_prompt = ns.system_prompt + (message['content'] or '') -%}
28+
{%- set ns.is_first_sp = false -%}
29+
{%- else -%}
30+
{%- set ns.system_prompt = ns.system_prompt + '\n\n' + (message['content'] or '') -%}
31+
{%- endif -%}
32+
{%- endif -%}
33+
{%- endfor -%}
34+
{%- if tools is defined and tools -%}
35+
{%- set has_tools = true -%}
36+
{%- set ts = namespace(schemas='') -%}
37+
{%- for tool in tools -%}
38+
{%- if tool['type'] == 'function' -%}
39+
{%- set ts.schemas = ts.schemas + (tool['function'] | tojson) + '\n' -%}
40+
{%- endif -%}
41+
{%- endfor -%}
42+
{%- if ns.system_prompt -%}
43+
{%- set ns.system_prompt = ns.system_prompt + '\n\n' + tools_header + ts.schemas + tools_footer -%}
44+
{%- else -%}
45+
{%- set ns.system_prompt = tools_header + ts.schemas + tools_footer -%}
46+
{%- endif -%}
47+
{%- endif -%}
48+
{%- if response_format is defined -%}
49+
{%- if ns.system_prompt -%}
50+
{%- set ns.system_prompt = ns.system_prompt + '\n\n' -%}
51+
{%- endif -%}
52+
{%- set ns.system_prompt = ns.system_prompt + response_format_template + (response_format | tojson) -%}
53+
{%- endif -%}
54+
{{- bos_token -}}
55+
{%- if messages and thinking and reasoning_effort is defined and reasoning_effort == 'high' -%}
56+
{{- reasoning_effort_high -}}
57+
{%- elif messages and thinking and reasoning_effort is defined and reasoning_effort == 'max' -%}
58+
{{- reasoning_effort_max -}}
59+
{%- endif -%}
60+
{{- ns.system_prompt -}}
61+
{%- set last_user_idx = namespace(value=-1) -%}
62+
{%- for message in messages -%}
63+
{%- if message['role'] == 'user' or message['role'] == 'developer' or message['role'] == 'tool' -%}
64+
{%- set last_user_idx.value = loop.index0 -%}
65+
{%- endif -%}
66+
{%- endfor -%}
67+
{%- set state = namespace(in_user=false) -%}
68+
{%- for message in messages -%}
69+
{%- if message['role'] == 'tool' -%}
70+
{%- set ns.has_tool_calls = true -%}
71+
{%- endif -%}
72+
{%- endfor -%}
73+
{%- for message in messages -%}
74+
{%- if message['role'] == 'user' or message['role'] == 'developer' -%}
75+
{%- if state.in_user -%}
76+
{{- '\n\n' -}}
77+
{%- else -%}
78+
{{- '<|User|>' -}}
79+
{%- set state.in_user = true -%}
80+
{%- endif -%}
81+
{{- message['content'] or '' -}}
82+
{%- elif message['role'] == 'tool' -%}
83+
{%- if state.in_user -%}
84+
{{- '\n\n' -}}
85+
{%- else -%}
86+
{{- '<|User|>' -}}
87+
{%- set state.in_user = true -%}
88+
{%- endif -%}
89+
{{- '<tool_result>' + (message['content'] or '') + '</tool_result>' -}}
90+
{%- elif message['role'] == 'assistant' -%}
91+
{%- set state.in_user = false -%}
92+
{{- '<|Assistant|>' -}}
93+
{%- set is_after_last_user = loop.index0 > last_user_idx.value -%}
94+
{%- set keep_reasoning = thinking and ((not drop_thinking) or has_tools or is_after_last_user or ns.has_tool_calls) -%}
95+
{%- if keep_reasoning -%}
96+
{{- thinking_start_token -}}
97+
{%- if message['reasoning_content'] is defined and message['reasoning_content'] -%}
98+
{{- message['reasoning_content'] -}}
99+
{%- endif -%}
100+
{{- thinking_end_token -}}
101+
{%- else -%}
102+
{{- thinking_end_token -}}
103+
{%- endif -%}
104+
{%- if message['content'] is defined and message['content'] -%}
105+
{{- message['content'] -}}
106+
{%- endif -%}
107+
{%- if message['tool_calls'] -%}
108+
{{- '\n\n<' + dsml_token + 'tool_calls>\n' -}}
109+
{%- for tool in message['tool_calls'] -%}
110+
{%- set func = tool['function'] -%}
111+
{{- '<' + dsml_token + 'invoke name="' + func['name'] + '">\n' -}}
112+
{%- set args = func['arguments'] -%}
113+
{%- if args is string -%}
114+
{%- set args = args | from_json -%}
115+
{%- endif -%}
116+
{%- for key, val in args.items() -%}
117+
{%- if val is string -%}
118+
{{- '<' + dsml_token + 'parameter name="' + key + '" string="true">' + val + '</' + dsml_token + 'parameter>\n' -}}
119+
{%- else -%}
120+
{{- '<' + dsml_token + 'parameter name="' + key + '" string="false">' + (val | tojson) + '</' + dsml_token + 'parameter>\n' -}}
121+
{%- endif -%}
122+
{%- endfor -%}
123+
{%- if not args -%}
124+
{{- '\n' -}}
125+
{%- endif -%}
126+
{{- '</' + dsml_token + 'invoke>\n' -}}
127+
{%- endfor -%}
128+
{{- '</' + dsml_token + 'tool_calls>' -}}
129+
{%- endif -%}
130+
{{- '<|end▁of▁sentence|>' -}}
131+
{%- endif -%}
132+
{%- endfor -%}
133+
{%- if add_generation_prompt -%}
134+
{{- '<|Assistant|>' -}}
135+
{%- if thinking -%}
136+
{{- thinking_start_token -}}
137+
{%- else -%}
138+
{{- thinking_end_token -}}
139+
{%- endif -%}
140+
{%- endif -%}

models/templates/deepseek-ai-DeepSeek-V4.jinja

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99
{%- endif -%}
1010
{%- endif -%}
1111
{%- if not drop_thinking is defined -%}
12-
{%- set drop_thinking = false -%}
12+
{%- set drop_thinking = true -%}
1313
{%- endif -%}
1414
{%- set dsml_token = '|DSML|' -%}
1515
{%- set thinking_start_token = '<think>' -%}
1616
{%- set thinking_end_token = '</think>' -%}
17+
{%- set reasoning_effort_max = 'Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n' -%}
18+
{%- set response_format_template = '## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n' -%}
19+
{%- set has_tools = false -%}
1720
{%- set tools_header = '## Tools\n\nYou have access to a set of tools to help answer the user\'s question. You can invoke tools by writing a "<' + dsml_token + 'tool_calls>" block like the following:\n\n<' + dsml_token + 'tool_calls>\n<' + dsml_token + 'invoke name="$TOOL_NAME">\n<' + dsml_token + 'parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</' + dsml_token + 'parameter>\n...\n</' + dsml_token + 'invoke>\n<' + dsml_token + 'invoke name="$TOOL_NAME2">\n...\n</' + dsml_token + 'invoke>\n</' + dsml_token + 'tool_calls>\n\nString parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.\n\nIf thinking_mode is enabled (triggered by ' + thinking_start_token + '), you MUST output your complete reasoning inside ' + thinking_start_token + '...' + thinking_end_token + ' BEFORE any tool calls or final response.\n\nOtherwise, output directly after ' + thinking_end_token + ' with tool calls or final response.\n\n### Available Tool Schemas\n\n' -%}
1821
{%- set tools_footer = '\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n' -%}
1922
{%- set ns = namespace(system_prompt='', is_first_sp=true, has_tool_calls=false) -%}
@@ -28,6 +31,7 @@
2831
{%- endif -%}
2932
{%- endfor -%}
3033
{%- if tools is defined and tools -%}
34+
{%- set has_tools = true -%}
3135
{%- set ts = namespace(schemas='') -%}
3236
{%- for tool in tools -%}
3337
{%- if tool['type'] == 'function' -%}
@@ -40,7 +44,16 @@
4044
{%- set ns.system_prompt = tools_header + ts.schemas + tools_footer -%}
4145
{%- endif -%}
4246
{%- endif -%}
47+
{%- if response_format is defined -%}
48+
{%- if ns.system_prompt -%}
49+
{%- set ns.system_prompt = ns.system_prompt + '\n\n' -%}
50+
{%- endif -%}
51+
{%- set ns.system_prompt = ns.system_prompt + response_format_template + (response_format | tojson) -%}
52+
{%- endif -%}
4353
{{- bos_token -}}
54+
{%- if messages and thinking and reasoning_effort is defined and reasoning_effort == 'max' -%}
55+
{{- reasoning_effort_max -}}
56+
{%- endif -%}
4457
{{- ns.system_prompt -}}
4558
{%- set last_user_idx = namespace(value=-1) -%}
4659
{%- for message in messages -%}
@@ -75,8 +88,8 @@
7588
{%- set state.in_user = false -%}
7689
{{- '<|Assistant|>' -}}
7790
{%- set is_after_last_user = loop.index0 > last_user_idx.value -%}
78-
{%- set retain_reasoning = (not drop_thinking) or (is_after_last_user or ns.has_tool_calls) -%}
79-
{%- if retain_reasoning and thinking -%}
91+
{%- set keep_reasoning = thinking and ((not drop_thinking) or has_tools or is_after_last_user or ns.has_tool_calls) -%}
92+
{%- if keep_reasoning -%}
8093
{{- thinking_start_token -}}
8194
{%- if message['reasoning_content'] is defined and message['reasoning_content'] -%}
8295
{{- message['reasoning_content'] -}}
@@ -104,6 +117,9 @@
104117
{{- '<' + dsml_token + 'parameter name="' + key + '" string="false">' + (val | tojson) + '</' + dsml_token + 'parameter>\n' -}}
105118
{%- endif -%}
106119
{%- endfor -%}
120+
{%- if not args -%}
121+
{{- '\n' -}}
122+
{%- endif -%}
107123
{{- '</' + dsml_token + 'invoke>\n' -}}
108124
{%- endfor -%}
109125
{{- '</' + dsml_token + 'tool_calls>' -}}
@@ -118,4 +134,4 @@
118134
{%- else -%}
119135
{{- thinking_end_token -}}
120136
{%- endif -%}
121-
{%- endif -%}
137+
{%- endif -%}

0 commit comments

Comments
 (0)