Skip to content

Commit 4cb0e1d

Browse files
author
Olivier Chafik
committed
Merge branch 'jinja-chatml' into r1-toolcall
2 parents 2b3c482 + aa98e59 commit 4cb0e1d

File tree

3 files changed

+106
-32
lines changed

3 files changed

+106
-32
lines changed

‎common/common.cpp‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1869,11 +1869,18 @@ std::string common_chat_format_example(const common_chat_template & tmpl, bool u
18691869
return common_chat_apply_template(tmpl, msgs, true, use_jinja);
18701870
}
18711871

1872+
#define CHATML_TEMPLATE_SRC \
1873+
"{%- for message in messages -%}\n" \
1874+
" {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' -}}\n" \
1875+
"{%- endfor -%}\n" \
1876+
"{%- if add_generation_prompt -%}\n" \
1877+
" {{- '<|im_start|>assistant\n' -}}\n" \
1878+
"{%- endif -%}"
1879+
18721880
common_chat_templates common_chat_templates_from_model(const struct llama_model * model, const std::string & chat_template_override)
18731881
{
1874-
auto vocab = llama_model_get_vocab(model);
1875-
std::string default_template_src = chat_template_override;
1876-
std::string template_tool_use_src = chat_template_override;
1882+
std::string default_template_src = chat_template_override == "chatml" ? CHATML_TEMPLATE_SRC : chat_template_override;
1883+
std::string template_tool_use_src = chat_template_override == "chatml" ? CHATML_TEMPLATE_SRC : "";
18771884
bool has_explicit_template = !chat_template_override.empty();
18781885
if (chat_template_override.empty()) {
18791886
auto str = llama_model_chat_template(model, /* name */ nullptr);
@@ -1891,16 +1898,14 @@ common_chat_templates common_chat_templates_from_model(const struct llama_model
18911898
if (!template_tool_use_src.empty()) {
18921899
default_template_src = template_tool_use_src;
18931900
} else {
1894-
default_template_src = R"(
1895-
{%- for message in messages -%}
1896-
{{- "<|im_start|>" + message.role + "\n" + message.content + "<|im_end|>\n" -}}
1897-
{%- endfor -%}
1898-
{%- if add_generation_prompt -%}
1899-
{{- "<|im_start|>assistant\n" -}}
1900-
{%- endif -%}
1901-
)";
1901+
default_template_src = CHATML_TEMPLATE_SRC;
19021902
}
19031903
}
1904+
std::string token_bos;
1905+
std::string token_eos;
1906+
// TODO: update logic that adds BOS and EOS tokens to the tokenized prompt, in favour of the template.
1907+
#if 0
1908+
auto vocab = llama_model_get_vocab(model);
19041909
const auto get_token = [&](llama_token token, const char * name, const char * jinja_variable_name) {
19051910
if (token == LLAMA_TOKEN_NULL) {
19061911
if (default_template_src.find(jinja_variable_name) != std::string::npos
@@ -1912,8 +1917,9 @@ common_chat_templates common_chat_templates_from_model(const struct llama_model
19121917
return common_token_to_piece(vocab, token, true);
19131918
}
19141919
};
1915-
auto token_bos = get_token(llama_vocab_bos(vocab), "BOS", "bos_token");
1916-
auto token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");
1920+
token_bos = get_token(llama_vocab_bos(vocab), "BOS", "bos_token");
1921+
token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");
1922+
#endif
19171923
return {
19181924
has_explicit_template,
19191925
std::make_unique<minja::chat_template>(default_template_src, token_bos, token_eos),

‎examples/server/tests/unit/test_chat_completion.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@ def create_server():
1313
@pytest.mark.parametrize(
1414
"model,system_prompt,user_prompt,max_tokens,re_content,n_prompt,n_predicted,finish_reason,jinja,chat_template",
1515
[
16+
(None, "Book", "Hey", 8, "But she couldn't", 69, 8, "length", False, None),
17+
(None, "Book", "Hey", 8, "But she couldn't", 69, 8, "length", True, None),
1618
(None, "Book", "What is the best book", 8, "(Suddenly)+|\\{ \" Sarax.", 77, 8, "length", False, None),
17-
(None, "Book", "What is the best book", 8, "(Suddenly)+|\\{ \" Sarax.", 77, 8, "length", True, None),
18-
(None, "Book", "What is the best book", 8, "^ blue", 23, 8, "length", True, "This is not a chat template, it is"),
19+
(None, "Book", "What is the best book", 8, "(Suddenly)+|\\{ \" Sarax.", 77, 8, "length", True, None),
20+
(None, "Book", "What is the best book", 8, "(Suddenly)+|\\{ \" Sarax.", 77, 8, "length", True, 'chatml'),
21+
(None, "Book", "What is the best book", 8, "^ blue", 23, 8, "length", True, "This is not a chat template, it is"),
1922
("codellama70b", "You are a coding assistant.", "Write the fibonacci function in c++.", 128, "(Aside|she|felter|alonger)+", 104, 64, "length", False, None),
2023
("codellama70b", "You are a coding assistant.", "Write the fibonacci function in c++.", 128, "(Aside|she|felter|alonger)+", 104, 64, "length", True, None),
2124
]

‎examples/server/tests/unit/test_tool_call.py‎

Lines changed: 82 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ def create_server():
6767

6868

6969
def do_test_completion_with_required_tool_tiny(template_name: str, tool: dict, argument_key: str | None):
70-
n_predict = 512
7170
global server
71+
n_predict = 512
7272
# server = ServerPreset.stories15m_moe()
7373
server.jinja = True
7474
server.n_predict = n_predict
@@ -139,40 +139,62 @@ def test_completion_with_required_tool_tiny_slow(template_name: str, tool: dict,
139139
@pytest.mark.parametrize("tool,argument_key,hf_repo,template_override", [
140140
(TEST_TOOL, "success", "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
141141
(PYTHON_TOOL, "code", "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
142+
(PYTHON_TOOL, "code", "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
143+
144+
# Note: gemma-2-2b-it knows itself as "model", not "assistant", so we don't test the ill-suited chatml on it.
142145
(TEST_TOOL, "success", "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
143146
(PYTHON_TOOL, "code", "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
147+
144148
(TEST_TOOL, "success", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
145149
(PYTHON_TOOL, "code", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
150+
(PYTHON_TOOL, "code", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),
151+
146152
(TEST_TOOL, "success", "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),
147153
(PYTHON_TOOL, "code", "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),
154+
(PYTHON_TOOL, "code", "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),
155+
148156
(TEST_TOOL, "success", "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
149157
(PYTHON_TOOL, "code", "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
158+
(PYTHON_TOOL, "code", "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),
159+
150160
(TEST_TOOL, "success", "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
151161
(PYTHON_TOOL, "code", "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
162+
(PYTHON_TOOL, "code", "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),
163+
152164
(TEST_TOOL, "success", "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
153165
(PYTHON_TOOL, "code", "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
154-
(TEST_TOOL, "success", "bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai/functionary-medium-v3.2", None)),
155-
(PYTHON_TOOL, "code", "bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai/functionary-medium-v3.2", None)),
166+
(PYTHON_TOOL, "code", "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
167+
168+
(TEST_TOOL, "success", "bartowski/functionary-small-v3.2-GGUF:Q4_K_M", ("meetkai/functionary-medium-v3.2", None)),
169+
(PYTHON_TOOL, "code", "bartowski/functionary-small-v3.2-GGUF:Q4_K_M", ("meetkai/functionary-medium-v3.2", None)),
170+
(PYTHON_TOOL, "code", "bartowski/functionary-small-v3.2-GGUF:Q4_K_M", "chatml"),
171+
156172
(TEST_TOOL, "success", "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
157173
(PYTHON_TOOL, "code", "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
174+
(PYTHON_TOOL, "code", "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),
175+
158176
(TEST_TOOL, "success", "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
159177
(PYTHON_TOOL, "code", "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
178+
(PYTHON_TOOL, "code", "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", "chatml"),
160179
# TODO: fix these
161180
# (TEST_TOOL, "success", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
162181
# (PYTHON_TOOL, "code", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
163182
])
164-
def test_completion_with_required_tool_real_model(tool: dict, argument_key: str | None, hf_repo: str, template_override: Tuple[str, str | None] | None):
183+
def test_completion_with_required_tool_real_model(tool: dict, argument_key: str | None, hf_repo: str, template_override: str | Tuple[str, str | None] | None):
184+
global server
165185
n_predict = 512
166186
server.n_slots = 1
167187
server.jinja = True
168188
server.n_ctx = 8192
169189
server.n_predict = n_predict
170190
server.model_hf_repo = hf_repo
171191
server.model_hf_file = None
172-
if template_override:
192+
if isinstance(template_override, tuple):
173193
(template_hf_repo, template_variant) = template_override
174194
server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"
175195
assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."
196+
elif isinstance(template_override, str):
197+
server.chat_template = template_override
176198
server.start(timeout_seconds=TIMEOUT_SERVER_START)
177199
res = server.make_request("POST", "/chat/completions", data={
178200
"max_tokens": n_predict,
@@ -253,17 +275,35 @@ def test_completion_without_tool_call_slow(template_name: str, n_predict: int, t
253275
@pytest.mark.parametrize("hf_repo,template_override", [
254276
("bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
255277
("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
256-
("bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
278+
("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
279+
257280
("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
281+
("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),
282+
258283
("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),
259-
("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
260-
("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
284+
("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),
285+
286+
("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
287+
("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),
288+
289+
("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
290+
("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),
291+
261292
("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
293+
("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
294+
262295
("bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai/functionary-medium-v3.2", None)),
296+
("bartowski/functionary-small-v3.2-GGUF:Q8_0", "chatml"),
297+
263298
("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
299+
("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),
300+
301+
# Note: gemma-2-2b-it knows itself as "model", not "assistant", so we don't test the ill-suited chatml on it.
302+
("bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
303+
264304
# ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
265305
])
266-
def test_weather_tool_call(hf_repo: str, template_override: Tuple[str, str | None] | None):
306+
def test_weather_tool_call(hf_repo: str, template_override: str | Tuple[str, str | None] | None):
267307
global server
268308
n_predict = 512
269309
server.n_slots = 1
@@ -272,10 +312,12 @@ def test_weather_tool_call(hf_repo: str, template_override: Tuple[str, str | Non
272312
server.n_predict = n_predict
273313
server.model_hf_repo = hf_repo
274314
server.model_hf_file = None
275-
if template_override:
315+
if isinstance(template_override, tuple):
276316
(template_hf_repo, template_variant) = template_override
277317
server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"
278318
assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."
319+
elif isinstance(template_override, str):
320+
server.chat_template = template_override
279321
server.start(timeout_seconds=TIMEOUT_SERVER_START)
280322
res = server.make_request("POST", "/chat/completions", data={
281323
"max_tokens": n_predict,
@@ -301,29 +343,52 @@ def test_weather_tool_call(hf_repo: str, template_override: Tuple[str, str | Non
301343
@pytest.mark.slow
302344
@pytest.mark.parametrize("expected_arguments_override,hf_repo,template_override", [
303345
(None, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
304-
(None, "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
346+
(None, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", "chatml"),
347+
305348
(None, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
349+
(None, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),
350+
306351
(None, "bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai-functionary-medium-v3.2", None)),
307-
('{"code":"print("}', "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
308-
(None, "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),
352+
(None, "bartowski/functionary-small-v3.2-GGUF:Q8_0", "chatml"),
353+
354+
(None, "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
355+
('{"code":"print("}', "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
356+
357+
('{"code":"print("}', "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),
358+
(None, "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", "chatml"),
359+
309360
('{"code":"print("}', "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),
361+
('{"code":"print("}', "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),
362+
310363
(None, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),
311-
(None, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
312-
(None, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch-Hermes-3-Llama-3.1-8B", "tool_use")),
364+
(None, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),
365+
366+
(None, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
367+
(None, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),
368+
369+
(None, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch-Hermes-3-Llama-3.1-8B", "tool_use")),
370+
(None, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),
371+
313372
(None, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
373+
(None, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
374+
375+
# Note: gemma-2-2b-it knows itself as "model", not "assistant", so we don't test the ill-suited chatml on it.
376+
(None, "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
314377
])
315-
def test_hello_world_tool_call(expected_arguments_override: str | None, hf_repo: str, template_override: Tuple[str, str | None] | None):
378+
def test_hello_world_tool_call(expected_arguments_override: str | None, hf_repo: str, template_override: str | Tuple[str, str | None] | None):
316379
global server
317380
server.n_slots = 1
318381
server.jinja = True
319382
server.n_ctx = 8192
320383
server.n_predict = 512 # High because of DeepSeek R1
321384
server.model_hf_repo = hf_repo
322385
server.model_hf_file = None
323-
if template_override:
386+
if isinstance(template_override, tuple):
324387
(template_hf_repo, template_variant) = template_override
325388
server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"
326389
assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."
390+
elif isinstance(template_override, str):
391+
server.chat_template = template_override
327392
server.start(timeout_seconds=TIMEOUT_SERVER_START)
328393
res = server.make_request("POST", "/chat/completions", data={
329394
"max_tokens": 256,

0 commit comments

Comments
 (0)