Skip to content

Commit e6d9b52

Browse files
author
Olivier Chafik
committed
align Command R7B w/ --think / reasoning_content behaviour
1 parent 3841a16 commit e6d9b52

File tree

9 files changed

+176
-87
lines changed

9 files changed

+176
-87
lines changed

‎common/arg.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1978,7 +1978,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
19781978
add_opt(common_arg(
19791979
{"--think"},
19801980
"*experimental* thinking mode (default: disabled)\n"
1981-
"returns reasoning_content in messages, forcing model to think unless it supports native <think> tags (DeepSeek R1)\n"
1981+
"returns reasoning_content in messages, forcing model to think unless it supports native <think> tags (DeepSeek R1, Command R7B)\n"
19821982
"only supported for non-streamed responses",
19831983
[](common_params & params) {
19841984
params.think = true;

‎common/chat-template.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,7 @@ class chat_template {
316316

317317
auto polyfill_system_role = opts.polyfill_system_role && !caps_.supports_system_role;
318318
auto polyfill_tools = opts.polyfill_tools && has_tools && !caps_.supports_tools;
319-
auto polyfill_tool_call_example = polyfill_tools && opts.polyfill_tool_call_examples;
319+
auto polyfill_tool_call_example = polyfill_tools && opts.polyfill_tool_call_examples && caps_.supports_tool_calls;
320320
auto polyfill_tool_calls = opts.polyfill_tool_calls && has_tool_calls && !caps_.supports_tool_calls;
321321
auto polyfill_tool_responses = opts.polyfill_tool_responses && has_tool_responses && !caps_.supports_tool_responses;
322322
auto polyfill_object_arguments = opts.polyfill_object_arguments && has_tool_calls && caps_.requires_object_arguments;

‎common/chat.cpp‎

Lines changed: 51 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,13 @@ std::string common_chat_format_name(common_chat_format format) {
1212
case COMMON_CHAT_FORMAT_LLAMA_3_X: return "Llama 3.x";
1313
case COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS: return "Llama 3.x with builtin tools";
1414
case COMMON_CHAT_FORMAT_DEEPSEEK_R1: return "DeepSeek R1";
15-
case COMMON_CHAT_FORMAT_DEEPSEEK_R1_THINK: return "DeepSeek R1 (extract <think>)";
15+
case COMMON_CHAT_FORMAT_DEEPSEEK_R1_THINK: return "DeepSeek R1 (extract reasoning_content)";
1616
case COMMON_CHAT_FORMAT_FIREFUNCTION_V2: return "FireFunction v2";
1717
case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2: return "Functionary v3.2";
1818
case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1: return "Functionary v3.1 Llama 3.1";
1919
case COMMON_CHAT_FORMAT_HERMES_2_PRO: return "Hermes 2 Pro";
2020
case COMMON_CHAT_FORMAT_COMMAND_R7B: return "Command R7B";
21+
case COMMON_CHAT_FORMAT_COMMAND_R7B_THINK: return "Command R7B (extract reasoning_content)";
2122
default:
2223
throw std::runtime_error("Unknown chat format");
2324
}
@@ -469,22 +470,49 @@ static common_chat_params common_chat_params_init_command_r7b(const common_chat_
469470
"<|END_THINKING|>",
470471
"<|END_ACTION|>",
471472
};
472-
data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);
473-
data.format = COMMON_CHAT_FORMAT_COMMAND_R7B;
473+
auto adjusted_messages = json::array();
474+
for (const auto & msg : inputs.messages) {
475+
auto has_reasoning_content = msg.contains("reasoning_content") && msg["reasoning_content"].is_string();
476+
auto has_tool_calls = msg.contains("tool_calls") && msg["tool_calls"].is_array();
477+
if (has_reasoning_content && has_tool_calls) {
478+
auto adjusted_message = msg;
479+
adjusted_message["tool_plan"] = msg["reasoning_content"];
480+
adjusted_message.erase("reasoning_content");
481+
adjusted_messages.push_back(adjusted_message);
482+
} else {
483+
adjusted_messages.push_back(msg);
484+
}
485+
}
486+
// } else {
487+
// adjusted_messages = inputs.messages;
488+
// }
489+
data.prompt = apply(tmpl, adjusted_messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt, {});
490+
data.format = inputs.think ? COMMON_CHAT_FORMAT_COMMAND_R7B_THINK : COMMON_CHAT_FORMAT_COMMAND_R7B;
474491
return data;
475492
}
476-
static common_chat_msg common_chat_parse_command_r7b(const std::string & input) {
477-
static std::regex response_regex("<\\|START_RESPONSE\\|>([\\s\\S\\n\\r]*?)<\\|END_RESPONSE\\|>");
478-
static std::regex thought_action_regex("<\\|START_THINKING\\|>([\\s\\S\\n\\r]*?)<\\|END_THINKING\\|><\\|START_ACTION\\|>([\\s\\S\\n\\r]*?)<\\|END_ACTION\\|>");
493+
static common_chat_msg common_chat_parse_command_r7b(const std::string & input, bool think) {
494+
static std::regex thought_regex("(<\\|START_THINKING\\|>([\\s\\S\\n\\r]*?)<\\|END_THINKING\\|>)([\\s\\S\\n\\r]*)");
495+
static std::regex action_regex("<\\|START_ACTION\\|>([\\s\\S\\n\\r]*?)<\\|END_ACTION\\|>");
496+
static std::regex response_regex("(?:<\\|START_RESPONSE\\|>)?([\\s\\S\\n\\r]*?)<\\|END_RESPONSE\\|>");
497+
479498
std::smatch match;
480499

481500
common_chat_msg result;
482501
result.role = "assistant";
483-
if (std::regex_match(input, match, response_regex)) {
484-
result.content = match[1].str();
485-
} else if (std::regex_match(input, match, thought_action_regex)) {
486-
result.tool_plan = match[1].str();
487-
auto actions_str = match[2].str();
502+
503+
std::string rest = input;
504+
505+
if (std::regex_match(rest, match, thought_regex)) {
506+
if (think) {
507+
result.reasoning_content = match[2].str();
508+
} else if (!match[2].str().empty()) {
509+
// Let the unparsed thinking tags through in content only if their insides aren't empty.
510+
result.content = match[1].str();
511+
}
512+
rest = match[3].str();
513+
}
514+
if (std::regex_match(rest, match, action_regex)) {
515+
auto actions_str = match[1].str();
488516
auto actions = json::parse(actions_str);
489517
for (const auto & action : actions) {
490518
result.tool_calls.push_back({
@@ -493,9 +521,12 @@ static common_chat_msg common_chat_parse_command_r7b(const std::string & input)
493521
/* .id = */ action["tool_call_id"],
494522
});
495523
}
524+
} else if (std::regex_match(rest, match, response_regex)) {
525+
auto response = match[1].str();
526+
result.content += response;
496527
} else {
497528
LOG_ERR("Failed to parse command_r output");
498-
result.content = input;
529+
result.content += rest;
499530
}
500531
return result;
501532
}
@@ -1038,6 +1069,11 @@ common_chat_params common_chat_params_init(const common_chat_template & tmpl, co
10381069
return common_chat_params_init_deepseek_r1(tmpl, inputs);
10391070
}
10401071

1072+
// Command R7B: : use handler in all cases except json schema (thinking / tools).
1073+
if (src.find("<|END_THINKING|><|START_ACTION|>") != std::string::npos && inputs.json_schema.is_null()) {
1074+
return common_chat_params_init_command_r7b(tmpl, inputs);
1075+
}
1076+
10411077
// Use generic handler when forcing thoughts or JSON schema for final output
10421078
// TODO: support thinking mode and/or JSON schema in handlers below this.
10431079
if (inputs.think || (!inputs.tools.is_null() && inputs.json_schema.is_object())) {
@@ -1081,11 +1117,6 @@ common_chat_params common_chat_params_init(const common_chat_template & tmpl, co
10811117
return common_chat_params_init_mistral_nemo(tmpl, inputs);
10821118
}
10831119

1084-
// Command R7B (w/ tools)
1085-
if (src.find("<|END_THINKING|><|START_ACTION|>") != std::string::npos) {
1086-
return common_chat_params_init_command_r7b(tmpl, inputs);
1087-
}
1088-
10891120
// Generic fallback
10901121
return common_chat_params_init_generic(tmpl, inputs);
10911122
}
@@ -1123,7 +1154,9 @@ common_chat_msg common_chat_parse(const std::string & input, common_chat_format
11231154
case COMMON_CHAT_FORMAT_FIREFUNCTION_V2:
11241155
return common_chat_parse_firefunction_v2(input);
11251156
case COMMON_CHAT_FORMAT_COMMAND_R7B:
1126-
return common_chat_parse_command_r7b(input);
1157+
return common_chat_parse_command_r7b(input, /* think= */ false);
1158+
case COMMON_CHAT_FORMAT_COMMAND_R7B_THINK:
1159+
return common_chat_parse_command_r7b(input, /* think= */ true);
11271160
default:
11281161
throw std::runtime_error("Unsupported format: " + common_chat_format_name(format));
11291162
}

‎common/chat.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ enum common_chat_format {
3535
COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1,
3636
COMMON_CHAT_FORMAT_HERMES_2_PRO,
3737
COMMON_CHAT_FORMAT_COMMAND_R7B,
38+
COMMON_CHAT_FORMAT_COMMAND_R7B_THINK,
3839

3940
COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats
4041
};

‎common/common.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -625,7 +625,6 @@ struct common_chat_msg {
625625
std::string content;
626626
std::vector<common_tool_call> tool_calls;
627627
std::string reasoning_content = "";
628-
std::string tool_plan = "";
629628
};
630629

631630
// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid

‎examples/server/README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ The project is under active development, and we are [looking for feedback and co
127127
| `--grammar-file FNAME` | file to read grammar from |
128128
| `-j, --json-schema SCHEMA` | JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object<br/>For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead |
129129
| `--jinja` | Enable experimental Jinja templating engine (required for tool use) |
130+
| `--think` | Enable experimental thinking mode (extracts DeepSeek R1 & Command R7B's native thinking tags and forces any other model to think before responding, resulting thoughts are in the `reasoning_content` output field) (requires `--jinja`) |
131+
--think
130132

131133
**Example-specific params**
132134

@@ -1223,10 +1225,10 @@ curl http://localhost:8080/v1/chat/completions \
12231225

12241226
# Native support for DeepSeek R1 works best w/ our own template (official template buggy)
12251227

1226-
llama-server --jinja -fa -hf bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q6_K_L \
1228+
llama-server --jinja -fa -hf bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q6_K_L --think \
12271229
--chat-template-file models/templates/llama-cpp-deepseek-r1.jinja
12281230

1229-
llama-server --jinja -fa -hf bartowski/DeepSeek-R1-Distill-Qwen-32B-GGUF:Q4_K_M \
1231+
llama-server --jinja -fa -hf bartowski/DeepSeek-R1-Distill-Qwen-32B-GGUF:Q4_K_M --think \
12301232
--chat-template-file models/templates/llama-cpp-deepseek-r1.jinja
12311233

12321234
# Native support requires the right template for these GGUFs:
@@ -1240,7 +1242,7 @@ curl http://localhost:8080/v1/chat/completions \
12401242
llama-server --jinja -fa -hf bartowski/firefunction-v2-GGUF -hff firefunction-v2-IQ1_M.gguf \
12411243
--chat-template-file <( python scripts/get_chat_template.py fireworks-ai/llama-3-firefunction-v2 tool_use )
12421244

1243-
llama-server --jinja -fa -hf bartowski/c4ai-command-r7b-12-2024-GGUF:Q6_K_L \
1245+
llama-server --jinja -fa -hf bartowski/c4ai-command-r7b-12-2024-GGUF:Q6_K_L --think \
12441246
--chat-template-file <( python scripts/get_chat_template.py CohereForAI/c4ai-command-r7b-12-2024 tool_use )
12451247

12461248
# Generic format support

‎examples/server/server.cpp‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -748,9 +748,6 @@ struct server_task_result_cmpl_final : server_task_result {
748748
if (!msg.reasoning_content.empty()) {
749749
message["reasoning_content"] = msg.reasoning_content;
750750
}
751-
if (!msg.tool_plan.empty()) {
752-
message["tool_plan"] = msg.tool_plan;
753-
}
754751

755752
json choice {
756753
{"finish_reason", finish_reason},

‎examples/server/tests/unit/test_tool_call.py‎

Lines changed: 46 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -274,43 +274,44 @@ def test_completion_without_tool_call_slow(template_name: str, n_predict: int, t
274274

275275

276276
@pytest.mark.slow
277-
@pytest.mark.parametrize("hf_repo,template_override", [
278-
("bartowski/c4ai-command-r7b-12-2024-GGUF:Q4_K_M", ("CohereForAI/c4ai-command-r7b-12-2024", "tool_use")),
277+
@pytest.mark.parametrize("think,hf_repo,template_override", [
278+
(True, "bartowski/c4ai-command-r7b-12-2024-GGUF:Q4_K_M", ("CohereForAI/c4ai-command-r7b-12-2024", "tool_use")),
279279
280-
("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
281-
("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
280+
(False, "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
281+
(False, "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
282282
283-
("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
284-
("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),
283+
(False, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
284+
(False, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),
285285
286-
("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),
287-
("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),
286+
(False, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),
287+
(False, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),
288288
289-
("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
290-
("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),
289+
(False, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
290+
(False, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),
291291
292-
("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
293-
("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),
292+
(False, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),
293+
(False, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),
294294
295-
("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
296-
("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
295+
(False, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
296+
(False, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
297297
298-
("bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai/functionary-medium-v3.2", None)),
299-
("bartowski/functionary-small-v3.2-GGUF:Q8_0", "chatml"),
298+
(False, "bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai/functionary-medium-v3.2", None)),
299+
(False, "bartowski/functionary-small-v3.2-GGUF:Q8_0", "chatml"),
300300
301-
("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
302-
("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),
301+
(False, "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
302+
(False, "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),
303303
304-
("bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
304+
(True, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
305305
306306
# Note: gemma-2-2b-it knows itself as "model", not "assistant", so we don't test the ill-suited chatml on it.
307-
("bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
307+
(False, "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
308308
309309
# ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),
310310
])
311-
def test_weather(hf_repo: str, template_override: Tuple[str, str | None] | None):
311+
def test_weather(think: bool, hf_repo: str, template_override: Tuple[str, str | None] | None):
312312
global server
313313
n_predict = 512
314+
server.think = think
314315
server.n_slots = 1
315316
server.jinja = True
316317
server.n_ctx = 8192
@@ -488,44 +489,45 @@ def test_thoughts(n_predict: int, think: bool, expect_content: str | None, expec
488489

489490

490491
@pytest.mark.slow
491-
@pytest.mark.parametrize("expected_arguments_override,hf_repo,template_override", [
492-
(None, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
493-
(None, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", "chatml"),
492+
@pytest.mark.parametrize("think,expected_arguments_override,hf_repo,template_override", [
493+
(True, None, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),
494+
(True, None, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", "chatml"),
494495
495-
(None, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
496-
(None, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),
496+
(False, None, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),
497+
(False, None, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),
497498
498-
(None, "bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai-functionary-medium-v3.2", None)),
499-
(None, "bartowski/functionary-small-v3.2-GGUF:Q8_0", "chatml"),
499+
(False, None, "bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai-functionary-medium-v3.2", None)),
500+
(False, None, "bartowski/functionary-small-v3.2-GGUF:Q8_0", "chatml"),
500501
501-
(None, "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
502-
('{"code":"print("}', "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
502+
(False, None, "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),
503+
(False, '{"code":"print("}', "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),
503504
504-
('{"code":"print("}', "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),
505-
(None, "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", "chatml"),
505+
(False, '{"code":"print("}', "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),
506+
(False, None, "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", "chatml"),
506507
507-
('{"code":"print("}', "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),
508-
('{"code":"print("}', "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),
508+
(False, '{"code":"print("}', "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),
509+
(False, '{"code":"print("}', "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),
509510
510-
(None, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),
511-
(None, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),
511+
(False, None, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),
512+
(False, None, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),
512513
513-
(None, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
514-
(None, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),
514+
(False, None, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),
515+
(False, None, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),
515516
516-
(None, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch-Hermes-3-Llama-3.1-8B", "tool_use")),
517-
(None, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),
517+
(False, None, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch-Hermes-3-Llama-3.1-8B", "tool_use")),
518+
(False, None, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),
518519
519-
(None, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
520-
(None, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
520+
(False, None, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),
521+
(False, None, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),
521522
522523
# Note: gemma-2-2b-it knows itself as "model", not "assistant", so we don't test the ill-suited chatml on it.
523-
(None, "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
524+
(False, None, "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),
524525
])
525-
def test_hello_world_tool_call(expected_arguments_override: str | None, hf_repo: str, template_override: str | Tuple[str, str | None] | None):
526+
def test_hello_world(think: bool, expected_arguments_override: str | None, hf_repo: str, template_override: str | Tuple[str, str | None] | None):
526527
global server
527528
server.n_slots = 1
528529
server.jinja = True
530+
server.think = think
529531
server.n_ctx = 8192
530532
server.n_predict = 512 # High because of DeepSeek R1
531533
server.model_hf_repo = hf_repo

0 commit comments

Comments
 (0)