Skip to content

Commit 90372ac

Browse files
author
ochafik
committed
allow all parsers to parse non-tool-call content.
1 parent 06b72b9 commit 90372ac

File tree

6 files changed

+89
-33
lines changed

6 files changed

+89
-33
lines changed

‎common/chat-parser.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class common_chat_msg_parser {
3030
const std::string & healing_marker() const { return healing_marker_; }
3131
const bool & is_partial() const { return is_partial_; }
3232
const common_chat_msg & result() const { return result_; }
33+
const common_chat_syntax & syntax() const { return syntax_; }
3334

3435
void move_to(size_t pos) {
3536
if (pos > input_.size()) {

‎common/chat.cpp‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -820,6 +820,10 @@ static common_chat_params common_chat_params_init_generic(const common_chat_temp
820820
return data;
821821
}
822822
static void common_chat_parse_generic(common_chat_msg_parser & builder) {
823+
if (!builder.syntax().parse_tool_calls) {
824+
builder.add_content(builder.consume_rest());
825+
return;
826+
}
823827
static const std::vector<std::vector<std::string>> content_paths = {
824828
{"response"},
825829
};
@@ -892,6 +896,11 @@ static common_chat_params common_chat_params_init_mistral_nemo(const common_chat
892896
return data;
893897
}
894898
static void common_chat_parse_mistral_nemo(common_chat_msg_parser & builder) {
899+
if (!builder.syntax().parse_tool_calls) {
900+
builder.add_content(builder.consume_rest());
901+
return;
902+
}
903+
895904
static const common_regex prefix(regex_escape("[TOOL_CALLS]"));
896905
parse_prefixed_json_tool_call_array(builder, prefix);
897906
}
@@ -1104,6 +1113,11 @@ static common_chat_params common_chat_params_init_llama_3_x(const common_chat_te
11041113
return data;
11051114
}
11061115
static void common_chat_parse_llama_3_1(common_chat_msg_parser & builder, bool with_builtin_tools = false) {
1116+
if (!builder.syntax().parse_tool_calls) {
1117+
builder.add_content(builder.consume_rest());
1118+
return;
1119+
}
1120+
11071121
static const common_regex function_regex(
11081122
"\\s*\\{\\s*(?:\"type\"\\s*:\\s*\"function\"\\s*,\\s*)?\"name\"\\s*:\\s*\"([^\"]+)\"\\s*,\\s*\"parameters\"\\s*: ");
11091123
static const common_regex close_regex("\\}\\s*");
@@ -1225,6 +1239,10 @@ static common_chat_params common_chat_params_init_deepseek_r1(const common_chat_
12251239
}
12261240
static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) {
12271241
builder.try_parse_reasoning("<think>", "</think>");
1242+
if (!builder.syntax().parse_tool_calls) {
1243+
builder.add_content(builder.consume_rest());
1244+
return;
1245+
}
12281246

12291247
static const common_regex tool_calls_begin("(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>|<｜tool▁calls｜>)");
12301248
static const common_regex tool_calls_end("<｜tool▁calls▁end｜>");
@@ -1286,6 +1304,10 @@ static common_chat_params common_chat_params_init_firefunction_v2(const common_c
12861304
return data;
12871305
}
12881306
static void common_chat_parse_firefunction_v2(common_chat_msg_parser & builder) {
1307+
if (!builder.syntax().parse_tool_calls) {
1308+
builder.add_content(builder.consume_rest());
1309+
return;
1310+
}
12891311
static const common_regex prefix(regex_escape(" functools["));
12901312
parse_prefixed_json_tool_call_array(builder, prefix, /* rstrip_prefix= */ 1);
12911313
}
@@ -1427,6 +1449,10 @@ static common_chat_params common_chat_params_init_functionary_v3_1_llama_3_1(con
14271449
return data;
14281450
}
14291451
static void common_chat_parse_functionary_v3_1_llama_3_1(common_chat_msg_parser & builder) {
1452+
if (!builder.syntax().parse_tool_calls) {
1453+
builder.add_content(builder.consume_rest());
1454+
return;
1455+
}
14301456
// This version of Functionary still supports the llama 3.1 tool call format for the python tool.
14311457
static const common_regex python_tag_regex(regex_escape("<|python_tag|>"));
14321458

@@ -1554,6 +1580,10 @@ static common_chat_params common_chat_params_init_hermes_2_pro(const common_chat
15541580
}
15551581
static void common_chat_parse_hermes_2_pro(common_chat_msg_parser & builder) {
15561582
builder.try_parse_reasoning("<think>", "</think>");
1583+
if (!builder.syntax().parse_tool_calls) {
1584+
builder.add_content(builder.consume_rest());
1585+
return;
1586+
}
15571587

15581588
static const common_regex open_regex(
15591589
"(?:"
@@ -1809,10 +1839,10 @@ static void common_chat_parse_content_only(common_chat_msg_parser & builder) {
18091839
builder.add_content(builder.consume_rest());
18101840
}
18111841

1812-
static void common_chat_parse(common_chat_msg_parser & builder, common_chat_format format) {
1813-
LOG_DBG("Parsing input with format %s: %s\n", common_chat_format_name(format).c_str(), builder.input().c_str());
1842+
static void common_chat_parse(common_chat_msg_parser & builder) {
1843+
LOG_DBG("Parsing input with format %s: %s\n", common_chat_format_name(builder.syntax().format).c_str(), builder.input().c_str());
18141844

1815-
switch (format) {
1845+
switch (builder.syntax().format) {
18161846
case COMMON_CHAT_FORMAT_CONTENT_ONLY:
18171847
common_chat_parse_content_only(builder);
18181848
break;
@@ -1847,15 +1877,15 @@ static void common_chat_parse(common_chat_msg_parser & builder, common_chat_form
18471877
common_chat_parse_command_r7b(builder);
18481878
break;
18491879
default:
1850-
throw std::runtime_error("Unsupported format: " + common_chat_format_name(format));
1880+
throw std::runtime_error(std::string("Unsupported format: ") + common_chat_format_name(builder.syntax().format));
18511881
}
18521882
builder.finish();
18531883
}
18541884

18551885
common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax) {
18561886
common_chat_msg_parser builder(input, is_partial, syntax);
18571887
try {
1858-
common_chat_parse(builder, syntax.format);
1888+
common_chat_parse(builder);
18591889
} catch (const common_chat_msg_partial_exception & ex) {
18601890
LOG_DBG("Partial parse: %s\n", ex.what());
18611891
if (!is_partial) {

‎common/chat.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ struct common_chat_syntax {
143143
// Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
144144
bool reasoning_in_content = false;
145145
bool thinking_forced_open = false;
146+
bool parse_tool_calls = true;
146147
};
147148

148149
// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid

‎tests/test-chat.cpp‎

Lines changed: 46 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -401,9 +401,12 @@ static common_chat_msg simple_assist_msg(const std::string & content, const std:
401401
}
402402
return msg;
403403
}
404-
const common_chat_msg message_assist = simple_assist_msg("Hello, world!\nWhat's up?");
405-
const common_chat_msg message_assist_empty = simple_assist_msg("");
406-
const common_chat_msg message_assist_thoughts_unparsed_deepseek = simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?");
404+
const common_chat_msg message_assist = simple_assist_msg("Hello, world!\nWhat's up?");
405+
const common_chat_msg message_assist_empty = simple_assist_msg("");
406+
const common_chat_msg message_assist_thoughts_unparsed_deepseek = simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?");
407+
const common_chat_msg message_assist_thoughts_unparsed_md = simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?\n```json\n{}```");
408+
const common_chat_msg message_assist_thoughts_unparsed_md_partial = simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?\n```json\n{}");
409+
407410
const common_chat_msg message_assist_thoughts_unparsed_r7b = simple_assist_msg("<|START_THINKING|>I'm\nthinking<|END_THINKING|>Hello, world!\nWhat's up?");
408411
const common_chat_msg message_assist_thoughts = simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking");
409412
const common_chat_msg message_assist_thoughts_unopened_unparsed = simple_assist_msg("I'm\nthinking</think>Hello, world!\nWhat's up?");
@@ -591,8 +594,6 @@ static void test_template_output_parsers() {
591594
{
592595
/* .format = */ COMMON_CHAT_FORMAT_COMMAND_R7B,
593596
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
594-
/* .reasoning_in_content = */ false,
595-
/* .thinking_forced_open = */ false,
596597
}));
597598
assert_msg_equals(message_assist_thoughts_unparsed_deepseek,
598599
common_chat_parse(
@@ -619,8 +620,6 @@ static void test_template_output_parsers() {
619620
{
620621
/* .format = */ COMMON_CHAT_FORMAT_COMMAND_R7B,
621622
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
622-
/* .reasoning_in_content = */ false,
623-
/* .thinking_forced_open = */ false,
624623
}));
625624
assert_msg_equals(message_assist_thoughts_call_idx,
626625
common_chat_parse(
@@ -632,8 +631,6 @@ static void test_template_output_parsers() {
632631
{
633632
/* .format = */ COMMON_CHAT_FORMAT_COMMAND_R7B,
634633
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
635-
/* .reasoning_in_content = */ false,
636-
/* .thinking_forced_open = */ false,
637634
}));
638635
assert_msg_equals(message_assist_thoughts_no_content,
639636
common_chat_parse(
@@ -644,8 +641,6 @@ static void test_template_output_parsers() {
644641
{
645642
/* .format = */ COMMON_CHAT_FORMAT_COMMAND_R7B,
646643
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
647-
/* .reasoning_in_content = */ false,
648-
/* .thinking_forced_open = */ false,
649644
}));
650645

651646
test_templates(tmpls.get(), end_tokens, message_assist_call_idx, tools,
@@ -675,6 +670,18 @@ static void test_template_output_parsers() {
675670

676671
// Generic tool calls doesn't generate / parse content-only messages symmetrically.
677672

673+
assert_equals(
674+
simple_assist_msg("{ \"tool_call\" : { \"name\" : \"t"),
675+
common_chat_parse(
676+
"{ \"tool_call\" : { \"name\" : \"t",
677+
/* is_partial= */ true,
678+
{
679+
/* .format = */ COMMON_CHAT_FORMAT_GENERIC,
680+
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
681+
/* .reasoning_in_content = */ false,
682+
/* .thinking_forced_open = */ true,
683+
/* .parse_tool_calls = */ false,
684+
}));
678685
assert_equals(
679686
message_assist_empty,
680687
common_chat_parse(
@@ -776,8 +783,6 @@ static void test_template_output_parsers() {
776783
{
777784
/* .format = */ COMMON_CHAT_FORMAT_HERMES_2_PRO,
778785
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
779-
/* .reasoning_in_content = */ false,
780-
/* .thinking_forced_open = */ false,
781786
}));
782787
assert_msg_equals(
783788
simple_assist_msg("Let's call something\n"),
@@ -788,8 +793,6 @@ static void test_template_output_parsers() {
788793
{
789794
/* .format = */ COMMON_CHAT_FORMAT_HERMES_2_PRO,
790795
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
791-
/* .reasoning_in_content = */ false,
792-
/* .thinking_forced_open = */ false,
793796
}));
794797
assert_msg_equals(message_assist_call_thoughts,
795798
common_chat_parse(
@@ -979,7 +982,34 @@ static void test_template_output_parsers() {
979982
{
980983
/* .format = */ COMMON_CHAT_FORMAT_HERMES_2_PRO,
981984
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
982-
/* .reasoning_in_content = */ false,
985+
}));
986+
assert_msg_equals(message_assist_thoughts,
987+
common_chat_parse(
988+
"<think>I'm\nthinking</think>Hello, world!\nWhat's up?",
989+
/* is_partial= */ true,
990+
{
991+
/* .format = */ COMMON_CHAT_FORMAT_HERMES_2_PRO,
992+
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
993+
}));
994+
assert_msg_equals(message_assist_thoughts_unparsed_md,
995+
common_chat_parse(
996+
"<think>I'm\nthinking</think>Hello, world!\nWhat's up?\n```json\n{}```",
997+
/* is_partial= */ false,
998+
{
999+
/* .format = */ COMMON_CHAT_FORMAT_HERMES_2_PRO,
1000+
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
1001+
/* .reasoning_in_content = */ true,
1002+
/* .thinking_forced_open = */ false,
1003+
/* .parse_tool_calls = */ false,
1004+
}));
1005+
assert_msg_equals(message_assist_thoughts_unparsed_md_partial,
1006+
common_chat_parse(
1007+
"<think>I'm\nthinking</think>Hello, world!\nWhat's up?\n```json\n{}```",
1008+
/* is_partial= */ true,
1009+
{
1010+
/* .format = */ COMMON_CHAT_FORMAT_HERMES_2_PRO,
1011+
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
1012+
/* .reasoning_in_content = */ true,
9831013
/* .thinking_forced_open = */ false,
9841014
}));
9851015
assert_msg_equals(message_assist_thoughts_unopened_unparsed,
@@ -989,8 +1019,6 @@ static void test_template_output_parsers() {
9891019
{
9901020
/* .format = */ COMMON_CHAT_FORMAT_HERMES_2_PRO,
9911021
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
992-
/* .reasoning_in_content = */ false,
993-
/* .thinking_forced_open = */ false,
9941022
}));
9951023
assert_msg_equals(message_assist_thoughts,
9961024
common_chat_parse(
@@ -1187,8 +1215,6 @@ static void test_template_output_parsers() {
11871215
{
11881216
/* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_R1,
11891217
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
1190-
/* .reasoning_in_content = */ false,
1191-
/* .thinking_forced_open = */ false,
11921218
}));
11931219
assert_msg_equals(message_assist_thoughts_unopened_unparsed,
11941220
common_chat_parse(
@@ -1197,8 +1223,6 @@ static void test_template_output_parsers() {
11971223
{
11981224
/* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_R1,
11991225
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
1200-
/* .reasoning_in_content = */ false,
1201-
/* .thinking_forced_open = */ false,
12021226
}));
12031227
assert_msg_equals(message_assist_thoughts,
12041228
common_chat_parse(
@@ -1252,8 +1276,6 @@ static void test_template_output_parsers() {
12521276
{
12531277
/* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_R1,
12541278
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
1255-
/* .reasoning_in_content = */ false,
1256-
/* .thinking_forced_open = */ false,
12571279
}));
12581280
assert_msg_equals(message_assist_thoughts,
12591281
common_chat_parse(
@@ -1295,8 +1317,6 @@ static void test_template_output_parsers() {
12951317
{
12961318
/* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_R1,
12971319
/* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK,
1298-
/* .reasoning_in_content = */ false,
1299-
/* .thinking_forced_open = */ false,
13001320
}));
13011321
test_templates(tmpls.get(), end_tokens, message_assist_call, tools,
13021322
"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>special_function\n"

‎tools/server/server.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,7 @@ struct server_task {
364364
params.oaicompat_chat_syntax.reasoning_format = params_base.reasoning_format;
365365
params.oaicompat_chat_syntax.reasoning_in_content = params.stream;
366366
params.oaicompat_chat_syntax.thinking_forced_open = json_value(data, "thinking_forced_open", false);
367+
params.oaicompat_chat_syntax.parse_tool_calls = json_value(data, "parse_tool_calls", false);
367368
}
368369

369370
{

‎tools/server/utils.hpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -732,8 +732,11 @@ static json oaicompat_chat_params_parse(
732732
inputs.use_jinja = opt.use_jinja;
733733
inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", false);
734734
inputs.reasoning_format = opt.reasoning_format;
735-
if (!inputs.tools.empty() && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && body.contains("grammar")) {
736-
throw std::runtime_error("Cannot use custom grammar constraints with tools.");
735+
if (!inputs.tools.empty() && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
736+
if (body.contains("grammar")) {
737+
throw std::runtime_error("Cannot use custom grammar constraints with tools.");
738+
}
739+
llama_params["parse_tool_calls"] = true;
737740
}
738741

739742
// if the assistant message appears at the end of list, we do not add end-of-turn token

0 commit comments

Comments
 (0)