tool-call: test/fix functionary-medium-v3.1's template (can "look" like llama3.1 template)

Author: ochafik

common/tool-call.cpp

@@ -191,6 +191,16 @@ static llama_tool_calls parse_functionary_tool_calls(const std::string& input, c
 }
 
 static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
+    // This version of Functionary still supports the llama 3.1 tool call format for the python tool.
+    static std::regex python_tag_regex(R"(<\|python_tag\|>([\s\S\n]*)$)");
+    std::smatch match;
+    if (std::regex_search(input, match, python_tag_regex)) {
+        return {
+            match.prefix().str(), {
+                {"ipython", (json {{"code", match[1].str()}}).dump()},
+            }
+        };
+    }
     static std::regex function_regex(R"(<function=(\w+)>)");
     static std::regex close_regex(R"(</function>)");
     return parse_functionary_tool_calls(input, function_regex, close_regex);
@@ -205,12 +215,12 @@ static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input
 llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) {
     if (needs_hermes_pro_tool_call(chat_template)) {
         return parse_hermes_tool_calls(input);
-    } else if (needs_llama_3_1_tool_call(chat_template)) {
-        return parse_llama_3_1_tool_calls(tools, input);
     } else if (needs_functionary_v3_tool_call(chat_template)) {
         return parse_functionary_v3_tool_calls(input);
     } else if (needs_functionary_v3_llama_3_1_tool_call(chat_template)) {
         return parse_functionary_v3_llama_3_1_tool_calls(input);
+    } else if (needs_llama_3_1_tool_call(chat_template)) {
+        return parse_llama_3_1_tool_calls(tools, input);
     } else {
         throw std::runtime_error("Unsupported chat template for tool calls");
     }

examples/server/tests/features/tool_call.feature

@@ -12,29 +12,31 @@ Feature: llama.cpp server
     And   8192 KV cache size
     And   32 as batch size
     And   2 slots
-    And   64 server max tokens to predict
     And   prometheus compatible metrics exposed
     And   jinja templates are enabled
 
-  @wip
+
   Scenario Outline: OAI Compatibility w/ required tool
     Given a chat template file ../../../tests/chat/templates/<template_name>.jinja
     And   the server is starting
     And   the server is healthy
     And   a model test
-    And   <n> max tokens to predict
+    And   <n_predict> max tokens to predict
     And   a user prompt write a hello world in python
     And   a tool choice <tool_choice>
     And   tools <tools>
     And   an OAI compatible chat completions request with no api error
     Then  tool <tool_name> is called with arguments <tool_arguments>
 
     Examples: Prompts
-      | template_name                         | n   | tool_name | tool_arguments       | tool_choice | tools |
-      | meta-llama-Meta-Llama-3.1-8B-Instruct | 64  | test      | {}                   | required    | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
-      | meta-llama-Meta-Llama-3.1-8B-Instruct | 16  | ipython   | {"code": "it and "}  | required    | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
-      | meetkai-functionary-medium-v3.2       | 64  | test      | {}                   | required    | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
-      | meetkai-functionary-medium-v3.2       | 64  | ipython   | {"code": "Yes,"}     | required    | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
+      | template_name                         | n_predict | tool_name | tool_arguments       | tool_choice | tools |
+      | meetkai-functionary-medium-v3.1       | 128       | test      | {}                   | required    | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}]                                                                       |
+      | meetkai-functionary-medium-v3.1       | 128       | ipython   | {"code": "Yes, you can."}     | required    | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
+      | meetkai-functionary-medium-v3.2       | 128       | test      | {}                   | required    | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}]                                                                       |
+      | meetkai-functionary-medium-v3.2       | 128       | ipython   | {"code": "Yes,"}     | required    | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
+      | meta-llama-Meta-Llama-3.1-8B-Instruct | 64        | test      | {}                   | required    | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}]                                                                       |
+      | meta-llama-Meta-Llama-3.1-8B-Instruct | 16        | ipython   | {"code": "it and "}  | required    | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
+
 
   Scenario: OAI Compatibility w/ no tool
     Given a chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja

tests/chat/goldens/meetkai-functionary-medium-v3.1-simple.txt

@@ -0,0 +1,11 @@
+<|startoftext|><|start_header_id|>system<|end_header_id|>
+
+
+Cutting Knowledge Date: December 2023
+
+<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+

tests/chat/goldens/meetkai-functionary-medium-v3.1-system.txt

@@ -0,0 +1,13 @@
+<|startoftext|><|start_header_id|>system<|end_header_id|>
+
+
+Cutting Knowledge Date: December 2023
+
+<|eot_id|><|start_header_id|>system<|end_header_id|>
+
+You only tell the truth.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+

tests/chat/goldens/meetkai-functionary-medium-v3.1-tool_use.txt

@@ -0,0 +1 @@
+ERROR: can only concatenate str (not "dict") to str

tests/chat/templates/meetkai-functionary-medium-v3.1.jinja

@@ -0,0 +1,58 @@
+{# version=v3-llama3.1 #}{%- if not tools is defined -%}
+    {%- set tools = none -%}
+{%- endif -%}
+
+{%- set has_code_interpreter = tools | selectattr("type", "equalto", "code_interpreter") | list | length > 0 -%}
+{%- if has_code_interpreter -%}
+    {%- set tools = tools | rejectattr("type", "equalto", "code_interpreter") | list -%}
+{%- endif -%}
+
+{#- System message + builtin tools #}
+{{- bos_token + "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if has_code_interpreter %}
+    {{- "Environment: ipython\n\n" }}
+{%- else -%}
+    {{ "\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n\n" }}
+{%- if tools %}
+    {{- "\nYou have access to the following functions:\n\n" }}
+    {%- for t in tools %}
+        {%- if "type" in t -%}
+            {{ "Use the function '"|safe + t["function"]["name"] + "' to '"|safe + t["function"]["description"] + "'\n"|safe + t["function"] | tojson() }}
+        {%- else -%}
+            {{ "Use the function '"|safe + t["name"] + "' to '"|safe + t["description"] + "'\n"|safe + t | tojson() }}
+        {%- endif -%}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- '\nThink very carefully before calling functions.\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => `<function`\nparameters => a JSON dict with the function argument name as key and function argument value as value.\nend_tag => `</function>`\n\nHere is an example,\n<function=example_function_name>{"example_name": "example_value"}</function>\n\nReminder:\n- If looking for real time information use relevant functions before falling back to brave_search\n- Function calls MUST follow the specified format, start with <function= and end with </function>\n- Required parameters MUST be specified\n- Only call one function at a time\n- Put the entire function call reply on one line\n\n' -}}
+{%- endif %}
+{{- "<|eot_id|>" -}}
+
+{%- for message in messages -%}
+    {%- if message['role'] == 'user' or message['role'] == 'system' -%}
+        {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }}
+    {%- elif message['role'] == 'tool' -%}
+        {{ '<|start_header_id|>ipython<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }}
+    {%- else -%}
+        {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'}}
+        {%- if message['content'] -%}
+            {{ message['content'] }}
+        {%- endif -%}
+        {%- if 'tool_calls' in message and message['tool_calls'] -%}
+            {%- for tool_call in message['tool_calls'] -%}
+                {%- if tool_call["function"]["name"] == "python" -%}
+                    {{ '<|python_tag|>' + tool_call['function']['arguments'] }}
+                {%- else -%}
+                    {{ '<function=' + tool_call['function']['name'] + '>' + tool_call['function']['arguments'] + '</function>' }}
+                {%- endif -%}
+            {%- endfor -%}
+            {{ '<|eom_id|>' }}
+        {%- else -%}
+            {{ '<|eot_id|>' }}
+        {%- endif -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif -%}

tests/test-tool-call.cpp

@@ -116,6 +116,15 @@ int main() {
           }}
         },
       });
+    test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl,
+      "<function=test>{ } </function> ",
+      " ",
+      json {{
+        {"function", {
+          {"name", "test"},
+          {"arguments", "{}"}
+        }}
+      }});
 
     std::string llama_3_1_like_tmpl = "Llama 3.1 template should have <|start_header_id|> and <|python_tag|> inside it";
     test_parse_tool_call(tools, llama_3_1_like_tmpl,

tests/update_jinja_goldens.py

@@ -26,13 +26,15 @@
 import re
 # import requests
 
+logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 model_ids = [
     "NousResearch/Hermes-3-Llama-3.1-70B",
     "NousResearch/Hermes-2-Pro-Llama-3-8B",
     "NousResearch/Hermes-2-Pro-Mistral-7B",
     "meetkai/functionary-medium-v3.2",
+    "meetkai/functionary-medium-v3.1",
     "Qwen/Qwen2-7B-Instruct",
     "Qwen/Qwen2-VL-7B-Instruct",
     "Qwen/Qwen2.5-7B-Instruct",