Split toolcall into separate library

Author: bandoti

CMakeLists.txt

@@ -82,6 +82,9 @@ option(LLAMA_BUILD_SERVER   "llama: build server example" ${LLAMA_STANDALONE})
 option(LLAMA_CURL "llama: use libcurl to download model from an URL" OFF)
 option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
 
+# Toolcall support - needs LLAMA_CURL support to connect with SSE endpoints
+option(LLAMA_TOOLCALL "llama: add toolcall support via Model Context Protocol" OFF)
+
 # Required for relocatable CMake package
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
@@ -160,6 +163,10 @@ add_subdirectory(src)
 # utils, programs, examples and tests
 #
 
+if (LLAMA_TOOLCALL)
+    add_subdirectory(toolcall)
+endif()
+
 if (LLAMA_BUILD_COMMON)
     add_subdirectory(common)
 endif()

common/CMakeLists.txt

@@ -75,15 +75,6 @@ add_library(${TARGET} STATIC
     sampling.h
     speculative.cpp
     speculative.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/params.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/params.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/handler.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/handler.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_messages.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_messages.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_transport.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_stdio_transport.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_stdio_transport.h
     )
 
 if (BUILD_SHARED_LIBS)
@@ -99,11 +90,6 @@ if (LLAMA_CURL)
     include_directories(${CURL_INCLUDE_DIRS})
     find_library(CURL_LIBRARY curl REQUIRED)
     set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} ${CURL_LIBRARY})
-
-    target_sources(${TARGET}
-        PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_sse_transport.cpp
-                ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_sse_transport.h)
-
 endif ()
 
 if (LLAMA_LLGUIDANCE)
@@ -153,3 +139,7 @@ endif ()
 target_include_directories(${TARGET} PUBLIC .)
 target_compile_features   (${TARGET} PUBLIC cxx_std_17)
 target_link_libraries     (${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama Threads::Threads)
+
+if (LLAMA_TOOLCALL)
+    target_link_libraries(${TARGET} PUBLIC toolcall)
+endif()

common/arg.cpp

@@ -2143,21 +2143,24 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         }
     ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CHAT_TEMPLATE_FILE"));
 
+#ifdef LLAMA_USE_TOOLCALL
     add_opt(common_arg(
         {"--tools"}, "JINJA_TOOLS",
-        "set to JSON array of tool definitions used for assistant function-calling (requires --jinja)",
+        "set to URI of a Model Context Protocol server, or "
+        "a JSON array containing tool definitions (requires --jinja)",
         [](common_params &params, const std::string & value) {
             params.jinja_tools.tools(value);
 
         }).set_examples({LLAMA_EXAMPLE_MAIN}));
 
     add_opt(common_arg(
         {"--tool-choice"}, "JINJA_TOOL_CHOICE",
-        "set to \"auto\", \"required\", \"none\" or a JSON object specifying a tool function (default: \"auto\")",
+        "set to \"auto\", \"required\", or \"none\" (default: \"auto\")",
         [](common_params &params, const std::string & value) {
             params.jinja_tools.choice(value);
 
         }).set_examples({LLAMA_EXAMPLE_MAIN}));
+#endif
 
     add_opt(common_arg(
         {"-sps", "--slot-prompt-similarity"}, "SIMILARITY",

common/common.h

@@ -3,7 +3,11 @@
 #pragma once
 
 #include "llama-cpp.h"
-#include "toolcall/params.h"
+
+#ifdef LLAMA_USE_TOOLCALL
+#    include "toolcall-params.h"
+#endif
+
 #include <set>
 #include <string>
 #include <vector>
@@ -353,7 +357,11 @@ struct common_params {
     std::string chat_template = "";                                                                         // NOLINT
     bool use_jinja = false;                                                                                 // NOLINT
     bool enable_chat_template = true;
+
+#ifdef LLAMA_USE_TOOLCALL
     toolcall::params jinja_tools;
+#endif
+
     common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
 
     std::vector<std::string> api_keys;

examples/main/main.cpp

@@ -5,7 +5,6 @@
 #include "sampling.h"
 #include "llama.h"
 #include "chat.h"
-#include "toolcall/handler.h"
 
 #include <cstdio>
 #include <cstring>
@@ -16,6 +15,10 @@
 #include <string>
 #include <vector>
 
+#ifdef LLAMA_USE_TOOLCALL
+#    include "toolcall-handler.h"
+#endif
+
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
 #include <signal.h>
 #include <unistd.h>
@@ -264,11 +267,14 @@ int main(int argc, char ** argv) {
 
     std::vector<llama_token> embd_inp;
 
-    auto toolcall_handler = toolcall::create_handler(params.jinja_tools);
+#ifdef LLAMA_USE_TOOLCALL
+    auto tc_handler = toolcall::create_handler(params.jinja_tools);
+#else
+    void * tc_handler = nullptr; // placeholder
+#endif
 
-    auto chat_add_and_format = [&chat_msgs, &chat_templates, &sparams, vocab](
-        const std::string & role, const std::string & content,
-        toolcall::handler::ptr handler = nullptr)
+    auto chat_add_and_format = [&chat_msgs, &chat_templates, &sparams, vocab, tc_handler](
+        const std::string & role, const std::string & content, bool use_toolcalls = false)
     {
         bool add_ass = (role == "user");
 
@@ -277,11 +283,13 @@ int main(int argc, char ** argv) {
         new_msg.content = content;
 
         common_chat_templates_inputs cinputs;
-        if (handler != nullptr) {
-            cinputs.tool_choice = common_chat_tool_choice_parse_oaicompat(handler->tool_choice());
-            cinputs.tools = common_chat_tools_parse_oaicompat(handler->tool_list());
-        }
 
+#ifdef LLAMA_USE_TOOLCALL
+        if (tc_handler != nullptr && use_toolcalls) {
+            cinputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tc_handler->tool_choice());
+            cinputs.tools = common_chat_tools_parse_oaicompat(tc_handler->tool_list());
+        }
+#endif
         common_chat_params cparams;
         auto formatted =
             common_chat_format_single(chat_templates.get(), chat_msgs, new_msg, add_ass, g_params->use_jinja,
@@ -290,23 +298,24 @@ int main(int argc, char ** argv) {
         chat_msgs.push_back(new_msg);
         LOG_DBG("formatted: '%s'\n", formatted.c_str());
 
+#ifdef LLAMA_USE_TOOLCALL
         if (g_params->use_jinja) {
             common_chat_grammar_to_sampler(&cparams, vocab, &sparams);
-            if (handler != nullptr) {
+            if (tc_handler != nullptr) {
                 std::string response;
-                handler->call(formatted, response);
+                tc_handler->call(formatted, response);
                 return std::string(response);
             }
         }
+#endif
         return formatted;
     };
 
     {
-        std::string system_prompt (params.prompt.empty() ? DEFAULT_SYSTEM_MESSAGE : params.prompt);
-        bool use_conversation_prompt (params.conversation_mode && params.enable_chat_template);
+        std::string system_prompt = params.prompt.empty() ? DEFAULT_SYSTEM_MESSAGE : params.prompt;
+        bool use_conversation_prompt = params.conversation_mode && params.enable_chat_template;
         auto prompt = use_conversation_prompt ?
-            chat_add_and_format("system", system_prompt, toolcall_handler)
-            : params.prompt;
+            chat_add_and_format("system", system_prompt, true) : params.prompt;
 
         if (params.interactive_first || !params.prompt.empty() || session_tokens.empty()) {
             LOG_DBG("tokenize the prompt\n");
@@ -814,23 +823,22 @@ int main(int argc, char ** argv) {
                     }
 
                     if (params.enable_chat_template) {
-                        auto output = chat_add_and_format("assistant", assistant_ss.str(), toolcall_handler);
-                        if (toolcall_handler != nullptr) {
-                            auto action = toolcall_handler->last_action();
-                            if (action == toolcall::ACCEPT) {
-                                LOG_DBG("tokenizing toolcall response");
-                                auto response = common_tokenize(ctx, output, false, true);
-                                embd_inp.insert(embd_inp.end(), response.begin(), response.end());
-
-                            } else {
-                                is_interacting = true;
-                                LOG("\n");
-                            }
-
-                        } else {
+#ifdef LLAMA_USE_TOOLCALL
+                        auto output = chat_add_and_format("assistant", assistant_ss.str(), true);
+                        if (tc_handler == nullptr || tc_handler->last_action() != toolcall::ACCEPT) {
                             is_interacting = true;
                             LOG("\n");
+
+                        } else {
+                            LOG_DBG("tokenizing toolcall response");
+                            auto response = common_tokenize(ctx, output, false, true);
+                            embd_inp.insert(embd_inp.end(), response.begin(), response.end());
                         }
+#else
+                        chat_add_and_format("assistant", assistant_ss.str());
+                        is_interacting = true;
+                        LOG("\n");
+#endif
                     }
                 }
             }

toolcall/CMakeLists.txt

@@ -0,0 +1,36 @@
+
+set(TARGET toolcall)
+
+set(SOURCES
+    handler.cpp
+    mcp_messages.cpp
+    mcp_stdio_transport.cpp
+    params.cpp
+)
+
+set(HEADERS
+    toolcall-params.h
+    toolcall-handler.h
+    mcp_transport.h
+    mcp_messages.h
+    mcp_stdio_transport.h
+)
+
+add_library(${TARGET} STATIC ${SOURCES} ${HEADERS})
+
+target_include_directories(${TARGET}  # Right now only for "json.hpp"
+    PRIVATE $<TARGET_PROPERTY:common,INTERFACE_INCLUDE_DIRECTORIES>)
+
+if (LLAMA_CURL)
+    find_package(CURL REQUIRED)
+    target_compile_definitions(${TARGET} PUBLIC LLAMA_USE_CURL)
+    include_directories(${CURL_INCLUDE_DIRS})
+    find_library(CURL_LIBRARY curl REQUIRED)
+
+    target_link_libraries(${TARGET} PRIVATE ${CURL_LIBRARY})
+    target_sources(${TARGET} PRIVATE mcp_sse_transport.cpp mcp_sse_transport.h)
+
+endif()
+
+target_compile_definitions(${TARGET} INTERFACE LLAMA_USE_TOOLCALL)
+target_include_directories(${TARGET} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)

common/toolcall/handler.cpp renamed to toolcall/handler.cpp

@@ -1,6 +1,6 @@
 
 #include <json.hpp>
-#include "handler.h"
+#include "toolcall-handler.h"
 
 #ifdef LLAMA_USE_CURL
 #    include "mcp_sse_transport.h"