Add handler and MCP message types

Author: bandoti

common/CMakeLists.txt

@@ -75,6 +75,10 @@ add_library(${TARGET} STATIC
     sampling.h
     speculative.cpp
     speculative.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/handler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/handler.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_messages.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/toolcall/mcp_messages.hpp
     )
 
 if (BUILD_SHARED_LIBS)

common/common.cpp

@@ -7,6 +7,9 @@
 
 #include "common.h"
 #include "log.h"
+// Change JSON_ASSERT from assert() to GGML_ASSERT:
+#define JSON_ASSERT GGML_ASSERT
+#include "json.hpp"
 #include "json-schema-to-grammar.h"
 #include "llama.h"
 #include "chat.hpp"
@@ -1769,46 +1772,6 @@ std::string common_detokenize(const struct llama_vocab * vocab, const std::vecto
 // Chat template utils
 //
 
-common_params_tools::common_params_tools(std::string tools, std::string choice) {
-    this->tools(tools);
-    this->choice(choice);
-}
-
-void common_params_tools::tools(std::string tools) {
-    if (tools.empty()) {
-        tools_.reset();
-        return;
-    }
-    try {
-        tools_ = std::make_shared<json>(json::parse(tools));
-        if (! tools_->is_array()) {
-            throw std::invalid_argument("tools must be a valid JSON array");
-        }
-
-    } catch (const json::exception & err) {
-        throw std::invalid_argument(err.what());
-    }
-}
-
-void common_params_tools::choice(std::string choice) {
-    try {
-        if (choice == "auto" || choice == "required" || choice == "none") {
-            tool_choice_ = std::move(choice);
-
-        } else {
-            auto choice_ptr = std::make_shared<json>(json::parse(choice));
-            tool_choice_ = choice_ptr;
-            if (! choice_ptr->is_object()) {
-                throw std::invalid_argument(
-                    "tool choice must be a valid JSON object, \"auto\", \"required\", or \"none\"");
-            }
-        }
-
-    } catch (const json::exception & err) {
-        throw std::invalid_argument(err.what());
-    }
-}
-
 bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {
     if (use_jinja) {
         try {
@@ -1830,7 +1793,7 @@ bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {
     return res >= 0;
 }
 
-static void copy_chat_params(const common_chat_params & src, common_chat_sampling_updater * update_sparams)
+static void copy_chat_params(const common_chat_params & src, toolcall::sampling_updater * update_sparams)
 {
     GGML_ASSERT(update_sparams && update_sparams->sparams && update_sparams->vocab);
 
@@ -1873,11 +1836,11 @@ std::string common_chat_apply_template(
         const std::vector<common_chat_msg> & msgs,
         bool add_ass,
         bool use_jinja,
-        const common_params_tools & tools,
-        common_chat_sampling_updater * update_sparams)
+        toolcall::handler::ptr handler,
+        toolcall::sampling_updater * update_sparams)
 {
     const auto & tmpl_selected =
-        tools.tools() && tmpl.template_tool_use ? *tmpl.template_tool_use : *tmpl.template_default;
+        handler != nullptr && tmpl.template_tool_use ? *tmpl.template_tool_use : *tmpl.template_default;
 
     if (use_jinja) {
         common_chat_inputs inputs;
@@ -1886,29 +1849,38 @@ std::string common_chat_apply_template(
         for (const auto & msg : msgs) {
             messages.push_back({{"role", msg.role}, {"content", msg.content}});
         }
+        inputs.messages = messages;
+        inputs.add_generation_prompt = add_ass;
 
-        if (tools.tools() != nullptr) {
-            inputs.tools = *tools.tools();
-        }
-
-        auto choice = tools.choice();
-        if (std::holds_alternative<std::string>(choice)) {
-            inputs.tool_choice = std::get<std::string>(choice);
+        if (handler != nullptr) {
+            auto choice = handler->tool_choice();
+            if (std::holds_alternative<std::string>(choice)) {
+                inputs.tool_choice = std::get<std::string>(choice);
 
-        } else {
-            auto choice_ptr = std::get<common_params_tools::json_ptr>(choice);
-            if (choice_ptr != nullptr) {
-                inputs.tool_choice = *choice_ptr;
+            } else {
+                auto choice_ptr = std::get<toolcall::json_ptr>(choice);
+                if (choice_ptr != nullptr) {
+                    inputs.tool_choice = *choice_ptr;
+                }
             }
+
+            inputs.tools = handler->tool_list();
         }
 
-        inputs.messages = messages;
-        inputs.add_generation_prompt = add_ass;
         auto chat_params = common_chat_params_init(tmpl_selected, inputs);
         if (update_sparams) {
             copy_chat_params(chat_params, update_sparams);
         }
-        return chat_params.prompt;
+
+        auto prompt = chat_params.prompt;
+        if (handler != nullptr) {
+            json response;
+            handler->call(prompt, response);
+            return response; // Caller will determine what to do based upon last_action
+
+        } else {
+            return prompt;
+        }
     }
 
     int alloc_size = 0;
@@ -1946,12 +1918,12 @@ std::string common_chat_format_single(
         const common_chat_msg & new_msg,
         bool add_ass,
         bool use_jinja,
-        const common_params_tools & tools,
-        common_chat_sampling_updater * update_sparams)
+        toolcall::handler::ptr handler,
+        toolcall::sampling_updater * update_sparams)
 {
     std::ostringstream ss;
     auto fmt_past_msg = past_msg.empty() ? ""
-        : common_chat_apply_template(tmpl, past_msg, false, use_jinja, tools, update_sparams);
+        : common_chat_apply_template(tmpl, past_msg, false, use_jinja, handler, update_sparams);
 
     std::vector<common_chat_msg> chat_new(past_msg);
     // if the past_msg ends with a newline, we must preserve it in the formatted version
@@ -1960,7 +1932,7 @@ std::string common_chat_format_single(
     };
     // format chat with new_msg
     chat_new.push_back(new_msg);
-    auto fmt_new_msg = common_chat_apply_template(tmpl, chat_new, add_ass, use_jinja, tools, update_sparams);
+    auto fmt_new_msg = common_chat_apply_template(tmpl, chat_new, add_ass, use_jinja, handler, update_sparams);
     // get the diff part
     ss << fmt_new_msg.substr(fmt_past_msg.size(), fmt_new_msg.size() - fmt_past_msg.size());
     return ss.str();

common/common.h

@@ -3,15 +3,12 @@
 #pragma once
 
 #include "llama-cpp.h"
-
+#include "toolcall/handler.hpp"
 #include <set>
 #include <string>
 #include <vector>
 #include <sstream>
 #include <variant>
-// Change JSON_ASSERT from assert() to GGML_ASSERT:
-#define JSON_ASSERT GGML_ASSERT
-#include "json.hpp"
 
 #ifdef _WIN32
 #define DIRECTORY_SEPARATOR '\\'
@@ -206,31 +203,6 @@ struct common_params_vocoder {
     bool use_guide_tokens = false; // enable guide tokens to improve TTS accuracy            // NOLINT
 };
 
-class common_params_tools {
-public:
-    using json = nlohmann::ordered_json;
-    using json_ptr = std::shared_ptr<json>;
-    using tool_choice_t = std::variant<std::string, json_ptr>;
-
-    common_params_tools(std::string tools  = "",
-                        std::string choice = "auto");
-
-    common_params_tools(const common_params_tools & other) = default;
-    common_params_tools(common_params_tools && other) noexcept = default;
-    common_params_tools & operator=(const common_params_tools & other) = default;
-    common_params_tools & operator=(common_params_tools && other) noexcept = default;
-
-    void tools(std::string tools);
-    const json * tools() const { return tools_.get(); }
-
-    void choice(std::string choice);
-    const tool_choice_t & choice() const { return tool_choice_; }
-
-private:
-    json_ptr tools_;
-    tool_choice_t tool_choice_;
-};
-
 struct common_params {
     int32_t n_predict             =    -1; // new tokens to predict
     int32_t n_ctx                 =  4096; // context size
@@ -375,7 +347,7 @@ struct common_params {
     std::string chat_template = "";                                                                         // NOLINT
     bool use_jinja = false;                                                                                 // NOLINT
     bool enable_chat_template = true;
-    common_params_tools jinja_tools;
+    toolcall::params jinja_tools;
 
     std::vector<std::string> api_keys;
 
@@ -671,10 +643,12 @@ struct common_chat_templates {
     std::unique_ptr<common_chat_template> template_tool_use;
 };
 
-struct common_chat_sampling_updater {
-    common_params_sampling * sparams;
-    const llama_vocab      * vocab;
-};
+namespace toolcall {
+    struct sampling_updater {
+	common_params_sampling * sparams;
+	const llama_vocab      * vocab;
+    };
+}
 
 // CPP wrapper for llama_chat_apply_template
 // If the built-in template is not supported, we default to chatml
@@ -684,8 +658,8 @@ std::string common_chat_apply_template(
         const std::vector<common_chat_msg> & chat,
         bool add_ass,
         bool use_jinja,
-        const common_params_tools & tools = common_params_tools(),
-        common_chat_sampling_updater * update_sparams = nullptr);
+        toolcall::handler::ptr handler = nullptr,
+        toolcall::sampling_updater * update_sparams = nullptr);
 
 // Format single message, while taking into account the position of that message in chat history
 std::string common_chat_format_single(
@@ -694,8 +668,8 @@ std::string common_chat_format_single(
         const common_chat_msg & new_msg,
         bool add_ass,
         bool use_jinja,
-        const common_params_tools & tools = common_params_tools(),
-        common_chat_sampling_updater * update_sparams = nullptr);
+        toolcall::handler::ptr handler = nullptr,
+        toolcall::sampling_updater * update_sparams = nullptr);
 
 // Returns an example of formatted chat
 std::string common_chat_format_example(

common/toolcall/handler.cpp

@@ -0,0 +1,110 @@
+
+#include "handler.hpp"
+
+using json = toolcall::json;
+
+toolcall::params::params(std::string tools, std::string choice) {
+    this->tools(tools);
+    this->choice(choice);
+}
+
+static bool starts_with(const std::string & str, const std::string & prefix) {
+    return str.size() >= prefix.size()
+        && str.compare(0, prefix.size(), prefix) == 0;
+}
+
+std::shared_ptr<toolcall::handler> toolcall::create_handler(const toolcall::params & params) {
+    std::shared_ptr<toolcall::handler> result;
+
+    auto tools = params.tools();
+    auto choice = params.choice();
+    bool has_uri = std::holds_alternative<std::string>(tools);
+    if (has_uri) {
+        auto tools_str = std::get<std::string>(tools);
+        result.reset(new toolcall::handler(std::make_unique<toolcall::mcp_impl>(tools_str, choice)));
+
+    } else {
+        auto tools_ptr = std::get<toolcall::json_ptr>(tools);
+        if (tools_ptr != nullptr) {
+            result.reset(new toolcall::handler(std::make_unique<toolcall::loopback_impl>(*tools_ptr, choice)));
+        }
+    }
+
+    return result;
+}
+
+void toolcall::params::tools(std::string tools) {
+    try {
+        if (tools.empty() || starts_with(tools, "mcp+http")) {
+            tools_ = std::move(tools);
+
+        } else {
+            tools_ = std::make_shared<json>(json::parse(tools));
+            auto tools_ptr = std::get<std::shared_ptr<json>>(tools_);
+            if (! tools_ptr->is_array()) {
+                throw std::invalid_argument("tools must be a valid JSON array");
+            }
+        }
+
+    } catch (const json::exception & err) {
+        throw std::invalid_argument(err.what());
+    }
+}
+
+void toolcall::params::choice(std::string choice) {
+    try {
+        if (choice == "auto" || choice == "required" || choice == "none") {
+            tool_choice_ = std::move(choice);
+
+        } else {
+            auto choice_ptr = std::make_shared<json>(json::parse(choice));
+            tool_choice_ = choice_ptr;
+            if (! choice_ptr->is_object()) {
+                throw std::invalid_argument(
+                    "tool choice must be a valid JSON object, \"auto\", \"required\", or \"none\"");
+            }
+        }
+
+    } catch (const json::exception & err) {
+        throw std::invalid_argument(err.what());
+    }
+}
+
+toolcall::params::operator bool() const  {
+    if (std::holds_alternative<std::string>(tools_)) {
+        return ! std::get<std::string>(tools_).empty();
+
+    } else {
+        return std::get<toolcall::json_ptr>(tools_) != nullptr;
+    }
+}
+
+json toolcall::handler::tool_list() {
+    return impl_->tool_list();
+}
+
+toolcall::action toolcall::handler::call(const json & request, json & response) {
+    last_action_ = impl_->call(request, response);
+    return last_action_;
+}
+
+const toolcall::tool_choice_t & toolcall::handler::tool_choice() const {
+    return impl_->tool_choice();
+}
+toolcall::action toolcall::handler::last_action() const {
+    return last_action_;
+}
+
+toolcall::mcp_impl::mcp_impl(std::string server_uri, tool_choice_t tool_choice)
+    : handler_impl(tool_choice)
+{
+    // TODO
+}
+
+json toolcall::mcp_impl::tool_list() {
+    return json{};// TODO
+}
+
+toolcall::action toolcall::mcp_impl::call(const json & request, json & response) {
+    return toolcall::ACCEPT; // TODO
+}