Refactor tool/call response

Author: bandoti

examples/main/main.cpp

@@ -106,10 +106,26 @@ class chat_formatter {
 #endif
 
     std::string operator () (const std::string & role, const std::string & content, [[maybe_unused]] bool use_toolcalls = false) {
+
         common_chat_msg new_msg;
         new_msg.role = role;
         new_msg.content = content;
 
+#ifdef LLAMA_USE_TOOLCALL
+        if (params_.use_jinja && use_toolcalls) {
+            if (tc_handler_ != nullptr) {
+                if (nlohmann::json::accept(content)) { // Need a better way to know this is for a toolcall
+                    toolcall::result_set res = tc_handler_->call(content);
+                    std::string new_content;
+                    for (const auto & r : res) {
+                        new_content += (r.data + "\n");
+                    }
+                    new_msg.content = new_content; // TODO: this is not wiring correctly into the prompt
+                }
+            }
+        }
+#endif
+
         common_chat_params cparams;
         common_chat_templates_inputs cinputs;
 #ifdef LLAMA_USE_TOOLCALL
@@ -126,21 +142,8 @@ class chat_formatter {
         chat_msgs_.push_back(new_msg);
         LOG_DBG("formatted: '%s'\n", formatted.c_str());
 
-#ifdef LLAMA_USE_TOOLCALL
-        if (params_.use_jinja && use_toolcalls) {
-            common_chat_grammar_to_sampler(&cparams, vocab_, &params_.sampling);
-            if (tc_handler_ != nullptr) {
-                if (nlohmann::json::accept(formatted)) {   // May need a better way to ensure
-                    std::string response;                  // this is intended for a tool-call.
-                    tc_handler_->call(formatted, response);
-                    return std::string(response);
+        common_chat_grammar_to_sampler(&cparams, vocab_, &params_.sampling);
 
-                } else {
-                    return formatted;
-                }
-            }
-        }
-#endif
         return formatted;
     }
 
@@ -855,22 +858,9 @@ int main(int argc, char ** argv) {
                     }
 
                     if (params.enable_chat_template) {
-#ifdef LLAMA_USE_TOOLCALL
-                        auto output = chat_add_and_format("assistant", assistant_ss.str(), true);
-                        if (tc_handler == nullptr || tc_handler->last_action() != toolcall::ACCEPT) {
-                            is_interacting = true;
-                            LOG("\n");
-
-                        } else {
-                            LOG_DBG("tokenizing toolcall response");
-                            auto response = common_tokenize(ctx, output, false, true);
-                            embd_inp.insert(embd_inp.end(), response.begin(), response.end());
-                        }
-#else
-                        chat_add_and_format("assistant", assistant_ss.str());
+                        chat_add_and_format("assistant", assistant_ss.str(), true);
                         is_interacting = true;
                         LOG("\n");
-#endif
                     }
                 }
             }

toolcall/handler.cpp

@@ -5,7 +5,7 @@
 #include <stdexcept>
 
 #ifdef LLAMA_USE_CURL
-#  include "mcp_sse_transport.h"
+#    include "mcp_sse_transport.h"
 #endif
 
 #include "mcp_stdio_transport.h"
@@ -39,19 +39,14 @@ bool toolcall::handler::tool_list_dirty() const {
     return impl_->tool_list_dirty();
 }
 
-toolcall::action toolcall::handler::call(const std::string & request, std::string & response) {
-    last_action_ = impl_->call(request, response);
-    return last_action_;
+toolcall::result_set toolcall::handler::call(const std::string & request) {
+    return impl_->call(request);
 }
 
 const std::string & toolcall::handler::tool_choice() const {
     return impl_->tool_choice();
 }
 
-toolcall::action toolcall::handler::last_action() const {
-    return last_action_;
-}
-
 void toolcall::handler::initialize() {
     impl_->initialize();
 }
@@ -194,26 +189,32 @@ static mcp::tools_call_request tools_call_request_from_local_json(nlohmann::json
     return mcp::tools_call_request(id, j["name"], args);
 }
 
-static std::string tools_call_response_to_local_json(const mcp::tools_call_response & resp) {
-    return resp.toJson().dump(-1); // The AI will figure it out?
+static toolcall::result_set tools_call_response_to_result(const mcp::tools_call_response & resp) {
+    toolcall::result_set result;
+    for (const auto & res : resp.tool_result()) {
+        result.push_back(toolcall::result{
+                res.type, res.value, res.mime_type.value_or("text/plain"), res.uri, resp.tool_error()
+            });
+    }
+    return std::move(result);
 }
 
-toolcall::action toolcall::mcp_impl::call(const std::string & request, std::string & response) {
+toolcall::result_set toolcall::mcp_impl::call(const std::string & request) {
     using on_response = toolcall::callback<mcp::tools_call_response>;
 
     if (transport_ == nullptr) {
-        return toolcall::DEFER;
+        return toolcall::result_set();
     }
     std::unique_lock<std::mutex> lock(tools_mutex_);
 
-    response.clear();
+    toolcall::result_set response;
     on_response set_response = [this, &response] (const mcp::tools_call_response & resp) {
         std::unique_lock<std::mutex> lock(tools_mutex_);
-        response = tools_call_response_to_local_json(resp);
+        response = tools_call_response_to_result(resp);
         tools_populating_.notify_one();
     };
     transport_->send(tools_call_request_from_local_json(next_id_++, request), set_response);
     tools_populating_.wait_for(lock, std::chrono::seconds(15), [&response] { return ! response.empty(); });
 
-    return toolcall::ACCEPT;
+    return response;
 }

toolcall/toolcall-handler.h

@@ -10,32 +10,34 @@
 
 namespace toolcall
 {
-    enum action {
-        ACCEPT,
-        PENDING,
-        DEFER
+    struct result {
+        std::string type;
+        std::string data;
+        std::string mime_type;
+        std::optional<std::string> uri;
+        bool error;
     };
 
+    using result_set = std::vector<result>;
+
     class handler_impl;
     class handler {
     public:
         using ptr = std::shared_ptr<handler>;
 
         handler(std::unique_ptr<handler_impl> impl) : impl_(std::move(impl)) {}
 
-        action call(const std::string & request, std::string & response);
+        result_set call(const std::string & request);
 
         std::string tool_list();
         bool tool_list_dirty() const;
 
         const std::string & tool_choice() const;
-        action last_action() const;
 
         void initialize();
 
     private:
         std::unique_ptr<handler_impl> impl_;
-        action last_action_;
     };
 
     std::shared_ptr<toolcall::handler> create_handler(const toolcall::params & params);
@@ -53,7 +55,7 @@ namespace toolcall
             return tool_list_dirty_;
         }
 
-        virtual action call(const std::string & request, std::string & response) = 0;
+        virtual result_set call(const std::string & request) = 0;
 
         const std::string & tool_choice() const { return tool_choice_; }
 
@@ -74,9 +76,10 @@ namespace toolcall
             return tools_;
         }
 
-        virtual action call(const std::string & request, std::string & response) override {
-            response = request;
-            return toolcall::DEFER;
+        virtual result_set call(const std::string & request) override {
+            return {
+		{"text", request, "text/plain", std::nullopt, false}
+	    };
         }
 
     private:
@@ -90,7 +93,7 @@ namespace toolcall
         mcp_impl(std::vector<std::string> argv, std::string tool_choice);
 
         virtual std::string tool_list() override;
-        virtual action call(const std::string & request, std::string & response) override;
+        virtual result_set call(const std::string & request) override;
 
         virtual void initialize() override;