feat: PoC for sending multiple messages, ref #65

Author: pminev

ac-local-plugin/code/LocalLlama.cpp

@@ -38,6 +38,9 @@
 #include "aclp-llama-version.h"
 #include "aclp-llama-interface.hpp"
 
+// TODO: remove this include
+#include <iostream>
+
 namespace ac::local {
 
 namespace {
@@ -81,6 +84,15 @@ class ChatSession {
         m_instance.stopSession();
     }
 
+    xec::coro<void> sendMessages(Schema::OpSendMessages::Params& params) {
+        auto& messages = params.messages.value();
+        for (size_t i = 0; i < messages.size(); ++i) {
+            std::cout << messages[i].role.value() << ": " << messages[i].content.value() << "\n";
+        }
+
+        co_await m_io.push(Frame_from(schema::SimpleOpReturn<Schema::OpSendMessages>{}, {}));
+    }
+
     xec::coro<void> pushPrompt(Schema::OpAddChatPrompt::Params& params) {
         auto& prompt = params.prompt.value();
 
@@ -441,6 +453,8 @@ struct LocalLlama {
                     co_await chatSession.getResponse(*iparams, false);
                 } else if (auto iparams = Frame_optTo(schema::OpParams<Schema::OpStreamChatResponse>{}, *f)) {
                     co_await chatSession.getResponse(*iparams, true);
+                } else if (auto iparams = Frame_optTo(schema::OpParams<Schema::OpSendMessages>{}, *f)) {
+                    co_await chatSession.sendMessages(*iparams);
                 } else {
                     err = unknownOpError(*f);
                 }

ac-local-plugin/example/ep-chat.cpp

@@ -31,7 +31,8 @@ int main() try {
     std::cout << "Initial state: " << sid << '\n';
 
     for (auto x : llama.stream<schema::StateLlama::OpLoadModel>({
-        .ggufPath = AC_TEST_DATA_LLAMA_DIR "/gpt2-117m-q6_k.gguf"
+        // .ggufPath = AC_TEST_DATA_LLAMA_DIR "/gpt2-117m-q6_k.gguf"
+        .ggufPath = AC_TEST_DATA_LLAMA_DIR "/../../../tmp/Meta-Llama-3.1-8B-Instruct-Q6_K.gguf"
     })) {
         std::cout << "Model loaded: " << x.tag.value() << " " << x.progress.value() << '\n';
     }
@@ -42,11 +43,23 @@ int main() try {
     sid = llama.call<schema::StateModelLoaded::OpStartInstance>({
         .instanceType = "chat",
         .setup = "A chat between a human user and a helpful AI assistant.",
-        .roleUser = roleUser,
-        .roleAssistant = roleAssistant
+        // .roleUser = roleUser,
+        // .roleAssistant = roleAssistant
     });
     std::cout << "Instance started: " << sid << '\n';
 
+    std::vector<schema::Message> initMessages = {
+        {roleUser, "I need assistance for API design"},
+        {roleAssistant, "What aspect of API design are you looking for help with? Do you have a specific problem or question in mind?"},
+        {roleUser, "It's a C++ implementation of a class"},
+    };
+
+    llama.call<schema::StateChatInstance::OpSendMessages>({
+        .messages = initMessages
+    });
+
+    std::vector<schema::Message> messages;
+
     while (true) {
         std::cout << roleUser <<": ";
         std::string user;
@@ -55,19 +68,26 @@ int main() try {
         }
         if (user == "/quit") break;
         user = ' ' + user;
+        messages.push_back({roleUser, user});
+
         llama.call<schema::StateChatInstance::OpAddChatPrompt>({
             .prompt = user
         });
 
+        std::string text;
         std::cout << roleAssistant << ": ";
         constexpr bool streamChat = false;
         if (streamChat) {
             for(auto t: llama.stream<schema::StateChatInstance::OpStreamChatResponse>({})) {
+                text += t;
                 std::cout << t << std::flush;
             }
         } else {
-            std::cout << llama.call<schema::StateChatInstance::OpGetChatResponse>({}).response.value() << std::flush;
+            auto res = llama.call<schema::StateChatInstance::OpGetChatResponse>({});
+            text += res.response.value();
+            std::cout << res.response.value() << std::flush;
         }
+        messages.push_back({roleUser, text});
         std::cout << "\n";
     }
 

ac-local-plugin/schema/ac/schema/LlamaCpp.hpp

@@ -19,6 +19,20 @@ struct StreamToken {
     using Type = std::string;
 };
 
+struct Message {
+    static constexpr auto id = "chat-message";
+    static constexpr auto desc = "Chat message";
+
+    Field<std::string> role;
+    Field<std::string> content;
+
+    template <typename Visitor>
+    void visitFields(Visitor& v) {
+        v(role, "role", "Messages to add to the chat session");
+        v(content, "content", "Messages to add to the chat session");
+    }
+};
+
 struct StateLlama {
     static constexpr auto id = "llama.cpp";
     static constexpr auto desc = "Initial state";
@@ -68,6 +82,7 @@ struct StateModelLoaded {
         Field<std::vector<std::string>> ctrlVectorPaths = Default();
 
         Field<std::string> setup = Default();
+        Field<std::string> chatTemplate = Default();
         Field<std::string> roleUser = Default("User");
         Field<std::string> roleAssistant = Default("Assistant");
 
@@ -93,7 +108,6 @@ struct StateModelLoaded {
     };
 
     using Ops = std::tuple<OpStartInstance>;
-
 };
 
 struct StateGeneralInstance {
@@ -206,6 +220,22 @@ struct StateChatInstance {
     static constexpr auto id = "chat-instance";
     static constexpr auto desc = "Chat state";
 
+    struct OpSendMessages {
+        static inline constexpr std::string_view id = "send-messages";
+        static inline constexpr std::string_view desc = "Send messages to the chat session";
+
+        struct Params {
+            Field<std::vector<Message>> messages;
+
+            template <typename Visitor>
+            void visitFields(Visitor& v) {
+                v(messages, "messages", "Messages to add to the chat session");
+            }
+        };
+
+        using Return = nullptr_t;
+    };
+
     struct OpAddChatPrompt {
         static inline constexpr std::string_view id = "add-chat-prompt";
         static inline constexpr std::string_view desc = "Add a prompt to the chat session as a user";