Merge pull request #222 from iceljc/features/add-llm-completion-log

add llm completion log

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs

@@ -0,0 +1,12 @@
+namespace BotSharp.Abstraction.Conversations.Models;
+
+public class LlmCompletionLog
+{
+    public string Id { get; set; } = string.Empty;
+    public string ConversationId { get; set; } = string.Empty;
+    public string MessageId { get; set; } = string.Empty;
+    public string AgentId { get; set; } = string.Empty;
+    public string Prompt { get; set; } = string.Empty;
+    public string? Response { get; set; }
+    public DateTime CreateDateTime { get; set; } = DateTime.UtcNow;
+}

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -41,4 +41,8 @@ public interface IBotSharpRepository
     void AddExectionLogs(string conversationId, List<string> logs);
     List<string> GetExectionLogs(string conversationId);
     #endregion
+
+    #region LLM Completion Log
+    void SaveLlmCompletionLog(LlmCompletionLog log);
+    #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -197,4 +197,11 @@ public void CreateUser(User user)
         throw new NotImplementedException();
     }
     #endregion
+
+    #region LLM Completion Log
+    public void SaveLlmCompletionLog(LlmCompletionLog log)
+    {
+        throw new NotImplementedException();
+    }
+    #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs

@@ -7,6 +7,7 @@
 using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Utilities;
+using BotSharp.Abstraction.Conversations.Models;
 
 namespace BotSharp.Core.Repository;
 
@@ -559,7 +560,7 @@ public string GetAgentTemplate(string agentId, string templateName)
         foreach (var file in Directory.GetFiles(dir))
         {
             var fileName = file.Split(Path.DirectorySeparatorChar).Last();
-            var splits = fileName.ToLower().Split('.');
+            var splits = ParseFileNameByPath(fileName.ToLower());
             var name = splits[0];
             var extension = splits[1];
             if (name.IsEqualTo(templateName) && extension.IsEqualTo(_agentSettings.TemplateFormat))
@@ -617,10 +618,10 @@ public bool DeleteConversation(string conversationId)
     {
         if (string.IsNullOrEmpty(conversationId)) return false;
 
-        var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId);
-        if (!Directory.Exists(dir)) return false;
+        var convDir = FindConversationDirectory(conversationId);
+        if (string.IsNullOrEmpty(convDir)) return false;
 
-        Directory.Delete(dir, true);
+        Directory.Delete(convDir, true);
         return true;
     }
 
@@ -842,6 +843,25 @@ public void CreateUser(User user)
     }
     #endregion
 
+    #region LLM Completion Log
+    public void SaveLlmCompletionLog(LlmCompletionLog log)
+    {
+        var convDir = FindConversationDirectory(log.ConversationId);
+        if (!Directory.Exists(convDir)) return;
+
+        var logDir = Path.Combine(convDir, "llm_prompt_log");
+        if (!Directory.Exists(logDir))
+        {
+            Directory.CreateDirectory(logDir);
+        }
+
+        var index = GetLlmCompletionLogIndex(logDir, log.MessageId);
+        var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log");
+        File.WriteAllText(file, JsonSerializer.Serialize(log, _options));
+    }
+    #endregion
+
+
     #region Private methods
     private string GetAgentDataDir(string agentId)
     {
@@ -934,22 +954,10 @@ private List<AgentResponse> FetchResponses(string fileDir)
 
     private string? FindConversationDirectory(string conversationId)
     {
-        var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
-
-        foreach (var d in Directory.GetDirectories(dir))
-        {
-            var path = Path.Combine(d, "conversation.json");
-            if (!File.Exists(path)) continue;
-
-            var json = File.ReadAllText(path);
-            var conv = JsonSerializer.Deserialize<Conversation>(json, _options);
-            if (conv != null && conv.Id == conversationId)
-            {
-                return d;
-            }
-        }
+        var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId);
+        if (!Directory.Exists(dir)) return null;
 
-        return null;
+        return dir;
     }
 
     private List<DialogElement> CollectDialogElements(string dialogDir)
@@ -1000,5 +1008,30 @@ private List<StateKeyValue> CollectConversationStates(string stateDir)
         }
         return states;
     }
+
+    private int GetLlmCompletionLogIndex(string logDir, string id)
+    {
+        var files = Directory.GetFiles(logDir);
+        if (files.IsNullOrEmpty())
+            return 0;
+
+        var logIndexes = files.Where(file =>
+        {
+            var fileName = ParseFileNameByPath(file);
+            return fileName[0].IsEqualTo(id);
+        }).Select(file =>
+        {
+            var fileName = ParseFileNameByPath(file);
+            return int.Parse(fileName[1]);
+        }).ToList();
+
+        return logIndexes.IsNullOrEmpty() ? 0 : logIndexes.Max() + 1;
+    }
+
+    private string[] ParseFileNameByPath(string path, string separator = ".")
+    {
+        var name = path.Split(Path.DirectorySeparatorChar).Last();
+        return name.Split(separator);
+    }
     #endregion
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogCollection.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Plugin.MongoStorage.Collections;
+
+public class LlmCompletionLogCollection : MongoBase
+{
+    public string ConversationId { get; set; }
+    public string MessageId { get; set; }
+    public string AgentId { get; set; }
+    public string Prompt { get; set; }
+    public string? Response { get; set; }
+    public DateTime CreateDateTime { get; set; }
+}

src/Plugins/BotSharp.Plugin.MongoStorage/MongoBase.cs

@@ -1,5 +1,3 @@
-using MongoDB.Bson.Serialization.Attributes;
-
 namespace BotSharp.Plugin.MongoStorage;
 
 [BsonIgnoreExtraElements(Inherited = true)]

src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs

@@ -45,4 +45,7 @@ public IMongoCollection<UserCollection> Users
 
     public IMongoCollection<UserAgentCollection> UserAgents
         => Database.GetCollection<UserAgentCollection>($"{_collectionPrefix}_UserAgents");
+
+    public IMongoCollection<LlmCompletionLogCollection> LlmCompletionLogs
+        => Database.GetCollection<LlmCompletionLogCollection>($"{_collectionPrefix}_Llm_Completion_Logs");
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs

@@ -862,4 +862,22 @@ public void CreateUser(User user)
         _dc.Users.InsertOne(userCollection);
     }
     #endregion
+
+    #region LLM Completion Log
+    public void SaveLlmCompletionLog(LlmCompletionLog log)
+    {
+        var completiongLog = new LlmCompletionLogCollection
+        {
+            Id = string.IsNullOrEmpty(log.Id) ? Guid.NewGuid().ToString() : log.Id,
+            ConversationId = log.ConversationId,
+            MessageId = log.MessageId,
+            AgentId = log.AgentId,
+            Prompt = log.Prompt,
+            Response = log.Response,
+            CreateDateTime = log.CreateDateTime
+        };
+
+        _dc.LlmCompletionLogs.InsertOne(completiongLog);
+    }
+    #endregion
 }