Merge pull request #553 from iceljc/features/add-image-edit

Features/add image edit

Author: iceljc

src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

@@ -2,8 +2,7 @@ namespace BotSharp.Abstraction.Files;
 
 public interface IBotSharpFileService
 {
-    string GetDirectory(string conversationId);
-
+    #region Conversation
     /// <summary>
     /// Get the files that have been uploaded in the chat.
     /// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file.
@@ -19,14 +18,19 @@ Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string s
         IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
         bool includeScreenShot = false, int? offset = null);
 
+    /// <summary>
+    /// Get the files that have been uploaded in the chat. No screenshot images are included.
+    /// </summary>
+    /// <param name="conversationId"></param>
+    /// <param name="messageIds"></param>
+    /// <param name="source"></param>
+    /// <param name="imageOnly"></param>
+    /// <returns></returns>
     IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
     string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
     IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
     bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
 
-    string GetUserAvatar();
-    bool SaveUserAvatar(BotSharpFile file);
-
     /// <summary>
     /// Delete files under messages
     /// </summary>
@@ -37,21 +41,36 @@ Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string s
     /// <returns></returns>
     bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
     bool DeleteConversationFiles(IEnumerable<string> conversationIds);
+    #endregion
 
+    #region Image
+    Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text);
+    Task<RoleDialogModel> VarifyImage(string? provider, string? model, BotSharpFile file);
+    #endregion
+
+    #region Pdf
     /// <summary>
     /// Take screenshots of pdf pages and get response from llm
     /// </summary>
     /// <param name="prompt"></param>
     /// <param name="files">Pdf files</param>
     /// <returns></returns>
-    Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
+    Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
+    #endregion
+
+    #region User
+    string GetUserAvatar();
+    bool SaveUserAvatar(BotSharpFile file);
+    #endregion
 
+    #region Common
     /// <summary>
     /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
     /// </summary>
     /// <param name="data"></param>
     /// <returns></returns>
     (string, byte[]) GetFileInfoFromData(string data);
-
+    string GetDirectory(string conversationId);
     string GetFileContentType(string filePath);
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageEdit.cs

@@ -0,0 +1,5 @@
+namespace BotSharp.Abstraction.MLTasks;
+
+public interface IImageEdit
+{
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs

@@ -13,5 +13,5 @@ public interface IImageGeneration
     /// <param name="model">deployment name</param>
     void SetModelName(string model);
 
-    Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations);
+    Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message);
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageVariation.cs

@@ -0,0 +1,19 @@
+using System.IO;
+
+namespace BotSharp.Abstraction.MLTasks;
+
+public interface IImageVariation
+{
+    /// <summary>
+    /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI
+    /// </summary>
+    string Provider { get; }
+
+    /// <summary>
+    /// Set model name, one provider can consume different model or version(s)
+    /// </summary>
+    /// <param name="model">deployment name</param>
+    void SetModelName(string model);
+
+    Task<RoleDialogModel> GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
+}

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs

@@ -0,0 +1,46 @@
+using Microsoft.AspNetCore.StaticFiles;
+using System.IO;
+
+namespace BotSharp.Core.Files.Services;
+
+public partial class BotSharpFileService
+{
+    public string GetDirectory(string conversationId)
+    {
+        var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
+        if (!Directory.Exists(dir))
+        {
+            Directory.CreateDirectory(dir);
+        }
+        return dir;
+    }
+
+    public (string, byte[]) GetFileInfoFromData(string data)
+    {
+        if (string.IsNullOrEmpty(data))
+        {
+            return (string.Empty, new byte[0]);
+        }
+
+        var typeStartIdx = data.IndexOf(':');
+        var typeEndIdx = data.IndexOf(';');
+        var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
+
+        var base64startIdx = data.IndexOf(',');
+        var base64Str = data.Substring(base64startIdx + 1);
+
+        return (contentType, Convert.FromBase64String(base64Str));
+    }
+
+    public string GetFileContentType(string filePath)
+    {
+        string contentType;
+        var provider = new FileExtensionContentTypeProvider();
+        if (!provider.TryGetContentType(filePath, out contentType))
+        {
+            contentType = string.Empty;
+        }
+
+        return contentType;
+    }
+}

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Image.cs

@@ -0,0 +1,57 @@
+using System.IO;
+
+namespace BotSharp.Core.Files.Services;
+
+public partial class BotSharpFileService
+{
+    public async Task<RoleDialogModel> GenerateImage(string? provider, string? model, string text)
+    {
+        var completion = CompletionProvider.GetImageGeneration(_services, provider: provider ?? "openai", model: model ?? "dall-e-3");
+        var message = await completion.GetImageGeneration(new Agent()
+        {
+            Id = Guid.Empty.ToString(),
+        }, new RoleDialogModel(AgentRole.User, text));
+        return message;
+    }
+
+    public async Task<RoleDialogModel> VarifyImage(string? provider, string? model, BotSharpFile file)
+    {
+        if (string.IsNullOrWhiteSpace(file?.FileUrl) && string.IsNullOrWhiteSpace(file?.FileData))
+        {
+            throw new ArgumentException($"Please fill in at least file url or file data!");
+        }
+
+        var completion = CompletionProvider.GetImageVariation(_services, provider: provider ?? "openai", model: model ?? "dall-e-2");
+        var bytes = await DownloadFile(file);
+        using var stream = new MemoryStream();
+        stream.Write(bytes, 0, bytes.Length);
+        stream.Position = 0;
+
+        var message = await completion.GetImageVariation(new Agent()
+        {
+            Id = Guid.Empty.ToString()
+        }, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName ?? string.Empty);
+        stream.Close();
+
+        return message;
+    }
+
+    #region Private methods
+    private async Task<byte[]> DownloadFile(BotSharpFile file)
+    {
+        var bytes = new byte[0];
+        if (!string.IsNullOrEmpty(file.FileUrl))
+        {
+            var http = _services.GetRequiredService<IHttpClientFactory>();
+            using var client = http.CreateClient();
+            bytes = await client.GetByteArrayAsync(file.FileUrl);
+        }
+        else if (!string.IsNullOrEmpty(file.FileData))
+        {
+            (_, bytes) = GetFileInfoFromData(file.FileData);
+        }
+
+        return bytes;
+    }
+    #endregion
+}

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs

@@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services;
 
 public partial class BotSharpFileService
 {
-    public async Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
+    public async Task<string> ReadPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
     {
         var content = string.Empty;
 
@@ -22,7 +22,7 @@ public async Task<string> InstructPdf(string? provider, string? model, string? m
 
         try
         {
-            var pdfFiles = await SaveFiles(sessionDir, files);
+            var pdfFiles = await DownloadFiles(sessionDir, files);
             var images = await ConvertPdfToImages(pdfFiles);
             if (images.IsNullOrEmpty()) return content;
 
@@ -44,7 +44,7 @@ public async Task<string> InstructPdf(string? provider, string? model, string? m
         }
         catch (Exception ex)
         {
-            _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}");
+            _logger.LogError($"Error when analyzing pdf in file service: {ex.Message}\r\n{ex.InnerException}");
             return content;
         }
         finally
@@ -60,7 +60,7 @@ private string GetSessionDirectory(string id)
         return dir;
     }
 
-    private async Task<IEnumerable<string>> SaveFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
+    private async Task<IEnumerable<string>> DownloadFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
     {
         if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
         {
@@ -105,7 +105,7 @@ private async Task<IEnumerable<string>> SaveFiles(string dir, List<BotSharpFile>
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when saving pdf file: {ex.Message}");
+                _logger.LogWarning($"Error when saving pdf file: {ex.Message}\r\n{ex.InnerException}");
                 continue;
             }
         }
@@ -133,7 +133,7 @@ private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> f
             }
             catch (Exception ex)
             {
-                _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}");
+                _logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}\r\n{ex.InnerException}");
                 continue;
             }
         }

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs

@@ -40,7 +40,7 @@ public bool SaveUserAvatar(BotSharpFile file)
         }
         catch (Exception ex)
         {
-            _logger.LogWarning($"Error when saving user avatar: {ex.Message}");
+            _logger.LogWarning($"Error when saving user avatar: {ex.Message}\r\n{ex.InnerException}");
             return false;
         }
     }

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs

@@ -41,45 +41,6 @@ public BotSharpFileService(
         _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository);
     }
 
-    public string GetDirectory(string conversationId)
-    {
-        var dir = Path.Combine(_dbSettings.FileRepository, CONVERSATION_FOLDER, conversationId, "attachments");
-        if (!Directory.Exists(dir))
-        {
-            Directory.CreateDirectory(dir);
-        }
-        return dir;
-    }
-
-    public (string, byte[]) GetFileInfoFromData(string data)
-    {
-        if (string.IsNullOrEmpty(data))
-        {
-            return (string.Empty, new byte[0]);
-        }
-
-        var typeStartIdx = data.IndexOf(':');
-        var typeEndIdx = data.IndexOf(';');
-        var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
-
-        var base64startIdx = data.IndexOf(',');
-        var base64Str = data.Substring(base64startIdx + 1);
-
-        return (contentType, Convert.FromBase64String(base64Str));
-    }
-
-    public string GetFileContentType(string filePath)
-    {
-        string contentType;
-        var provider = new FileExtensionContentTypeProvider();
-        if (!provider.TryGetContentType(filePath, out contentType))
-        {
-            contentType = string.Empty;
-        }
-
-        return contentType;
-    }
-
     #region Private methods
     private bool ExistDirectory(string? dir)
     {