temp save

Author: Jicheng Lu

src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

@@ -2,8 +2,7 @@ namespace BotSharp.Abstraction.Files;
 
 public interface IBotSharpFileService
 {
-    string GetDirectory(string conversationId);
-
+    #region Conversation
     /// <summary>
     /// Get the files that have been uploaded in the chat.
     /// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file.
@@ -19,14 +18,19 @@ Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string s
         IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
         bool includeScreenShot = false, int? offset = null);
 
+    /// <summary>
+    /// Get the files that have been uploaded in the chat. No screenshot images are included.
+    /// </summary>
+    /// <param name="conversationId"></param>
+    /// <param name="messageIds"></param>
+    /// <param name="source"></param>
+    /// <param name="imageOnly"></param>
+    /// <returns></returns>
     IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
     string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
     IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);
     bool SaveMessageFiles(string conversationId, string messageId, string source, List<BotSharpFile> files);
 
-    string GetUserAvatar();
-    bool SaveUserAvatar(BotSharpFile file);
-
     /// <summary>
     /// Delete files under messages
     /// </summary>
@@ -37,21 +41,35 @@ Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string s
     /// <returns></returns>
     bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
     bool DeleteConversationFiles(IEnumerable<string> conversationIds);
+    #endregion
+
+    #region Image
 
+    #endregion
+
+    #region Pdf
     /// <summary>
     /// Take screenshots of pdf pages and get response from llm
     /// </summary>
     /// <param name="prompt"></param>
     /// <param name="files">Pdf files</param>
     /// <returns></returns>
     Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
+    #endregion
+
+    #region User
+    string GetUserAvatar();
+    bool SaveUserAvatar(BotSharpFile file);
+    #endregion
 
+    #region Common
     /// <summary>
     /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
     /// </summary>
     /// <param name="data"></param>
     /// <returns></returns>
     (string, byte[]) GetFileInfoFromData(string data);
-
+    string GetDirectory(string conversationId);
     string GetFileContentType(string filePath);
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageEdit.cs

@@ -0,0 +1,5 @@
+namespace BotSharp.Abstraction.MLTasks;
+
+public interface IImageEdit
+{
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs

@@ -13,5 +13,5 @@ public interface IImageGeneration
     /// <param name="model">deployment name</param>
     void SetModelName(string model);
 
-    Task<RoleDialogModel> GetImageGeneration(Agent agent, List<RoleDialogModel> conversations);
+    Task<RoleDialogModel> GetImageGeneration(Agent agent, RoleDialogModel message);
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageVariation.cs

@@ -0,0 +1,19 @@
+using System.IO;
+
+namespace BotSharp.Abstraction.MLTasks;
+
+public interface IImageVariation
+{
+    /// <summary>
+    /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI
+    /// </summary>
+    string Provider { get; }
+
+    /// <summary>
+    /// Set model name, one provider can consume different model or version(s)
+    /// </summary>
+    /// <param name="model">deployment name</param>
+    void SetModelName(string model);
+
+    RoleDialogModel GetImageVariation(Agent agent, RoleDialogModel message, Stream image, string imageFileName);
+}

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Common.cs

@@ -0,0 +1,5 @@
+namespace BotSharp.Core.Files.Services;
+
+public partial class BotSharpFileService
+{
+}

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageGeneration.cs

@@ -0,0 +1,5 @@
+namespace BotSharp.Core.Files.Services;
+
+public partial class BotSharpFileService
+{
+}

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.ImageVariation.cs

@@ -0,0 +1,5 @@
+namespace BotSharp.Core.Files.Services;
+
+public partial class BotSharpFileService
+{
+}

src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs

@@ -5,7 +5,7 @@ namespace BotSharp.Core.Infrastructures;
 
 public class CompletionProvider
 {
-    public static object GetCompletion(IServiceProvider services, 
+    public static object? GetCompletion(IServiceProvider services, 
         string? provider = null, 
         string? model = null, 
         AgentLlmConfig? agentConfig = null)
@@ -23,13 +23,21 @@ public static object GetCompletion(IServiceProvider services,
                 model: model, 
                 agentConfig: agentConfig);
         }
-        else
+        else if (settings.Type == LlmModelType.Embedding)
         {
-            return GetChatCompletion(services, 
-                provider: provider, 
-                model: model, 
+            return GetTextEmbedding(services,
+                provider: provider,
+                model: model);
+        }
+        else if (settings.Type == LlmModelType.Chat)
+        {
+            return GetChatCompletion(services,
+                provider: provider,
+                model: model,
                 agentConfig: agentConfig);
         }
+
+        return null;
     }
 
     public static IChatCompletion GetChatCompletion(IServiceProvider services, 
@@ -51,7 +59,6 @@ public static IChatCompletion GetChatCompletion(IServiceProvider services,
         }
 
         completer?.SetModelName(model);
-
         return completer;
     }
 
@@ -72,7 +79,6 @@ public static ITextCompletion GetTextCompletion(IServiceProvider services,
         }
 
         completer.SetModelName(model);
-
         return completer;
     }
 
@@ -95,7 +101,26 @@ public static IImageGeneration GetImageGeneration(IServiceProvider services,
         }
 
         completer?.SetModelName(model);
+        return completer;
+    }
 
+    public static IImageVariation GetImageVariation(IServiceProvider services,
+        string? provider = null,
+        string? model = null,
+        string? modelId = null,
+        bool imageGenerate = false)
+    {
+        var completions = services.GetServices<IImageVariation>();
+        (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, imageGenerate: imageGenerate);
+
+        var completer = completions.FirstOrDefault(x => x.Provider == provider);
+        if (completer == null)
+        {
+            var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
+            logger.LogError($"Can't resolve completion provider by {provider}");
+        }
+
+        completer?.SetModelName(model);
         return completer;
     }
 
@@ -152,7 +177,6 @@ private static (string, string) GetProviderAndModel(IServiceProvider services,
 
         state.SetState("provider", provider);
         state.SetState("model", model);
-
         return (provider, model);
     }
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -119,10 +119,7 @@ public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingM
             var message = await completion.GetImageGeneration(new Agent()
             {
                 Id = Guid.Empty.ToString(),
-            }, new List<RoleDialogModel>
-            {
-                new RoleDialogModel(AgentRole.User, input.Text)
-            });
+            }, new RoleDialogModel(AgentRole.User, input.Text));
 
             imageViewModel.Content = message.Content;
             imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
@@ -137,6 +134,34 @@ public async Task<ImageGenerationViewModel> ImageGeneration([FromBody] IncomingM
         }
     }
 
+    //[HttpPost("/instruct/image-variation")]
+    //public ImageGenerationViewModel ImageVariation([FromBody] IncomingMessageModel input)
+    //{
+    //    var state = _services.GetRequiredService<IConversationStateService>();
+    //    input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External));
+    //    var imageViewModel = new ImageGenerationViewModel();
+
+    //    try
+    //    {
+    //        var completion = CompletionProvider.GetImageVariation(_services, provider: input.Provider ?? "openai", model: input.Model ?? "dall-e-2");
+    //        var message = completion.GetImageVariation(new Agent()
+    //        {
+    //            Id = Guid.Empty.ToString(),
+    //        }, new RoleDialogModel(AgentRole.User, input.Text));
+
+    //        imageViewModel.Content = message.Content;
+    //        imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList();
+    //        return imageViewModel;
+    //    }
+    //    catch (Exception ex)
+    //    {
+    //        var error = $"Error in image generation. {ex.Message}";
+    //        _logger.LogError(error);
+    //        imageViewModel.Message = error;
+    //        return imageViewModel;
+    //    }
+    //}
+
     [HttpPost("/instruct/pdf-completion")]
     public async Task<PdfCompletionViewModel> PdfCompletion([FromBody] IncomingMessageModel input)
     {

src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs

@@ -62,7 +62,7 @@ private async Task<string> GetImageGeneration(Agent agent, RoleDialogModel messa
             var completion = CompletionProvider.GetImageGeneration(_services, provider: "openai", model: "dall-e-3");
             var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content;
             var dialog = RoleDialogModel.From(message, AgentRole.User, text);
-            var result = await completion.GetImageGeneration(agent, new List<RoleDialogModel> { dialog });
+            var result = await completion.GetImageGeneration(agent, dialog);
             SaveGeneratedImages(result?.GeneratedImages);
             return result?.Content ?? string.Empty;
         }