Merge pull request #1181 from iceljc/features/add-image-llm-config

Features/add image llm config

Author: Oceania2018

BotSharp.sln

@@ -147,6 +147,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandle
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ExcelHandler", "src\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj", "{FC63C875-E880-D8BB-B8B5-978AB7B62983}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ImageHandler", "src\Plugins\BotSharp.Plugin.ImageHandler\BotSharp.Plugin.ImageHandler.csproj", "{242F2D93-FCCE-4982-8075-F3052ECCA92C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -619,6 +621,14 @@ Global
 		{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.Build.0 = Release|Any CPU
 		{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.ActiveCfg = Release|Any CPU
 		{FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.Build.0 = Release|Any CPU
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|x64.Build.0 = Debug|Any CPU
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|x64.ActiveCfg = Release|Any CPU
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -690,6 +700,7 @@ Global
 		{B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
 		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 		{FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
+		{242F2D93-FCCE-4982-8075-F3052ECCA92C} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Agents.Options;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Repositories.Filters;
@@ -65,4 +66,13 @@ public interface IAgentService
     Task<List<UserAgent>> GetUserAgents(string userId);
 
     PluginDef GetPlugin(string agentId);
+
+    Task<List<AgentCodeScript>> GetAgentCodeScripts(string agentId, AgentCodeScriptFilter? filter = null)
+        => Task.FromResult(new List<AgentCodeScript>());
+
+    Task<string?> GetAgentCodeScript(string agentId, string scriptName, string scriptType = AgentCodeScriptType.Src)
+        => Task.FromResult(string.Empty);
+
+    Task<bool> UpdateAgentCodeScripts(string agentId, List<AgentCodeScript> codeScripts, AgentCodeScriptUpdateOptions? options = null)
+        => Task.FromResult(false);
 }

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs

@@ -41,4 +41,49 @@ public class AgentLlmConfig
     [JsonPropertyName("reasoning_effort_level")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? ReasoningEffortLevel { get; set; }
+
+    /// <summary>
+    /// Image generation config
+    /// </summary>
+    [JsonPropertyName("image_generation")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public LlmImageGenerationConfig? ImageGeneration { get; set; }
+
+    /// <summary>
+    /// Image edit config
+    /// </summary>
+    [JsonPropertyName("image_edit")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public LlmImageEditConfig? ImageEdit { get; set; }
+
+    /// <summary>
+    /// Audio transcription config
+    /// </summary>
+    [JsonPropertyName("audio_transcription")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public LlmAudioTranscriptionConfig? AudioTranscription { get; set; }
+
+    /// <summary>
+    /// Realtime config
+    /// </summary>
+    [JsonPropertyName("realtime")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public LlmRealtimeConfig? Realtime { get; set; }
+}
+
+
+public class LlmImageGenerationConfig : LlmProviderModel
+{
+}
+
+public class LlmImageEditConfig : LlmProviderModel
+{
 }
+
+public class LlmAudioTranscriptionConfig : LlmProviderModel
+{
+}
+
+public class LlmRealtimeConfig : LlmProviderModel
+{
+}

src/Infrastructure/BotSharp.Abstraction/Agents/Options/AgentCodeScriptUpdateOptions.cs

@@ -0,0 +1,8 @@
+using BotSharp.Abstraction.Repositories.Models;
+
+namespace BotSharp.Abstraction.Agents.Options;
+
+public class AgentCodeScriptUpdateOptions : AgentCodeScriptDbUpdateOptions
+{
+    public bool DeleteIfNotIncluded { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs

@@ -141,7 +141,7 @@ public class RoleDialogModel : ITrackableMessage
     public bool IsFromAssistant => Role == AgentRole.Assistant || Role == AgentRole.Model;
 
     [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
-    public string RoleContent
+    public string LlmContent
     {
         get
         {
@@ -150,10 +150,6 @@ public string RoleContent
             {
                 text = !string.IsNullOrWhiteSpace(Payload) ? Payload : Content;
             }
-            else
-            {
-                text = !string.IsNullOrWhiteSpace(Content) ? Content : RichContent?.Message?.Text;
-            }
 
             return text;
         }

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileLlmProcessOptions.cs

@@ -0,0 +1,39 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class FileLlmProcessOptions
+{
+    /// <summary>
+    /// Llm provider
+    /// </summary>
+    public string? Provider { get; set; }
+
+    /// <summary>
+    /// llm model
+    /// </summary>
+    public string? Model { get; set; }
+
+    /// <summary>
+    /// Llm maximum output tokens
+    /// </summary>
+    public int? MaxOutputTokens { get; set; }
+
+    /// <summary>
+    /// Reasoning effort level
+    /// </summary>
+    public string? ReasoningEfforLevel { get; set; }
+
+    /// <summary>
+    /// Instruction
+    /// </summary>
+    public string? Instruction { get; set; }
+
+    /// <summary>
+    /// Template name in Agent
+    /// </summary>
+    public string? TemplateName { get; set; }
+
+    /// <summary>
+    /// Data that is used to render instruction
+    /// </summary>
+    public Dictionary<string, object>? Data { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs

@@ -10,7 +10,7 @@ public class SelectFileOptions : LlmConfigBase
     /// <summary>
     /// Template (prompt) name
     /// </summary>
-    public string? Template { get; set; }
+    public string? TemplateName { get; set; }
 
     /// <summary>
     /// Description that user provides to select files

src/Infrastructure/BotSharp.Abstraction/Files/Proccessors/IFileLlmProcessor.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Abstraction.Files.Proccessors;
+
+public interface IFileLlmProcessor
+{
+    public string Provider { get; }
+
+    Task<RoleDialogModel> GetFileLlmInferenceAsync(Agent agent, string text, IEnumerable<InstructFileModel> files, FileLlmProcessOptions? options = null)
+        => throw new NotImplementedException();
+}

src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs

@@ -13,10 +13,13 @@ public interface IInstructService
     /// <param name="templateName"></param>
     /// <param name="files"></param>
     /// <param name="codeOptions"></param>
+    /// <param name="fileOptions"></param>
     /// <returns></returns>
     Task<InstructResult> Execute(string agentId, RoleDialogModel message,
         string? instruction = null, string? templateName = null,
-        IEnumerable<InstructFileModel>? files = null, CodeInstructOptions? codeOptions = null);
+        IEnumerable<InstructFileModel>? files = null,
+        CodeInstructOptions? codeOptions = null,
+        FileInstructOptions? fileOptions = null);
 
     /// <summary>
     /// A generic way to execute completion by using specified instruction or template

src/Infrastructure/BotSharp.Abstraction/Instructs/Models/FileInstructOptions.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Instructs.Models;
+
+public class FileInstructOptions
+{
+    public string? FileLlmProcessorProvider { get; set; }
+}