add file llm processor

Author: Jicheng Lu

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileLlmProcessOptions.cs

@@ -0,0 +1,34 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class FileLlmProcessOptions
+{
+    /// <summary>
+    /// Llm provider
+    /// </summary>
+    public string? LlmProvider { get; set; }
+
+    /// <summary>
+    /// llm model
+    /// </summary>
+    public string? LlModel { get; set; }
+
+    /// <summary>
+    /// Reasoning effort level
+    /// </summary>
+    public string? ReasoningEfforLevel { get; set; }
+
+    /// <summary>
+    /// Instruction
+    /// </summary>
+    public string? Instruction { get; set; }
+
+    /// <summary>
+    /// Template name in Agent
+    /// </summary>
+    public string? TemplateName { get; set; }
+
+    /// <summary>
+    /// Data that is used to render instruction
+    /// </summary>
+    public Dictionary<string, object>? Data { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Files/Proccessors/IFileLlmProcessor.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Abstraction.Files.Proccessors;
+
+public interface IFileLlmProcessor
+{
+    public string Provider { get; }
+
+    Task<RoleDialogModel> GetFileLlmInferenceAsync(Agent agent, string text, IEnumerable<InstructFileModel> files, FileLlmProcessOptions? options = null)
+        => throw new NotImplementedException();
+}

src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructService.cs

@@ -13,10 +13,13 @@ public interface IInstructService
     /// <param name="templateName"></param>
     /// <param name="files"></param>
     /// <param name="codeOptions"></param>
+    /// <param name="fileOptions"></param>
     /// <returns></returns>
     Task<InstructResult> Execute(string agentId, RoleDialogModel message,
         string? instruction = null, string? templateName = null,
-        IEnumerable<InstructFileModel>? files = null, CodeInstructOptions? codeOptions = null);
+        IEnumerable<InstructFileModel>? files = null,
+        CodeInstructOptions? codeOptions = null,
+        FileInstructOptions? fileOptions = null);
 
     /// <summary>
     /// A generic way to execute completion by using specified instruction or template

src/Infrastructure/BotSharp.Abstraction/Instructs/Models/FileInstructOptions.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Instructs.Models;
+
+public class FileInstructOptions
+{
+    public string? FileLlmProcessorProvider { get; set; }
+}

src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.CodeInterpreter;
+using BotSharp.Abstraction.Files.Proccessors;
 using BotSharp.Abstraction.Instructs;
 using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.MLTasks;
@@ -14,9 +15,11 @@ public async Task<InstructResult> Execute(
         string? instruction = null,
         string? templateName = null,
         IEnumerable<InstructFileModel>? files = null,
-        CodeInstructOptions? codeOptions = null)
+        CodeInstructOptions? codeOptions = null,
+        FileInstructOptions? fileOptions = null)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
+        var state = _services.GetRequiredService<IConversationStateService>();
         Agent agent = await agentService.LoadAgent(agentId);
 
         var response = new InstructResult
@@ -68,6 +71,7 @@ public async Task<InstructResult> Execute(
 
         var provider = string.Empty;
         var model = string.Empty;
+        var result = string.Empty;
 
         // Render prompt
         var prompt = string.IsNullOrEmpty(templateName) ?
@@ -83,35 +87,45 @@ public async Task<InstructResult> Execute(
             provider = textCompleter.Provider;
             model = textCompleter.Model;
 
-            var result = await textCompleter.GetCompletion(prompt, agentId, message.MessageId);
+            result = await GetTextCompletion(textCompleter, agent, prompt, message.MessageId);
             response.Text = result;
         }
         else if (completer is IChatCompletion chatCompleter)
         {
             provider = chatCompleter.Provider;
             model = chatCompleter.Model;
+           
 
             if (instruction == "#TEMPLATE#")
             {
                 instruction = prompt;
                 prompt = message.Content;
             }
 
-            var result = await chatCompleter.GetChatCompletions(new Agent
+            IFileLlmProcessor? fileProcessor = null;
+            if (!files.IsNullOrEmpty() && !string.IsNullOrEmpty(fileOptions?.FileLlmProcessorProvider))
             {
-                Id = agentId,
-                Name = agent.Name,
-                Instruction = instruction
-            }, new List<RoleDialogModel>
+                fileProcessor = _services.GetServices<IFileLlmProcessor>()
+                                         .FirstOrDefault(x => x.Provider.IsEqualTo(fileOptions?.FileLlmProcessorProvider));
+            }
+
+            if (fileProcessor != null)
             {
-                new RoleDialogModel(AgentRole.User, prompt)
+                var inference = await fileProcessor.GetFileLlmInferenceAsync(agent, prompt, files, new FileLlmProcessOptions
                 {
-                    CurrentAgentId = agentId,
-                    MessageId = message.MessageId,
-                    Files = files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData, ContentType = x.ContentType }).ToList() ?? []
-                }
-            });
-            response.Text = result.Content;
+                    LlmProvider = provider,
+                    LlModel = model,
+                    Instruction = instruction,
+                    TemplateName = templateName,
+                    Data = state.GetStates().ToDictionary(x => x.Key, x => (object)x.Value)
+                });
+                result = inference?.Content ?? string.Empty;
+            }
+            else
+            {
+                result = await GetChatCompletion(chatCompleter, agent, instruction, prompt, message.MessageId, files);
+            }
+            response.Text = result;
         }
 
         // After completion hooks
@@ -141,7 +155,11 @@ await hook.OnResponseGenerated(new InstructResponseModel
     /// <param name="templateName"></param>
     /// <param name="codeOptions"></param>
     /// <returns></returns>
-    private async Task<InstructResult?> GetCodeResponse(Agent agent, RoleDialogModel message, string templateName, CodeInstructOptions? codeOptions)
+    private async Task<InstructResult?> GetCodeResponse(
+        Agent agent,
+        RoleDialogModel message,
+        string templateName,
+        CodeInstructOptions? codeOptions)
     {
         InstructResult? response = null;
 
@@ -261,4 +279,40 @@ await hook.OnResponseGenerated(new InstructResponseModel
 
         return response;
     }
+
+    private async Task<string> GetTextCompletion(
+        ITextCompletion textCompleter,
+        Agent agent,
+        string text,
+        string messageId)
+    {
+        var result = await textCompleter.GetCompletion(text, agent.Id, messageId);
+        return result;
+    }
+
+    private async Task<string> GetChatCompletion(
+        IChatCompletion chatCompleter,
+        Agent agent,
+        string instruction,
+        string text,
+        string messageId,
+        IEnumerable<InstructFileModel>? files = null)
+    {
+        var result = await chatCompleter.GetChatCompletions(new Agent
+        {
+            Id = agent.Id,
+            Name = agent.Name,
+            Instruction = instruction
+        }, new List<RoleDialogModel>
+        {
+            new RoleDialogModel(AgentRole.User, text)
+            {
+                CurrentAgentId = agent.Id,
+                MessageId = messageId,
+                Files = files?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData, ContentType = x.ContentType }).ToList() ?? []
+            }
+        });
+
+        return result.Content;
+    }
 }

src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs

@@ -39,7 +39,8 @@ public async Task<InstructResult> InstructCompletion([FromRoute] string agentId,
             instruction: input.Instruction,
             templateName: input.Template,
             files: input.Files,
-            codeOptions: input.CodeOptions);
+            codeOptions: input.CodeOptions,
+            fileOptions: input.FileOptions);
 
         result.States = state.GetStates();
         return result; 

src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/Request/InstructMessageModel.cs

@@ -12,6 +12,7 @@ public class InstructMessageModel : IncomingMessageModel
     public string? Template { get; set; }
     public List<InstructFileModel> Files { get; set; } = [];
     public CodeInstructOptions? CodeOptions { get; set; }
+    public FileInstructOptions? FileOptions { get; set; }
 }