Merge branch 'master' into sk/text-completion

xbotter · web-flow · commit 83e3c38999ef · 2023-11-13T22:30:30.000+08:00
diff --git a/BotSharp.sln b/BotSharp.sln
@@ -67,6 +67,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.SemanticKer
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.SemanticKernel.UnitTests", "tests\BotSharp.Plugin.SemanticKernel.UnitTests\BotSharp.Plugin.SemanticKernel.UnitTests.csproj", "{BC57D428-A1A4-4D38-A2D0-AC6CA943F247}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Twilio", "src\Plugins\BotSharp.Plugin.Twilio\BotSharp.Plugin.Twilio.csproj", "{E627F1E3-BE03-443A-83A2-86A855A278EB}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -243,6 +245,14 @@ Global
 		{BC57D428-A1A4-4D38-A2D0-AC6CA943F247}.Release|Any CPU.Build.0 = Release|Any CPU
 		{BC57D428-A1A4-4D38-A2D0-AC6CA943F247}.Release|x64.ActiveCfg = Release|Any CPU
 		{BC57D428-A1A4-4D38-A2D0-AC6CA943F247}.Release|x64.Build.0 = Release|Any CPU
+		{E627F1E3-BE03-443A-83A2-86A855A278EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E627F1E3-BE03-443A-83A2-86A855A278EB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E627F1E3-BE03-443A-83A2-86A855A278EB}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E627F1E3-BE03-443A-83A2-86A855A278EB}.Debug|x64.Build.0 = Debug|Any CPU
+		{E627F1E3-BE03-443A-83A2-86A855A278EB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E627F1E3-BE03-443A-83A2-86A855A278EB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E627F1E3-BE03-443A-83A2-86A855A278EB}.Release|x64.ActiveCfg = Release|Any CPU
+		{E627F1E3-BE03-443A-83A2-86A855A278EB}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -276,6 +286,7 @@ Global
 		{8BC29F8A-78D6-422C-B522-10687ADC38ED} = {D5293208-2BEF-42FC-A64C-5954F61720BA}
 		{73EE2CD0-3B27-4F02-A67B-762CBDD740D0} = {D5293208-2BEF-42FC-A64C-5954F61720BA}
 		{BC57D428-A1A4-4D38-A2D0-AC6CA943F247} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
+		{E627F1E3-BE03-443A-83A2-86A855A278EB} = {64264688-0F5C-4AB0-8F2B-B59B717CCE00}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}
diff --git a/README.md b/README.md
@@ -24,7 +24,7 @@ It's written in C# running on .Net Core that is full cross-platform framework, t
 * Built-in multi-agents and conversation with state management.
 * Support multiple LLM Planning approaches to handle different tasks.
 * Built-in RAG related interfaces, Memeory based vector searching.
-* Support multiple LLM platforms (ChatGPT 3.5 / 4.0, PaLM 2, LLaMA 2).
+* Support multiple LLM platforms (ChatGPT 3.5 / 4.0, PaLM 2, LLaMA 2, HuggingFace).
 * Allow multiple agents with different responsibilities cooperate to complete complex tasks. 
 * Build, test, evaluate and audit your LLM agent in one place.
 * Support different open source UI [Chatbot UI](src/Plugins/BotSharp.Plugin.ChatbotUI/Chatbot-UI.md), [HuggingChat UI](src/Plugins/BotSharp.Plugin.HuggingFace/HuggingChat-UI.md).
@@ -39,14 +39,39 @@ It's written in C# running on .Net Core that is full cross-platform framework, t
 ```
 2. Run UI project, reference to [Chatbot UI](src/Plugins/BotSharp.Plugin.ChatbotUI/Chatbot-UI.md).
 
-### Extension Libraries
+### Plugins
 
-BotSharp uses component design, the kernel is kept to a minimum, and business functions are implemented by external components. The modular design also allows contributors to better participate.
+BotSharp uses component design, the kernel is kept to a minimum, and business functions are implemented by external components. The modular design also allows contributors to better participate. Below are the bulit-in plugins:
 
-* Chatbot UI connector. 
-* A channel module of BotSharp for Facebook Messenger.
-* A channel module of BotSharp for Tencent Wechat.
-* A channel module of BotSharp for Telegram.
+#### Data Storages
+- BotSharp.Plugin.MongoStorage
+
+#### LLMs
+- BotSharp.Plugin.AzureOpenAI
+- BotSharp.Plugin.GoogleAI
+- BotSharp.Plugin.MetaAI
+- BotSharp.Plugin.HuggingFace
+- BotSharp.Plugin.LLamaSharp
+
+#### Messaging / Channel
+- BotSharp.OpenAPI
+- BotSharp.Plugin.MetaMessenger
+- BotSharp.Plugin.Twilio
+- BotSharp.Plugin.WeChat
+  
+#### RAGS
+- BotSharp.Plugin.KnowledgeBase
+- BotSharp.Plugin.Qdrant
+
+#### Visions
+- BotSharp.Plugin.PaddleSharp
+
+#### Tools
+- BotSharp.Plugin.RoutingSpeeder
+- BotSharp.Plugin.PizzaBot
+
+#### UIs
+- BotSharp.Plugin.ChatbotUI
 
 ### Documents
 
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationAttachmentService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationAttachmentService.cs
@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Conversations;
+
+public interface IConversationAttachmentService
+{
+    string GetDirectory(string conversationId);
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Attachment.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Attachment.cs
@@ -0,0 +1,21 @@
+namespace BotSharp.Abstraction.Conversations.Models;
+
+public class Attachment
+{
+    public string ContentType { get; set; }
+
+    //
+    // Summary:
+    //     Gets the raw Content-Disposition header of the uploaded file.
+    public string ContentDisposition { get; set; }
+
+    //
+    // Summary:
+    //     Gets the file length in bytes.
+    public long Length { get; set; }
+
+    //
+    // Summary:
+    //     Gets the file name from the Content-Disposition header.
+    public string FileName { get; set; }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs
@@ -1,4 +1,6 @@
 using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Messaging.Models;
+using BotSharp.Abstraction.Messaging.Models.RichContent;
 
 namespace BotSharp.Abstraction.Conversations.Models;
 
@@ -35,7 +37,7 @@ public class RoleDialogModel : ITrackableMessage
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public object Data { get; set; }
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public object? RichContent { get; set; }
+    public RichContent<IMessageTemplate>? RichContent { get; set; }
 
     /// <summary>
     /// Stop conversation completion
diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs
@@ -12,6 +12,8 @@ public class FunctionCallFromLlm : RoutingArgs
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public JsonDocument? Arguments { get; set; }
 
+    public bool IsExecutionOnce { get; set; }
+
     public override string ToString()
     {
         var route = string.IsNullOrEmpty(AgentName) ? "" : $"<Route to {AgentName.ToUpper()} because {Reason}>";
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/IMessageTemplate.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/IMessageTemplate.cs
@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Messaging.Models;
+
+public interface IMessageTemplate
+{
+    string Text { get; set; }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/QuickReplyElement.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/QuickReplyElement.cs
@@ -9,7 +9,6 @@ public class QuickReplyElement
         public string? Payload { get; set; }
         [JsonPropertyName("image_url")]
         public string? ImageUrl { get; set; }
-        [JsonPropertyName("postback_url")]
         public string? PostBackUrl { get; set; }
     }
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/QuickReplyMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/QuickReplyMessage.cs
@@ -1,8 +1,10 @@
 
 namespace BotSharp.Abstraction.Messaging.Models.RichContent
 {
-    public class QuickReplyMessage : TextMessage
+    public class QuickReplyMessage : IMessageTemplate
     {
+        public string Text { get; set; } = string.Empty;
+
         [JsonPropertyName("quick_replies")]
         public List<QuickReplyElement> QuickReplies { get; set; } = new List<QuickReplyElement>();
     }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/RichContent.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/RichContent.cs
@@ -1,6 +1,6 @@
 namespace BotSharp.Abstraction.Messaging.Models.RichContent
 {
-    public class RichContent<T>
+    public class RichContent<T> where T : IMessageTemplate
     {
         public Recipient Recipient { get; set; } = new Recipient();
         /// <summary>
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/AttachmentTemplate.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/AttachmentTemplate.cs
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ButtonTemplate.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ButtonTemplate.cs
@@ -1,8 +1,10 @@
 
 namespace BotSharp.Abstraction.Messaging.Models.RichContent.Template
 {
-    public class ButtonTemplate : TextMessage
+    public class ButtonTemplate : IMessageTemplate
     {
+        public string Text { get; set; } = string.Empty;
+
         [JsonPropertyName("template_type")]
         public string TemplateType => "button"; 
         public List<ButtonElement> Buttons { get; set; } = new List<ButtonElement>();
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/MultiSelectTemplate.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/MultiSelectTemplate.cs
@@ -0,0 +1,19 @@
+
+namespace BotSharp.Abstraction.Messaging.Models.RichContent.Template
+{
+    public class MultiSelectTemplate : IMessageTemplate
+    {
+        public string Text { get; set; } = string.Empty;
+
+        [JsonPropertyName("template_type")]
+        public string TemplateType => "multi-select";
+        public List<OptionElement> Options { get; set; } = new List<OptionElement>();
+    }
+
+    public class OptionElement
+    {
+        public string Title { get; set; } = string.Empty;
+        public string Type { get; set; } = string.Empty;
+        public string? Payload { get; set; }
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/TemplateMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/TemplateMessage.cs
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/TextMessage.cs
@@ -1,8 +1,7 @@
 
-namespace BotSharp.Abstraction.Messaging.Models.RichContent
+namespace BotSharp.Abstraction.Messaging.Models.RichContent;
+
+public class TextMessage : IMessageTemplate
 {
-    public class TextMessage
-    {
-        public string Text { get; set; } = string.Empty;
-    }
+    public string Text { get; set; } = string.Empty;
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContentJsonConverter .cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContentJsonConverter .cs
@@ -0,0 +1,16 @@
+using System.Text.Json;
+
+namespace BotSharp.Abstraction.Messaging.Models;
+
+public class RichContentJsonConverter : JsonConverter<IMessageTemplate>
+{
+    public override IMessageTemplate? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        throw new NotImplementedException();
+    }
+
+    public override void Write(Utf8JsonWriter writer, IMessageTemplate value, JsonSerializerOptions options)
+    {
+        JsonSerializer.Serialize(writer, value, value.GetType(), options);
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
@@ -56,7 +56,6 @@
     <PackageReference Include="Colorful.Console" Version="1.2.15" />
     <PackageReference Include="EntityFrameworkCore.BootKit" Version="6.2.1" />
     <PackageReference Include="Fluid.Core" Version="2.5.0" />
-    <PackageReference Include="TensorFlow.Keras" Version="0.11.4" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs b/src/Infrastructure/BotSharp.Core/BotSharpServiceCollectionExtensions.cs
@@ -102,6 +102,8 @@ public static IServiceCollection AddBotSharp(this IServiceCollection services, I
         services.AddScoped<IEvaluatingService, EvaluatingService>();
         services.AddScoped<IExecutionLogger, ExecutionLogger>();
 
+        services.AddScoped<IConversationAttachmentService, ConversationAttachmentService>();
+
         return services;
     }
 
diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationAttachmentService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationAttachmentService.cs
@@ -0,0 +1,28 @@
+using BotSharp.Abstraction.Repositories;
+using System.IO;
+
+namespace BotSharp.Core.Conversations.Services;
+
+public class ConversationAttachmentService : IConversationAttachmentService
+{
+    private readonly BotSharpDatabaseSettings _dbSettings;
+    private readonly IServiceProvider _services;
+
+    public ConversationAttachmentService(
+        BotSharpDatabaseSettings dbSettings,
+        IServiceProvider services)
+    {
+        _dbSettings = dbSettings;
+        _services = services;
+    }
+
+    public string GetDirectory(string conversationId)
+    {
+        var dir = Path.Combine(_dbSettings.FileRepository, "conversations", conversationId, "attachments");
+        if (!Directory.Exists(dir))
+        {
+            Directory.CreateDirectory(dir);
+        }
+        return dir;
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs
@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Abstraction.Messaging.Models;
 using BotSharp.Abstraction.Messaging.Models.RichContent;
 using BotSharp.Abstraction.Routing;
 using BotSharp.Abstraction.Routing.Settings;
@@ -100,9 +101,17 @@ private async Task HandleAssistantMessage(RoleDialogModel response, Func<RoleDia
         _logger.LogInformation(text);
 #endif
 
+        // Process rich content
+        if (response.RichContent != null &&
+            response.RichContent is RichContent<IMessageTemplate> template &&
+            string.IsNullOrEmpty(template.Message.Text))
+        {
+            template.Message.Text = response.Content;
+        }
+
         // Only read content from RichContent for UI rendering. When richContent is null, create a basic text message for richContent.
         var state = _services.GetRequiredService<IConversationStateService>();
-        response.RichContent = response.RichContent ?? new RichContent<TextMessage>
+        response.RichContent = response.RichContent ?? new RichContent<IMessageTemplate>
         {
             Recipient = new Recipient { Id = state.GetConversationId() },
             Message = new TextMessage { Text = response.Content }
diff --git a/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs b/src/Infrastructure/BotSharp.Core/Routing/Handlers/RouteToAgentRoutingHandler.cs
@@ -36,6 +36,10 @@ public async Task<bool> Handle(IRoutingService routing, FunctionCallFromLlm inst
         var ret = await function.Execute(message);
 
         var agentId = context.GetCurrentAgentId();
+        if (inst.IsExecutionOnce)
+        {
+            message.Content = inst.Question;
+        }
         ret = await routing.InvokeAgent(agentId, _dialogs);
 
         var response = _dialogs.Last();
diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs
@@ -44,7 +44,9 @@ public async Task<RoleDialogModel> ExecuteOnce(Agent agent, RoleDialogModel mess
             Function = "route_to_agent",
             Question = message.Content,
             Reason = message.Content,
-            AgentName = agent.Name
+            AgentName = agent.Name,
+            OriginalAgent = agent.Name,
+            IsExecutionOnce = true
         };
 
         var result = await handler.Handle(this, inst, message);
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
@@ -2,6 +2,8 @@
 using BotSharp.Abstraction.Conversations.Models;
 using BotSharp.Abstraction.Models;
 using BotSharp.OpenAPI.ViewModels.Conversations;
+using Microsoft.AspNetCore.Http;
+using System.Net.Http.Headers;
 
 namespace BotSharp.OpenAPI.Controllers;
 
@@ -78,4 +80,31 @@ await conv.SendMessage(agentId, inputMsg,
 
         return response;
     }
+
+    [HttpPost("/conversation/{agentId}/{conversationId}/attachments")]
+    public IActionResult UploadAttachments([FromRoute] string agentId,
+        [FromRoute] string conversationId, 
+        IFormFile[] files)
+    {
+        if (files != null && files.Length > 0)
+        {
+            var attachmentService = _services.GetRequiredService<IConversationAttachmentService>();
+            var dir = attachmentService.GetDirectory(conversationId);
+            foreach (var file in files)
+            {
+                // Save the file, process it, etc.
+                var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"');
+                var filePath = Path.Combine(dir, fileName);
+
+                using (var stream = new FileStream(filePath, FileMode.Create))
+                {
+                    file.CopyTo(stream);
+                }
+            }
+
+            return Ok(new { message = "File uploaded successfully." });
+        }
+
+        return BadRequest(new { message = "Invalid file." });
+    }
 }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Using.cs b/src/Plugins/BotSharp.Plugin.Twilio/Using.cs
diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj
diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,8 @@ public class FunctionCallFromLlm : RoutingArgs`
`12`	`12`	`[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]`
`13`	`13`	`public JsonDocument? Arguments { get; set; }`
`14`	`14`
	`15`	`+ public bool IsExecutionOnce { get; set; }`
	`16`	`+`
`15`	`17`	`public override string ToString()`
`16`	`18`	`{`
`17`	`19`	`var route = string.IsNullOrEmpty(AgentName) ? "" : $"<Route to {AgentName.ToUpper()} because {Reason}>";`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,6 @@ public class QuickReplyElement`
`9`	`9`	`public string? Payload { get; set; }`
`10`	`10`	`[JsonPropertyName("image_url")]`
`11`	`11`	`public string? ImageUrl { get; set; }`
`12`		`- [JsonPropertyName("postback_url")]`
`13`	`12`	`public string? PostBackUrl { get; set; }`
`14`	`13`	`}`
`15`	`14`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,10 @@`
`1`	`1`
`2`	`2`	`namespace BotSharp.Abstraction.Messaging.Models.RichContent`
`3`	`3`	`{`
`4`		`- public class QuickReplyMessage : TextMessage`
	`4`	`+ public class QuickReplyMessage : IMessageTemplate`
`5`	`5`	`{`
	`6`	`+ public string Text { get; set; } = string.Empty;`
	`7`	`+`
`6`	`8`	`[JsonPropertyName("quick_replies")]`
`7`	`9`	`public List<QuickReplyElement> QuickReplies { get; set; } = new List<QuickReplyElement>();`
`8`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`namespace BotSharp.Abstraction.Messaging.Models.RichContent`
`2`	`2`	`{`
`3`		`- public class RichContent<T>`
	`3`	`+ public class RichContent<T> where T : IMessageTemplate`
`4`	`4`	`{`
`5`	`5`	`public Recipient Recipient { get; set; } = new Recipient();`
`6`	`6`	`/// <summary>`