Merge branch 'master' of https://github.com/SciSharp/BotSharp

Author: Jicheng Lu

src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs

@@ -6,4 +6,5 @@ public class WebBrowsingSettings
     public bool Headless { get; set; }
     // Default timeout in milliseconds
     public float DefaultTimeout { get; set; } = 30000;
+    public bool IsEnableScreenshot { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs

@@ -78,4 +78,7 @@ public virtual Task OnBreakpointUpdated(string conversationId, bool resetStates)
 
     public virtual Task OnNotificationGenerated(RoleDialogModel message)
         => Task.CompletedTask;
+
+    public Task OnUserDisconnected(Conversation conversation)
+        => Task.CompletedTask;
 }

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs

@@ -25,6 +25,13 @@ public interface IConversationHook
     /// <returns></returns>
     Task OnUserAgentConnectedInitially(Conversation conversation);
 
+    /// <summary>
+    /// Triggered when user disconnects with agent.
+    /// </summary>
+    /// <param name="conversation"></param>
+    /// <returns></returns>
+    Task OnUserDisconnected(Conversation conversation);
+
     /// <summary>
     /// Triggered once for every new conversation.
     /// </summary>

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Realtime;
+
+public interface IRealtimeHook
+{
+    string[] OnModelTranscriptPrompt(Agent agent);
+}

src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj

@@ -8,6 +8,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
+    <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
 </Project>

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -1,4 +1,6 @@
 using BotSharp.Abstraction.Utilities;
+using BotSharp.Core.Infrastructures;
+using Microsoft.AspNetCore.Cors.Infrastructure;
 
 namespace BotSharp.Core.Realtime.Services;
 
@@ -86,11 +88,14 @@ private async Task ConnectToModel(WebSocket userWebSocket)
         var routing = _services.GetRequiredService<IRoutingService>();
         routing.Context.Push(agent.Id);
 
+        var storage = _services.GetRequiredService<IConversationStorage>();
         var dialogs = convService.GetDialogHistory();
         if (dialogs.Count == 0)
         {
             dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi"));
+            storage.Append(_conn.ConversationId, dialogs.First());
         }
+
         routing.Context.SetDialogs(dialogs);
 
         await _completer.Connect(_conn, 
@@ -155,6 +160,7 @@ await _completer.Connect(_conn,
                     {
                         // append output audio transcript to conversation
                         dialogs.Add(message);
+                        storage.Append(_conn.ConversationId, message);
 
                         foreach (var hook in hookProvider.HooksOrderByPriority)
                         {
@@ -174,6 +180,7 @@ await _completer.Connect(_conn,
             {
                 // append input audio transcript to conversation
                 dialogs.Add(message);
+                storage.Append(_conn.ConversationId, message);
 
                 foreach (var hook in hookProvider.HooksOrderByPriority)
                 {
@@ -224,6 +231,9 @@ private async Task HandleUserDtmfReceived()
         };
         dialogs.Add(message);
 
+        var storage = _services.GetRequiredService<IConversationStorage>();
+        storage.Append(_conn.ConversationId, message);
+
         foreach (var hook in hookProvider.HooksOrderByPriority)
         {
             hook.SetAgent(agent)
@@ -239,14 +249,9 @@ private async Task HandleUserDtmfReceived()
 
     private async Task HandleUserDisconnected()
     {
-        // Save dialog history
-        var routing = _services.GetRequiredService<IRoutingService>();
-        var storage = _services.GetRequiredService<IConversationStorage>();
-        var dialogs = routing.Context.GetDialogs();
-        foreach (var item in dialogs)
-        {
-            storage.Append(_conn.ConversationId, item);
-        }
+        var convService = _services.GetRequiredService<IConversationService>();
+        var conversation = await convService.GetConversation(_conn.ConversationId);
+        await HookEmitter.Emit<IConversationHook>(_services, x => x.OnUserDisconnected(conversation));
     }
 
     private async Task SendEventToUser(WebSocket webSocket, object message)

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -72,4 +72,11 @@ public class InputAudioTranscription
 {
     [JsonPropertyName("model")]
     public string Model { get; set; } = null!;
+
+    [JsonPropertyName("language")]
+    public string Language { get; set; } = "en";
+
+    [JsonPropertyName("prompt")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Prompt { get; set; }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -2,7 +2,9 @@
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Options;
+using BotSharp.Abstraction.Realtime;
 using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.OpenAI.Models.Realtime;
 using OpenAI.Chat;
@@ -42,7 +44,7 @@ public async Task Connect(RealtimeHubConnection conn,
         Action onModelReady,
         Action<string,string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
-        Action<string> onAudioTranscriptDone,
+        Action<string> onModelAudioTranscriptDone,
         Action<List<RoleDialogModel>> onModelResponseDone,
         Action<string> onConversationItemCreated,
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
@@ -64,7 +66,7 @@ public async Task Connect(RealtimeHubConnection conn,
                 onModelReady,
                 onModelAudioDeltaReceived,
                 onModelAudioResponseDone,
-                onAudioTranscriptDone,
+                onModelAudioTranscriptDone,
                 onModelResponseDone,
                 onConversationItemCreated,
                 onInputAudioTranscriptionCompleted,
@@ -125,10 +127,10 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
         Action onModelReady,
         Action<string,string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
-        Action<string> onAudioTranscriptDone,
+        Action<string> onModelAudioTranscriptDone,
         Action<List<RoleDialogModel>> onModelResponseDone,
         Action<string> onConversationItemCreated,
-        Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
+        Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
         Action onUserInterrupted)
     {
         var buffer = new byte[1024 * 32];
@@ -138,7 +140,7 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
         {
             result = await _webSocket.ReceiveAsync(
                 new ArraySegment<byte>(buffer), CancellationToken.None);
-
+            
             // Convert received data to text/audio (Twilio sends Base64-encoded audio)
             string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
             if (string.IsNullOrEmpty(receivedText))
@@ -171,7 +173,7 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
                 _logger.LogInformation($"{response.Type}: {receivedText}");
                 var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
                 await Task.Delay(1000);
-                onAudioTranscriptDone(data.Transcript);
+                onModelAudioTranscriptDone(data.Transcript);
             }
             else if (response.Type == "response.audio.delta")
             {
@@ -201,8 +203,11 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             else if (response.Type == "conversation.item.input_audio_transcription.completed")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
-                var message = await OnInputAudioTranscriptionCompleted(conn, receivedText);
-                onInputAudioTranscriptionCompleted(message);
+                var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
+                if (!string.IsNullOrEmpty(message.Content))
+                {
+                    onUserAudioTranscriptionCompleted(message);
+                }
             }
             else if (response.Type == "input_audio_buffer.speech_started")
             {
@@ -309,6 +314,9 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDet
             return fn;
         }).ToArray();
 
+        var words = new List<string>();
+        HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
+
         var sessionUpdate = new
         {
             type = "session.update",
@@ -319,6 +327,8 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDet
                 InputAudioTranscription = new InputAudioTranscription
                 {
                     Model = "whisper-1",
+                    Language = "en",
+                    Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
                 },
                 Voice = "alloy",
                 Instructions = instruction,
@@ -329,7 +339,7 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDet
                 MaxResponseOutputTokens = 512,
                 TurnDetection = new RealtimeSessionTurnDetection
                 {
-                    Threshold = 0.8f,
+                    Threshold = 0.9f,
                     PrefixPadding = 300,
                     SilenceDuration = 800
                 }
@@ -662,7 +672,7 @@ await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.don
         return outputs;
     }
 
-    public async Task<RoleDialogModel> OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
+    public async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
     {
         var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
         return new RoleDialogModel(AgentRole.User, data.Transcript)

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs

@@ -1,4 +1,7 @@
+using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+using Microsoft.VisualBasic;
 using Twilio.Rest.Api.V2010.Account;
+using Task = System.Threading.Tasks.Task;
 
 namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
 
@@ -20,6 +23,7 @@ public HangupPhoneCallFn(
 
     public async Task<bool> Execute(RoleDialogModel message)
     {
+        var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
         var states = _services.GetRequiredService<IConversationStateService>();
         var callSid = states.GetState("twilio_call_sid");
 
@@ -30,14 +34,20 @@ public async Task<bool> Execute(RoleDialogModel message)
             return false;
         }
 
-        // Have to find the SID by the phone number
-        var call = CallResource.Update(
-            status: CallResource.UpdateStatusEnum.Completed,
-            pathSid: callSid
-        );
+        message.Content = args.GoodbyeMessage;
 
-        message.Content = "The call has ended.";
-        message.StopCompletion = true;
+        _ = Task.Run(async () =>
+        {
+            await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400);
+            // Have to find the SID by the phone number
+            var call = CallResource.Update(
+                status: CallResource.UpdateStatusEnum.Completed,
+                pathSid: callSid
+            );
+
+            message.Content = "The call has been ended.";
+            message.StopCompletion = true;
+        });
 
         return true;
     }

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/HangupPhoneCallArgs.cs

@@ -0,0 +1,9 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+
+public class HangupPhoneCallArgs
+{
+    [JsonPropertyName("goodbye_message")]
+    public string? GoodbyeMessage { get; set; }
+}