IRealtimeHook

Author: Haiping Chen

src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs

@@ -78,4 +78,7 @@ public virtual Task OnBreakpointUpdated(string conversationId, bool resetStates)
 
     public virtual Task OnNotificationGenerated(RoleDialogModel message)
         => Task.CompletedTask;
+
+    public Task OnUserDisconnected(Conversation conversation)
+        => Task.CompletedTask;
 }

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs

@@ -25,6 +25,13 @@ public interface IConversationHook
     /// <returns></returns>
     Task OnUserAgentConnectedInitially(Conversation conversation);
 
+    /// <summary>
+    /// Triggered when user disconnects with agent.
+    /// </summary>
+    /// <param name="conversation"></param>
+    /// <returns></returns>
+    Task OnUserDisconnected(Conversation conversation);
+
     /// <summary>
     /// Triggered once for every new conversation.
     /// </summary>

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Realtime;
+
+public interface IRealtimeHook
+{
+    string[] OnModelTranscriptPrompt(Agent agent);
+}

src/Infrastructure/BotSharp.Core.Realtime/BotSharp.Core.Realtime.csproj

@@ -8,6 +8,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
+    <ProjectReference Include="..\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
 </Project>

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -1,4 +1,6 @@
 using BotSharp.Abstraction.Utilities;
+using BotSharp.Core.Infrastructures;
+using Microsoft.AspNetCore.Cors.Infrastructure;
 
 namespace BotSharp.Core.Realtime.Services;
 
@@ -86,11 +88,14 @@ private async Task ConnectToModel(WebSocket userWebSocket)
         var routing = _services.GetRequiredService<IRoutingService>();
         routing.Context.Push(agent.Id);
 
+        var storage = _services.GetRequiredService<IConversationStorage>();
         var dialogs = convService.GetDialogHistory();
         if (dialogs.Count == 0)
         {
             dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi"));
+            storage.Append(_conn.ConversationId, dialogs.First());
         }
+
         routing.Context.SetDialogs(dialogs);
 
         await _completer.Connect(_conn, 
@@ -155,6 +160,7 @@ await _completer.Connect(_conn,
                     {
                         // append output audio transcript to conversation
                         dialogs.Add(message);
+                        storage.Append(_conn.ConversationId, message);
 
                         foreach (var hook in hookProvider.HooksOrderByPriority)
                         {
@@ -174,6 +180,7 @@ await _completer.Connect(_conn,
             {
                 // append input audio transcript to conversation
                 dialogs.Add(message);
+                storage.Append(_conn.ConversationId, message);
 
                 foreach (var hook in hookProvider.HooksOrderByPriority)
                 {
@@ -224,6 +231,9 @@ private async Task HandleUserDtmfReceived()
         };
         dialogs.Add(message);
 
+        var storage = _services.GetRequiredService<IConversationStorage>();
+        storage.Append(_conn.ConversationId, message);
+
         foreach (var hook in hookProvider.HooksOrderByPriority)
         {
             hook.SetAgent(agent)
@@ -239,14 +249,9 @@ private async Task HandleUserDtmfReceived()
 
     private async Task HandleUserDisconnected()
     {
-        // Save dialog history
-        var routing = _services.GetRequiredService<IRoutingService>();
-        var storage = _services.GetRequiredService<IConversationStorage>();
-        var dialogs = routing.Context.GetDialogs();
-        foreach (var item in dialogs)
-        {
-            storage.Append(_conn.ConversationId, item);
-        }
+        var convService = _services.GetRequiredService<IConversationService>();
+        var conversation = await convService.GetConversation(_conn.ConversationId);
+        await HookEmitter.Emit<IConversationHook>(_services, x => x.OnUserDisconnected(conversation));
     }
 
     private async Task SendEventToUser(WebSocket webSocket, object message)

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -77,5 +77,6 @@ public class InputAudioTranscription
     public string Language { get; set; } = "en";
 
     [JsonPropertyName("prompt")]
-    public string Prompt { get; set; }
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Prompt { get; set; }
 }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -2,7 +2,9 @@
 using BotSharp.Abstraction.Files.Utilities;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Options;
+using BotSharp.Abstraction.Realtime;
 using BotSharp.Abstraction.Realtime.Models;
+using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.OpenAI.Models.Realtime;
 using OpenAI.Chat;
@@ -42,7 +44,7 @@ public async Task Connect(RealtimeHubConnection conn,
         Action onModelReady,
         Action<string,string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
-        Action<string> onAudioTranscriptDone,
+        Action<string> onModelAudioTranscriptDone,
         Action<List<RoleDialogModel>> onModelResponseDone,
         Action<string> onConversationItemCreated,
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
@@ -64,7 +66,7 @@ public async Task Connect(RealtimeHubConnection conn,
                 onModelReady,
                 onModelAudioDeltaReceived,
                 onModelAudioResponseDone,
-                onAudioTranscriptDone,
+                onModelAudioTranscriptDone,
                 onModelResponseDone,
                 onConversationItemCreated,
                 onInputAudioTranscriptionCompleted,
@@ -125,10 +127,10 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
         Action onModelReady,
         Action<string,string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
-        Action<string> onAudioTranscriptDone,
+        Action<string> onModelAudioTranscriptDone,
         Action<List<RoleDialogModel>> onModelResponseDone,
         Action<string> onConversationItemCreated,
-        Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
+        Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
         Action onUserInterrupted)
     {
         var buffer = new byte[1024 * 32];
@@ -171,7 +173,7 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
                 _logger.LogInformation($"{response.Type}: {receivedText}");
                 var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
                 await Task.Delay(1000);
-                onAudioTranscriptDone(data.Transcript);
+                onModelAudioTranscriptDone(data.Transcript);
             }
             else if (response.Type == "response.audio.delta")
             {
@@ -201,8 +203,11 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             else if (response.Type == "conversation.item.input_audio_transcription.completed")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
-                var message = await OnInputAudioTranscriptionCompleted(conn, receivedText);
-                onInputAudioTranscriptionCompleted(message);
+                var message = await OnUserAudioTranscriptionCompleted(conn, receivedText);
+                if (!string.IsNullOrEmpty(message.Content))
+                {
+                    onUserAudioTranscriptionCompleted(message);
+                }
             }
             else if (response.Type == "input_audio_buffer.speech_started")
             {
@@ -309,6 +314,9 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDet
             return fn;
         }).ToArray();
 
+        var words = new List<string>();
+        HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
+
         var sessionUpdate = new
         {
             type = "session.update",
@@ -320,6 +328,7 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDet
                 {
                     Model = "whisper-1",
                     Language = "en",
+                    Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
                 },
                 Voice = "alloy",
                 Instructions = instruction,
@@ -663,7 +672,7 @@ await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.don
         return outputs;
     }
 
-    public async Task<RoleDialogModel> OnInputAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
+    public async Task<RoleDialogModel> OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response)
     {
         var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(response);
         return new RoleDialogModel(AgentRole.User, data.Transcript)

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs

@@ -45,7 +45,7 @@ public async Task<bool> Execute(RoleDialogModel message)
                 pathSid: callSid
             );
 
-            message.Content = "The call has ended.";
+            message.Content = "The call has been ended.";
             message.StopCompletion = true;
         });