Merge pull request #946 from hchen2020/master

Optimize phone hang-up

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -23,7 +23,7 @@ Task Connect(RealtimeHubConnection conn,
     Task Disconnect();
 
     Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
-    Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDetection = true);
+    Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true);
     Task InsertConversationItem(RoleDialogModel message);
     Task RemoveConversationItem(string itemId);
     Task TriggerModelInference(string? instructions = null);

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class ModelTurnDetection
+{
+    public int PrefixPadding { get; set; } = 300;
+
+    public int SilenceDuration { get; set; } = 800;
+
+    public float Threshold { get; set; } = 0.8f;
+}

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs

@@ -0,0 +1,8 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class RealtimeModelSettings
+{
+    public float Temperature { get; set; } = 0.8f;
+    public int MaxResponseOutputTokens { get; set; } = 512;
+    public ModelTurnDetection TurnDetection { get; set; } = new();
+}

src/Infrastructure/BotSharp.Core.Realtime/RealtimePlugin.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Plugins;
+using BotSharp.Abstraction.Settings;
 using BotSharp.Core.Realtime.Hooks;
 using BotSharp.Core.Realtime.Services;
 using Microsoft.Extensions.Configuration;
@@ -14,6 +15,12 @@ public class RealtimePlugin : IBotSharpPlugin
 
     public void RegisterDI(IServiceCollection services, IConfiguration config)
     {
+        services.AddScoped(provider =>
+        {
+            var settingService = provider.GetRequiredService<ISettingService>();
+            return settingService.Bind<RealtimeModelSettings>("RealtimeModel");
+        });
+
         services.AddScoped<IRealtimeHub, RealtimeHub>();
         services.AddScoped<IConversationHook, RealtimeConversationHook>();
     }

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -98,15 +98,18 @@ private async Task ConnectToModel(WebSocket userWebSocket)
         await _completer.Connect(_conn, 
             onModelReady: async () => 
             {
-                if (states.ContainsState("init_audio_file"))
+                // Not TriggerModelInference, waiting for user utter.
+                await _completer.UpdateSession(_conn);
+
+                // Push dialogs into model context
+                foreach (var message in dialogs)
                 {
-                    await _completer.UpdateSession(_conn, turnDetection: true);
+                    await _completer.InsertConversationItem(message);
                 }
-                else
-                {
-                    // Control initial session, prevent initial response interruption
-                    await _completer.UpdateSession(_conn, turnDetection: false);
 
+                // Trigger model inference if there is no audio file in the conversation
+                if (!states.ContainsState("init_audio_file"))
+                {
                     if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
                     {
                         await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
@@ -115,10 +118,6 @@ await _completer.Connect(_conn,
                     {
                         await _completer.TriggerModelInference("Reply based on the conversation context.");
                     }
-
-                    // Start turn detection
-                    await Task.Delay(1000 * 8);
-                    await _completer.UpdateSession(_conn, turnDetection: true);
                 }
             },
             onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -52,6 +52,9 @@ public class RealtimeSessionBody
 
 public class RealtimeSessionTurnDetection
 {
+    [JsonPropertyName("interrupt_response")]
+    public bool InterruptResponse { get; set; } = true;
+
     /// <summary>
     /// Milliseconds
     /// </summary>

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -290,7 +290,7 @@ public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogMod
         return session;
     }
 
-    public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDetection = true)
+    public async Task<string> UpdateSession(RealtimeHubConnection conn, bool interruptResponse = true)
     {
         var convService = _services.GetRequiredService<IConversationService>();
         var conv = await convService.GetConversation(conn.ConversationId);
@@ -317,6 +317,8 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDet
         var words = new List<string>();
         HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
 
+        var realitmeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
+
         var sessionUpdate = new
         {
             type = "session.update",
@@ -335,22 +337,18 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn, bool turnDet
                 ToolChoice = "auto",
                 Tools = functions,
                 Modalities = [ "text", "audio" ],
-                Temperature = Math.Max(options.Temperature ?? 0f, 0.6f),
-                MaxResponseOutputTokens = 512,
+                Temperature = Math.Max(options.Temperature ?? realitmeModelSettings.Temperature, 0.6f),
+                MaxResponseOutputTokens = realitmeModelSettings.MaxResponseOutputTokens,
                 TurnDetection = new RealtimeSessionTurnDetection
                 {
-                    Threshold = 0.9f,
-                    PrefixPadding = 300,
-                    SilenceDuration = 800
+                    InterruptResponse = interruptResponse,
+                    Threshold = realitmeModelSettings.TurnDetection.Threshold,
+                    PrefixPadding = realitmeModelSettings.TurnDetection.PrefixPadding,
+                    SilenceDuration = realitmeModelSettings.TurnDetection.SilenceDuration
                 }
             }
         };
 
-        if (!turnDetection)
-        {
-            sessionUpdate.session.TurnDetection = null;
-        }
-
         await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
         {
             await hook.OnSessionUpdated(agent, instruction, functions);

src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj

@@ -19,12 +19,6 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-outbound_phone_call.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-hangup_phone_call.fn.liquid">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-twilio-outbound_phone_call.fn.liquid">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
   </ItemGroup>
 
   <ItemGroup>
@@ -39,4 +33,8 @@
     <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\" />
+  </ItemGroup>
+
 </Project>

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs

@@ -429,6 +429,15 @@ public async Task<FileContentResult> GetSpeechFile([FromRoute] string conversati
         return result;
     }
 
+    [ValidateRequest]
+    [HttpPost("twilio/voice/hang-up")]
+    public async Task<TwiMLResult> Hangup(ConversationalVoiceRequest request)
+    {
+        var twilio = _services.GetRequiredService<TwilioService>();
+        var response = twilio.HangUp("twilio/bye.mp3");
+        return TwiML(response);
+    }
+
     [ValidateRequest]
     [HttpPost("twilio/voice/status")]
     public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs

@@ -1,28 +1,34 @@
+using BotSharp.Abstraction.Routing;
 using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
 using Twilio.Rest.Api.V2010.Account;
-using Task = System.Threading.Tasks.Task;
 
 namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
 
 public class HangupPhoneCallFn : IFunctionCallback
 {
     private readonly IServiceProvider _services;
     private readonly ILogger<HangupPhoneCallFn> _logger;
+    private readonly TwilioSetting _twilioSetting;
 
     public string Name => "util-twilio-hangup_phone_call";
     public string Indication => "Hangup";
 
     public HangupPhoneCallFn(
         IServiceProvider services,
-        ILogger<HangupPhoneCallFn> logger)
+        ILogger<HangupPhoneCallFn> logger,
+        TwilioSetting twilioSetting)
     {
         _services = services;
         _logger = logger;
+        _twilioSetting = twilioSetting;
     }
 
     public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
+
+        var routing = _services.GetRequiredService<IRoutingService>();
+        var conversationId = routing.Context.ConversationId;
         var states = _services.GetRequiredService<IConversationStateService>();
         var callSid = states.GetState("twilio_call_sid");
 
@@ -33,20 +39,20 @@ public async Task<bool> Execute(RoleDialogModel message)
             return false;
         }
 
-        message.Content = args.GoodbyeMessage;
-
-        _ = Task.Run(async () =>
+        if (args.AnythingElseToHelp)
+        {
+            message.Content = "Tell me how I can help.";
+        }
+        else
         {
-            await Task.Delay(args.GoodbyeMessage.Split(' ').Length * 400);
-            // Have to find the SID by the phone number
             var call = CallResource.Update(
-                status: CallResource.UpdateStatusEnum.Completed,
+                url: new Uri($"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?conversation-id={conversationId}"),
                 pathSid: callSid
             );
 
-            message.Content = "The call has been ended.";
+            message.Content = "The call is ending.";
             message.StopCompletion = true;
-        });
+        }
 
         return true;
     }