Merge branch 'master' of https://github.com/SciSharp/BotSharp

Jicheng Lu · Jicheng Lu · commit eec704cce345 · 2025-03-13T15:36:25.000-05:00
diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/WebBrowsingSettings.cs
@@ -7,4 +7,6 @@ public class WebBrowsingSettings
     // Default timeout in milliseconds
     public float DefaultTimeout { get; set; } = 30000;
     public bool IsEnableScreenshot { get; set; }
+    // Default wait time in seconds after page is opened
+    public int DefaultWaitTime { get; set; } = 5;
 }
diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
@@ -26,7 +26,6 @@ public async Task Listen(WebSocket userWebSocket,
     {
         var buffer = new byte[1024 * 16];
         WebSocketReceiveResult result;
-
         
 
         do
@@ -98,24 +97,33 @@ private async Task ConnectToModel(WebSocket userWebSocket)
 
         routing.Context.SetDialogs(dialogs);
 
+        var states = _services.GetRequiredService<IConversationStateService>();
+
         await _completer.Connect(_conn, 
             onModelReady: async () => 
             {
-                // Control initial session, prevent initial response interruption
-                await _completer.UpdateSession(_conn, turnDetection: false);
-
-                if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
+                if (states.ContainsState("init_audio_file"))
                 {
-                    await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
+                    await _completer.UpdateSession(_conn, turnDetection: true);
                 }
                 else
                 {
-                    await _completer.TriggerModelInference("Reply based on the conversation context.");
-                }
+                    // Control initial session, prevent initial response interruption
+                    await _completer.UpdateSession(_conn, turnDetection: false);
 
-                // Start turn detection
-                await Task.Delay(1000 * 8);
-                await _completer.UpdateSession(_conn, turnDetection: true);
+                    if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
+                    {
+                        await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
+                    }
+                    else
+                    {
+                        await _completer.TriggerModelInference("Reply based on the conversation context.");
+                    }
+
+                    // Start turn detection
+                    await Task.Delay(1000 * 8);
+                    await _completer.UpdateSession(_conn, turnDetection: true);
+                }
             },
             onModelAudioDeltaReceived: async (audioDeltaData, itemId) =>
             {
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
@@ -35,21 +35,26 @@ public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceReq
             throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
         }
 
-        VoiceResponse response = null;
+        VoiceResponse response = default!;
+
+        if (request.AnsweredBy == "machine_start" &&
+            request.Direction == "outbound-api" &&
+            request.InitAudioFile != null)
+        {
+            response = new VoiceResponse();
+            response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{request.ConversationId}/{request.InitAudioFile}"));
+            return TwiML(response);
+        }
+
         var instruction = new ConversationalVoiceResponse
         {
             SpeechPaths = [],
             ActionOnEmptyResult = true
         };
 
-        if (_context.HttpContext.Request.Query.ContainsKey("init_audio_file"))
+        if (request.InitAudioFile != null)
         {
-            instruction.SpeechPaths.Add(_context.HttpContext.Request.Query["init_audio_file"]);
-        }
-
-        if (_context.HttpContext.Request.Query.ContainsKey("conversation_id"))
-        {
-            request.ConversationId = _context.HttpContext.Request.Query["conversation_id"];
+            instruction.SpeechPaths.Add(request.InitAudioFile);
         }
 
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
@@ -77,6 +82,24 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
         return TwiML(response);
     }
 
+    [ValidateRequest]
+    [HttpPost("twilio/stream/status")]
+    public async Task<ActionResult> StreamConversationStatus(ConversationalVoiceRequest request)
+    {
+        if (request.AnsweredBy == "machine_start" &&
+            request.Direction == "outbound-api" &&
+            request.InitAudioFile != null &&
+            request.CallStatus == "completed")
+        {
+            // voicemail
+            await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
+            {
+                await hook.OnVoicemailLeft(request.ConversationId);
+            });
+        }
+        return Ok();
+    }
+
     private async Task<string> InitConversation(ConversationalVoiceRequest request)
     {
         var convService = _services.GetRequiredService<IConversationService>();
@@ -104,6 +127,11 @@ private async Task<string> InitConversation(ConversationalVoiceRequest request)
                 new(StateConst.ROUTING_MODE, "lazy"),
             };
 
+        if (request.InitAudioFile != null)
+        {
+            states.Add(new("init_audio_file", request.InitAudioFile));
+        }
+
         convService.SetConversationId(conversation.Id, states);
         convService.SaveStates();
 
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
@@ -66,7 +66,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
         });
 
         request.ConversationId = $"TwilioVoice_{request.CallSid}";
-        instruction.CallbackPath = $"twilio/voice/{request.ConversationId}/receive/0?{GenerateStatesParameter(request.States)}";
+        instruction.CallbackPath = $"twilio/voice/receive/0?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}";
 
         var twilio = _services.GetRequiredService<TwilioService>();
         if (string.IsNullOrWhiteSpace(request.Intent))
@@ -89,7 +89,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
             };
             await messageQueue.EnqueueAsync(callerMessage);
             response = new VoiceResponse();
-            response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{seqNum}?{GenerateStatesParameter(request.States)}"), HttpMethod.Post);
+            response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}"), HttpMethod.Post);
         }
 
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
@@ -109,7 +109,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
     /// <param name="request"></param>
     /// <returns></returns>
     [ValidateRequest]
-    [HttpPost("twilio/voice/{conversationId}/receive/{seqNum}")]
+    [HttpPost("twilio/voice/receive/{seqNum}")]
     public async Task<TwiMLResult> ReceiveCallerMessage(ConversationalVoiceRequest request)
     {
         var twilio = _services.GetRequiredService<TwilioService>();
@@ -142,7 +142,7 @@ public async Task<TwiMLResult> ReceiveCallerMessage(ConversationalVoiceRequest r
             await messageQueue.EnqueueAsync(callerMessage);
 
             response = new VoiceResponse();
-            response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post);
+            response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime=0"), HttpMethod.Post);
 
             await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
             {
@@ -173,7 +173,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
                 var instruction = new ConversationalVoiceResponse
                 {
                     SpeechPaths = new List<string>(),
-                    CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{request.SeqNum}?{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
+                    CallbackPath = $"twilio/voice/receive/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&attempts={++request.Attempts}",
                     ActionOnEmptyResult = true
                 };
 
@@ -203,7 +203,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
     /// <param name="request"></param>
     /// <returns></returns>
     [ValidateRequest]
-    [HttpPost("twilio/voice/{conversationId}/reply/{seqNum}")]
+    [HttpPost("twilio/voice/reply/{seqNum}")]
     public async Task<TwiMLResult> ReplyCallerMessage(ConversationalVoiceRequest request)
     {
         var nextSeqNum = request.SeqNum + 1;
@@ -276,7 +276,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
                 var instruction = new ConversationalVoiceResponse
                 {
                     SpeechPaths = speechPaths,
-                    CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
+                    CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
                     ActionOnEmptyResult = true
                 };
 
@@ -315,7 +315,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
                 var instruction = new ConversationalVoiceResponse
                 {
                     SpeechPaths = instructions,
-                    CallbackPath = $"twilio/voice/{request.ConversationId}/reply/{request.SeqNum}?{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
+                    CallbackPath = $"twilio/voice/reply/{request.SeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}&AIResponseWaitTime={++request.AIResponseWaitTime}",
                     ActionOnEmptyResult = true
                 };
 
@@ -361,7 +361,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
                 var instruction = new ConversationalVoiceResponse
                 {
                     SpeechPaths = [$"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"],
-                    CallbackPath = $"twilio/voice/{request.ConversationId}/receive/{nextSeqNum}?{GenerateStatesParameter(request.States)}",
+                    CallbackPath = $"twilio/voice/receive/{nextSeqNum}?conversation-id={request.ConversationId}&{GenerateStatesParameter(request.States)}",
                     ActionOnEmptyResult = true,
                     Hints = reply.Hints
                 };
@@ -388,7 +388,7 @@ public TwiMLResult InitiateOutboundCall(VoiceRequest request, [Required][FromQue
         var instruction = new ConversationalVoiceResponse
         {
             ActionOnEmptyResult = true,
-            CallbackPath = $"twilio/voice/{conversationId}/receive/1",
+            CallbackPath = $"twilio/voice/receive/1?conversation-id={conversationId}",
             SpeechPaths = new List<string>
             {
                 $"twilio/voice/speeches/{conversationId}/intial.mp3"
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs
@@ -0,0 +1,8 @@
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Interfaces;
+
+public interface ITwilioCallStatusHook
+{
+    Task OnVoicemailLeft(string conversationId);
+}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs
@@ -7,7 +7,7 @@ public class ConversationalVoiceRequest : VoiceRequest
     [FromQuery(Name = "agent-id")]
     public string AgentId { get; set; } = string.Empty;
 
-    [FromRoute]
+    [FromQuery(Name = "conversation-id")]
     public string ConversationId { get; set; } = string.Empty;
 
     [FromRoute]
@@ -19,5 +19,26 @@ public class ConversationalVoiceRequest : VoiceRequest
 
     public string Intent { get; set; } = string.Empty;
 
+    [FromQuery(Name = "init-audio-file")]
+    public string? InitAudioFile { get; set; }
+
     public List<string> States { get; set; } = [];
+
+    [FromForm]
+    public string? CallbackSource { get; set; }
+
+    /// <summary>
+    /// machine_start
+    /// </summary>
+    [FromForm]
+    public string? AnsweredBy { get; set; }
+
+    [FromForm]
+    public int MachineDetectionDuration { get; set; }
+
+    [FromForm]
+    public int Duration { get; set; }
+
+    [FromForm]
+    public int CallDuration { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
@@ -66,9 +66,12 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         // Make outbound call
         var call = await CallResource.CreateAsync(
-            url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation_id={newConversationId}&init_audio_file={fileName}"),
+            url: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream?conversation-id={newConversationId}&init-audio-file={fileName}"),
             to: new PhoneNumber(args.PhoneNumber),
-            from: new PhoneNumber(_twilioSetting.PhoneNumber));
+            from: new PhoneNumber(_twilioSetting.PhoneNumber),
+            statusCallback: new Uri($"{_twilioSetting.CallbackHost}/twilio/stream/status?conversation-id={newConversationId}&init-audio-file={fileName}"),
+            // https://www.twilio.com/docs/voice/answering-machine-detection
+            machineDetection: "Enable");
 
         var convService = _services.GetRequiredService<IConversationService>();
         var routing = _services.GetRequiredService<IRoutingContext>();
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
@@ -199,6 +199,7 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversa
                 }
             }
         }
+
         var connect = new Connect();
         var host = _settings.CallbackHost.Split("://").Last();
         connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}");
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs
@@ -77,7 +77,8 @@ public async Task<IBrowserContext> InitContext(string ctxId, BrowserActionArgs a
                 Args =
                 [
                     "--disable-infobars",
-                    "--test-type"
+                    "--test-type",
+                    "--no-sandbox"
                     // "--start-maximized"
                 ]
             });
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs
@@ -26,7 +26,7 @@ public async Task<bool> Execute(RoleDialogModel message)
         var _webDriver = _services.GetRequiredService<WebBrowsingSettings>();
         args.Timeout = _webDriver.DefaultTimeout;
         args.WaitForNetworkIdle = false;
-        args.WaitTime = 5;
+        args.WaitTime = _webDriver.DefaultWaitTime;
         args.OpenNewTab = true;
 
         var conv = _services.GetRequiredService<IConversationService>();

Original file line number	Diff line number	Diff line change
`@@ -7,4 +7,6 @@ public class WebBrowsingSettings`
`7`	`7`	`// Default timeout in milliseconds`
`8`	`8`	`public float DefaultTimeout { get; set; } = 30000;`
`9`	`9`	`public bool IsEnableScreenshot { get; set; }`
	`10`	`+ // Default wait time in seconds after page is opened`
	`11`	`+ public int DefaultWaitTime { get; set; } = 5;`
`10`	`12`	`}`
Original file line number	Diff line number	Diff line change
`@@ -199,6 +199,7 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversa`
`199`	`199`	`}`
`200`	`200`	`}`
`201`	`201`	`}`
	`202`	`+`
`202`	`203`	`var connect = new Connect();`
`203`	`204`	`var host = _settings.CallbackHost.Split("://").Last();`
`204`	`205`	`connect.Stream(url: $"wss://{host}/twilio/stream/{conversationId}");`
Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,8 @@ public async Task<IBrowserContext> InitContext(string ctxId, BrowserActionArgs a`
`77`	`77`	`Args =`
`78`	`78`	`[`
`79`	`79`	`"--disable-infobars",`
`80`		`- "--test-type"`
	`80`	`+ "--test-type",`
	`81`	`+ "--no-sandbox"`
`81`	`82`	`// "--start-maximized"`
`82`	`83`	`]`
`83`	`84`	`});`