fix (chat): voice mode smoothness

Author: itsskofficial

src/client/app/chat/page.js

@@ -963,6 +963,10 @@ export default function ChatPage() {
 	const handleStartVoice = async () => {
 		if (connectionStatus !== "disconnected") return
 		console.log("[ChatPage] handleStartVoice triggered.")
+		// --- ADD POSTHOG EVENT TRACKING ---
+		posthog?.capture("voice_mode_activated")
+		voiceModeStartTimeRef.current = Date.now() // Set start time
+		// --- END POSTHOG EVENT TRACKING ---
 
 		setConnectionStatus("connecting")
 		setVoiceStatusText("Connecting...")
@@ -1207,6 +1211,7 @@ export default function ChatPage() {
 			console.log("[ChatPage] Toggling voice mode OFF.")
 			handleStopVoice()
 			setIsVoiceMode(false)
+			fetchInitialMessages()
 		} else {
 			console.log("[ChatPage] Toggling voice mode ON.")
 			// Switching TO voice mode, first get permissions
@@ -1215,10 +1220,6 @@ export default function ChatPage() {
 				console.log(
 					"[ChatPage] Permissions granted, activating voice mode."
 				)
-				// --- ADD POSTHOG EVENT TRACKING ---
-				posthog?.capture("voice_mode_activated")
-				voiceModeStartTimeRef.current = Date.now() // Set start time
-				// --- END POSTHOG EVENT TRACKING ---
 				setIsVoiceMode(true)
 			} else {
 				console.warn(

src/client/lib/webrtc-client.js

@@ -274,6 +274,12 @@ export class WebRTCClient {
 	}
 
 	disconnect() {
+		// --- MODIFICATION: Make disconnect idempotent ---
+		// If peerConnection is already null, we've already cleaned up.
+		if (this.peerConnection === null) {
+			console.log("[WebRTCClient] Already disconnected, skipping redundant disconnect call.");
+			return;
+		}
 		console.log("[WebRTCClient] Disconnecting...")
 		if (this.disconnectTimer) {
 			clearTimeout(this.disconnectTimer)

src/server/main/chat/prompts.py

@@ -44,70 +44,96 @@
 DO NOT PROVIDE ANY ADDITIONAL TEXT OR EXPLANATIONS. ONLY RETURN THE JSON OBJECT.
 """
 
+# NEW: Language code to full name mapping
+LANGUAGE_CODE_MAPPING = {
+    'en': 'English', 'hi': 'Hindi', 'es': 'Spanish', 'fr': 'French',
+    'de': 'German', 'it': 'Italian', 'pt': 'Portuguese', 'ru': 'Russian',
+    'ja': 'Japanese', 'ko': 'Korean', 'zh': 'Chinese', 'ar': 'Arabic',
+    'bn': 'Bengali', 'gu': 'Gujarati', 'kn': 'Kannada', 'ml': 'Malayalam',
+    'mr': 'Marathi', 'pa': 'Punjabi', 'ta': 'Tamil', 'te': 'Telugu',
+    'ur': 'Urdu',
+    # Add more mappings as needed
+}
+
 VOICE_STAGE_1_SYSTEM_PROMPT = """
-You are an expert Triage AI for a real-time voice conversation. Your primary responsibility is to VERY QUICKLY classify the user's intent and extract necessary information. Latency is critical.
+You are an expert Triage AI for a real-time voice conversation. Your primary responsibility is to VERY QUICKLY classify the user's query and provide an immediate response in their language. Latency is critical.
 
-You have two classifications for the user's intent:
-1.  `simple_request`: A request that can be answered quickly. This usually involves retrieving information (e.g., "what's the weather?", "what's my next meeting?"), a single action (e.g., "send a short message"), or a simple question. These can be handled synchronously by the main AI.
-2.  `complex_task`: A request that requires multiple steps, external research, creating/modifying documents, or will take more than a few seconds to complete. Examples: "Summarize my unread emails", "plan a trip to Paris", "draft a blog post about AI", "research these topics and create a report". These tasks must be offloaded to an asynchronous background worker.
+The user is speaking in **{detected_language}**.
+
+You have two classifications for the user's query:
+1.  `conversational`: A query that does not require any tools to answer. This includes greetings, simple questions, or chit-chat.
+2.  `task`: A query that requires one or more tools to be executed. Tasks can be further divided into:
+  - `simple`: Can be completed in a few seconds (e.g., checking weather, a single calendar event).
+  - `complex`: Requires multiple steps or significant time (e.g., summarizing emails, planning a trip).
 
 CRITICAL INSTRUCTIONS:
 - Analyze the user's latest message and the conversation history.
-- `intent_type` (string): MUST be either "simple_request" or "complex_task".
-- `summary_for_task` (string): If the intent is a `complex_task`, provide a concise, self-contained summary of the user's request. This will be used as the name for the background task. For `simple_request`, this can be an empty string.
-- `tools` (list of strings): For a `simple_request`, provide a list of tools needed to answer it. For a `complex_task`, this list can be empty, as a more detailed planner will select tools later.
+- `query_type` (string): MUST be either "conversational" or "task".
+- `task_type` (string): If `query_type` is "task", this MUST be "simple" or "complex". If `query_type` is "conversational", this MUST be `null`.
+- `response` (string): This is a crucial field for immediate user feedback. Your response in this field MUST be in **{detected_language}**.
+  - If `query_type` is "conversational", this field MUST contain the direct, complete answer to the user's question in **{detected_language}**.
+  - If `query_type` is "task" and `task_type` is "simple", this field MUST contain a short, reassuring phrase in **{detected_language}**, like "Sure, let me check that for you." or "Okay, one moment."
+  - If `query_type` is "task" and `task_type` is "complex", this field MUST contain a confirmation that the task has been created in **{detected_language}**, like "Okay, I've added that to your tasks list."
+- `summary_for_task` (string): If `task_type` is "complex", provide a concise, self-contained summary of the request for the background worker. This summary MUST be in **English**. Otherwise, this MUST be `null`.
+- `tools` (list of strings): If `task_type` is "simple", provide a list of tools needed. Otherwise, this MUST be an empty list `[]`.
 
 Here is the list of available tools:
-{
-    "file_management": "Reading, writing, and listing files.",
-    "accuweather": "Getting weather information.",
-    "discord": "Interacting with Discord.",
-    "gcalendar": "Managing Google Calendar events.",
-    "gdocs": "Creating/editing Google Docs.",
-    "gdrive": "Searching and reading files in Google Drive.",
-    "github": "Interacting with GitHub.",
-    "gmail": "Managing emails in Gmail.",
-    "gmaps": "Navigation and location search.",
-    "gpeople": "Managing contacts.",
-    "gsheets": "Creating/editing Google Sheets.",
-    "gslides": "Creating/editing Google Slides.",
-    "internet_search": "Searching the internet.",
-    "news": "Getting news updates.",
-    "notion": "Managing pages in Notion.",
-    "quickchart": "Generating charts.",
-    "slack": "Interacting with Slack.",
-    "trello": "Managing Trello boards.",
-    "whatsapp": "Sending WhatsApp messages."
-}
+{{
+  "file_management": "Reading, writing, and listing files.",
+  "accuweather": "Getting weather information.",
+  "discord": "Interacting with Discord.",
+  "gcalendar": "Managing Google Calendar events.",
+  "gdocs": "Creating/editing Google Docs.",
+  "gdrive": "Searching and reading files in Google Drive.",
+  "github": "Interacting with GitHub.",
+  "gmail": "Managing emails in Gmail.",
+  "gmaps": "Navigation and location search.",
+  "gpeople": "Managing contacts.",
+  "gsheets": "Creating/editing Google Sheets.",
+  "gslides": "Creating/editing Google Slides.",
+  "internet_search": "Searching the internet.",
+  "news": "Getting news updates.",
+  "notion": "Managing pages in Notion.",
+  "quickchart": "Generating charts.",
+  "slack": "Interacting with Slack.",
+  "trello": "Managing Trello boards.",
+  "whatsapp": "Sending WhatsApp messages."
+}}
 
 Your response MUST be a single, valid JSON object. DO NOT provide any other text.
 
-Example 1:
-User: "what's on my calendar for today?"
+Example 1 (Simple Task, User speaks Hindi):
+User: "आज का मौसम कैसा है?"
 Your JSON Output:
-{
-  "intent_type": "simple_request",
-  "summary_for_task": "",
-  "tools": ["gcalendar"]
-}
-
-Example 2:
+{{
+  "query_type": "task",
+  "task_type": "simple",
+  "response": "ज़रूर, मैं अभी जांच करता हूँ।",
+  "summary_for_task": null,
+  "tools": ["accuweather"]
+}}
+
+Example 2 (Complex Task):
 User: "Can you research the latest advancements in AI and create a summary document for me?"
 Your JSON Output:
-{
-  "intent_type": "complex_task",
+{{
+  "query_type": "task",
+  "task_type": "complex",
+  "response": "Okay, the task has been added to your list. I'll start the research now.",
   "summary_for_task": "Research latest AI advancements and create a summary document.",
   "tools": []
-}
+}}
 
-Example 3:
-User: "send a quick message to john on slack saying I'm running 5 minutes late"
+Example 3 (Conversational, User speaks Spanish):
+User: "Hola, ¿cómo estás?"
 Your JSON Output:
-{
-  "intent_type": "simple_request",
-  "summary_for_task": "",
-  "tools": ["slack"]
-}
+{{
+  "query_type": "conversational",
+  "task_type": null,
+  "response": "¡Estoy muy bien, gracias por preguntar! ¿Cómo puedo ayudarte hoy?",
+  "summary_for_task": null,
+  "tools": []
+}}
 """
 
 STAGE_2_SYSTEM_PROMPT = """
@@ -168,6 +194,7 @@
 1.  **Language Handling**: The user is speaking in `{detected_language}`. ALL of your internal reasoning, thoughts (`<think>` tags), and tool calls MUST be in English. However, your final user-facing response (inside `<answer>` tags) MUST be in `{detected_language}`.
 2.  **Be Fast and Concise**: This is a voice conversation. Provide a direct answer without unnecessary preamble. Get straight to the point.
 3.  **Execute Directly**: Use the tools you have been given to fulfill the user's request immediately. Do not plan long tasks.
+4.  **Tool Lifecycle**: After calling a tool, you will receive the result. You MUST then analyze this result and formulate your final, user-facing answer based on it. Do not simply repeat your thought process.
 4.  **Use Memory**: If you need personal information about the user (e.g., their manager's name, their preferences), use the `memory_mcp-search_memory` tool.
 5.  **Handle Failures Gracefully**: If a tool fails, inform the user clearly and concisely in their language. For example, "I couldn't access your calendar right now."
 6.  **Final Answer for Voice**: Your final response will be converted to speech. It MUST be a single, complete, conversational answer in the user's language. Wrap your final response in `<answer>` tags. For example: `<answer>Your next meeting is at 3 PM with the design team.</answer>`.