different tts method

Josh-XT · Josh-XT · commit 4d27a934bae0 · 2025-12-24T23:01:14.000-05:00
diff --git a/android/app/proguard-rules.pro b/android/app/proguard-rules.pro
@@ -22,4 +22,8 @@
 -dontwarn com.google.android.play.core.splitinstall.SplitInstallStateUpdatedListener
 -dontwarn com.google.android.play.core.tasks.OnFailureListener
 -dontwarn com.google.android.play.core.tasks.OnSuccessListener
--dontwarn com.google.android.play.core.tasks.Task
+-dontwarn com.google.android.play.core.tasks.Task
+
+# Vosk speech recognition (JNA required)
+-keep class com.sun.jna.* { *; }
+-keepclassmembers class * extends com.sun.jna.* { public *; }
diff --git a/android/app/src/main/kotlin/dev/agixt/agixt/MainActivity.kt b/android/app/src/main/kotlin/dev/agixt/agixt/MainActivity.kt
@@ -133,7 +133,7 @@ class MainActivity: FlutterActivity() {
      override fun onDestroy() {
         super.onDestroy()
         // Clean up handlers
-        wakeWordHandler?.destroy()
+        // wakeWordHandler?.destroy()  // Now using Vosk in Flutter
         watchHandler?.destroy()
         BackgroundService.stopService(this@MainActivity, null)
     }
@@ -205,10 +205,12 @@ class MainActivity: FlutterActivity() {
         // Mark that the method channels are initialized
         methodChannelInitialized = true
         
-        // Initialize Voice & Watch handlers
-        wakeWordHandler = WakeWordHandler(this, binaryMessenger)
-        wakeWordHandler?.initialize()
+        // Wake word detection is now handled by Vosk in Flutter (pure Dart)
+        // Native WakeWordHandler is no longer needed
+        // wakeWordHandler = WakeWordHandler(this, binaryMessenger)
+        // wakeWordHandler?.initialize()
         
+        // Initialize Watch handler for Pixel Watch support
         watchHandler = WatchHandler(this, binaryMessenger)
         watchHandler?.initialize()
         
diff --git a/lib/models/agixt/widgets/agixt_chat.dart b/lib/models/agixt/widgets/agixt_chat.dart
@@ -1,3 +1,4 @@
+import 'dart:async';
 import 'dart:convert';
 import 'package:agixt/models/agixt/auth/auth.dart';
 import 'package:agixt/models/agixt/calendar.dart';
@@ -12,6 +13,7 @@ import 'package:agixt/services/secure_storage_service.dart';
 import 'package:agixt/services/location_service.dart'; // Import LocationService
 import 'package:agixt/services/client_commands_service.dart'; // Import ClientSideTools
 import 'package:device_calendar/device_calendar.dart';
+import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:geolocator/geolocator.dart'; // Import Geolocator
 import 'package:hive/hive.dart';
@@ -232,6 +234,156 @@ class AGiXTChatWidget implements AGiXTWidget {
     }
   }
 
+  /// Send chat message with streaming response
+  /// Returns a stream of response chunks as they arrive
+  Stream<String> sendChatMessageStreaming(String message) async* {
+    try {
+      final jwt = await AuthService.getJwt();
+      if (jwt == null) {
+        yield "Please login to use AGiXT chat.";
+        return;
+      }
+
+      final conversationId = await _getCurrentConversationId();
+      debugPrint('Using conversation ID for streaming chat: $conversationId');
+
+      // Build context data with timeout
+      String contextData = '';
+      try {
+        contextData = await _buildContextData().timeout(
+          const Duration(seconds: 3),
+          onTimeout: () {
+            debugPrint('Context building timed out');
+            return '';
+          },
+        );
+      } catch (e) {
+        debugPrint('Error building context data: $e');
+      }
+
+      // Get available tools
+      final availableTools = await ClientSideTools.getToolDefinitions();
+
+      final Map<String, dynamic> requestBody = {
+        "model": await _getAgentName(),
+        "messages": [
+          {
+            "role": "user",
+            "content": message,
+            if (contextData.isNotEmpty) "context": contextData,
+          },
+        ],
+        "user": conversationId,
+        "stream": true, // Enable streaming!
+        if (availableTools.isNotEmpty) "tools": availableTools,
+        if (availableTools.isNotEmpty) "tool_choice": "auto",
+      };
+
+      // Create streaming request
+      final client = http.Client();
+      final request = http.Request(
+        'POST',
+        Uri.parse('${AuthService.serverUrl}/v1/chat/completions'),
+      );
+      request.headers.addAll({
+        'Content-Type': 'application/json',
+        'Authorization': 'Bearer $jwt',
+        'Accept': 'text/event-stream',
+      });
+      request.body = jsonEncode(requestBody);
+
+      debugPrint('Sending streaming chat request...');
+      final streamedResponse = await client.send(request);
+
+      if (streamedResponse.statusCode == 200) {
+        String buffer = '';
+        String fullResponse = '';
+        String? newConversationId;
+
+        await for (final chunk in streamedResponse.stream.transform(utf8.decoder)) {
+          buffer += chunk;
+          
+          // Process complete SSE events (lines starting with "data: ")
+          while (buffer.contains('\n')) {
+            final newlineIndex = buffer.indexOf('\n');
+            final line = buffer.substring(0, newlineIndex).trim();
+            buffer = buffer.substring(newlineIndex + 1);
+
+            if (line.isEmpty) continue;
+            if (!line.startsWith('data: ')) continue;
+
+            final data = line.substring(6); // Remove "data: " prefix
+            
+            // Check for stream end
+            if (data == '[DONE]') {
+              debugPrint('Stream complete');
+              continue;
+            }
+
+            try {
+              final jsonData = jsonDecode(data);
+              
+              // Extract conversation ID from first chunk
+              if (newConversationId == null && jsonData['id'] != null) {
+                newConversationId = jsonData['id'].toString();
+                debugPrint('Got conversation ID from stream: $newConversationId');
+              }
+
+              // Extract content delta
+              if (jsonData['choices'] != null &&
+                  jsonData['choices'].isNotEmpty) {
+                final delta = jsonData['choices'][0]['delta'];
+                if (delta != null && delta['content'] != null) {
+                  final content = delta['content'].toString();
+                  fullResponse += content;
+                  yield content; // Yield each chunk as it arrives
+                }
+              }
+            } catch (e) {
+              // Skip malformed JSON chunks
+              debugPrint('Error parsing SSE chunk: $e');
+            }
+          }
+        }
+
+        // Save conversation ID after stream completes
+        if (newConversationId != null && newConversationId != '-') {
+          final cookieManager = CookieManager();
+          await cookieManager.saveAgixtConversationId(newConversationId);
+          debugPrint('Saved conversation ID: $newConversationId');
+          _navigateToConversation(newConversationId, jwt);
+        }
+
+        // Save the full interaction
+        if (fullResponse.isNotEmpty) {
+          await _saveInteraction(message, fullResponse);
+        }
+
+        client.close();
+      } else if (streamedResponse.statusCode == 401) {
+        await SessionManager.clearSession();
+        yield "Authentication expired. Please login again.";
+        client.close();
+      } else {
+        debugPrint('Streaming API Error: ${streamedResponse.statusCode}');
+        yield "Error: Unable to get response (${streamedResponse.statusCode})";
+        client.close();
+      }
+    } catch (e) {
+      debugPrint('Streaming chat error: $e');
+      yield "An error occurred while connecting to AGiXT.";
+    }
+  }
+
+  /// Convenience method to collect full streaming response as a single string
+  Future<String?> sendChatMessageStreamingFull(String message) async {
+    final buffer = StringBuffer();
+    await for (final chunk in sendChatMessageStreaming(message)) {
+      buffer.write(chunk);
+    }
+    return buffer.isEmpty ? null : buffer.toString();
+  }
+
   // Navigate to the conversation in the WebView
   Future<void> _navigateToConversation(
     String conversationId,
diff --git a/lib/services/ai_service.dart b/lib/services/ai_service.dart
@@ -93,14 +93,38 @@ class AIService {
 
   /// Handle wake word events
   void _handleWakeWordEvent(WakeWordEvent event) {
-    if (event.type == WakeWordEventType.detected) {
-      debugPrint('AIService: Wake word detected from ${event.source}');
-      // Provide haptic feedback
-      _playWakeWordFeedback();
-      // Start voice recording when wake word is detected
-      if (!_isProcessing) {
-        _startVoiceRecording();
-      }
+    switch (event.type) {
+      case WakeWordEventType.detected:
+        debugPrint('AIService: Wake word detected from ${event.source}');
+        // Provide haptic feedback
+        _playWakeWordFeedback();
+        // Start voice recording when wake word is detected
+        if (!_isProcessing) {
+          _startVoiceRecording();
+        }
+        break;
+
+      case WakeWordEventType.modelDownloadStarted:
+        debugPrint('AIService: Wake word model download started');
+        _showInfoMessage('Downloading speech model for wake word...');
+        break;
+
+      case WakeWordEventType.modelDownloadProgress:
+        final progress = ((event.progress ?? 0) * 100).toInt();
+        debugPrint('AIService: Wake word model download progress: $progress%');
+        break;
+
+      case WakeWordEventType.modelDownloadComplete:
+        debugPrint('AIService: Wake word model download complete');
+        _showInfoMessage('Wake word ready! Say "computer" to activate.');
+        break;
+
+      case WakeWordEventType.error:
+        debugPrint('AIService: Wake word error: ${event.error}');
+        break;
+
+      default:
+        break;
     }
   }
 
@@ -171,12 +195,22 @@ class AIService {
 
       debugPrint('AIService: Transcription: $transcription');
 
-      // Send transcribed text to AGiXT chat
-      final response = await _chatWidget.sendChatMessage(transcription);
+      // Send transcribed text to AGiXT chat with streaming for better responsiveness
+      final responseBuffer = StringBuffer();
+      
+      await for (final chunk in _chatWidget.sendChatMessageStreaming(transcription)) {
+        responseBuffer.write(chunk);
+        
+        // Stream chunks to connected devices as they arrive
+        // For glasses and watch, we accumulate and send at reasonable intervals
+        // For TTS, we'll wait for complete sentences
+      }
+      
+      final fullResponse = responseBuffer.toString();
 
-      if (response != null && response.isNotEmpty) {
-        // Output response based on connected devices
-        await _outputResponse(response);
+      if (fullResponse.isNotEmpty) {
+        // Output the full response (TTS and final display)
+        await _outputResponse(fullResponse);
       } else {
         await _showErrorMessage('No response from AGiXT');
       }
@@ -410,16 +444,22 @@ class AIService {
     }
   }
 
-  // Send message to AGiXT API and display response
+  // Send message to AGiXT API and display response (with streaming)
   Future<void> _sendMessageToAGiXT(String message) async {
     try {
       // Show sending message using AI response method
       await _bluetoothManager.sendAIResponse('Sending to AGiXT: "$message"');
 
-      // Get response using the AGiXTChatWidget
-      final response = await _chatWidget.sendChatMessage(message);
+      // Stream response from AGiXT for better responsiveness
+      final responseBuffer = StringBuffer();
+      
+      await for (final chunk in _chatWidget.sendChatMessageStreaming(message)) {
+        responseBuffer.write(chunk);
+      }
+      
+      final response = responseBuffer.toString();
 
-      if (response != null && response.isNotEmpty) {
+      if (response.isNotEmpty) {
         // Output response to appropriate devices
         await _outputResponse(response);
       } else {
@@ -516,6 +556,13 @@ class AIService {
     }
   }
 
+  Future<void> _showInfoMessage(String message) async {
+    await _bluetoothManager.sendAIResponse(message);
+    if (_watchService.isConnected) {
+      await _watchService.displayMessage(message, durationMs: 5000);
+    }
+  }
+
   /// Check if AIService is in background mode
   bool get isBackgroundMode => _isBackgroundMode;
 
diff --git a/lib/services/wake_word_service.dart b/lib/services/wake_word_service.dart
diff --git a/pubspec.yaml b/pubspec.yaml
