Skip to content

Commit 1742c78

Browse files
Josh-XTJosh-XT
committed
maybe fix streaming tts
1 parent 3765eee commit 1742c78

File tree

4 files changed

+160
-29
lines changed

4 files changed

+160
-29
lines changed

‎android/wear/build.gradle‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,22 @@ plugins {
44
id 'org.jetbrains.kotlin.plugin.compose' version '2.1.0'
55
}
66

7+
// Load keystore properties (shared with main app)
8+
def keystorePropertiesFile = rootProject.file("keystore.properties")
9+
def keystoreProperties = new Properties()
10+
def useReleaseKeys = false
11+
12+
if (keystorePropertiesFile.exists()) {
13+
keystoreProperties.load(new FileInputStream(keystorePropertiesFile))
14+
def storeFilePath = keystoreProperties['storeFile']
15+
if (storeFilePath) {
16+
def storeFile = rootProject.file(storeFilePath)
17+
if (storeFile.exists()) {
18+
useReleaseKeys = true
19+
}
20+
}
21+
}
22+
723
android {
824
namespace 'dev.agixt.wear'
925
compileSdk 34
@@ -16,9 +32,24 @@ android {
1632
versionName "1.0.0"
1733
}
1834

35+
signingConfigs {
36+
debug {
37+
// Use debug keys by default
38+
}
39+
release {
40+
if (useReleaseKeys) {
41+
keyAlias keystoreProperties['keyAlias']
42+
keyPassword keystoreProperties['keyPassword']
43+
storeFile rootProject.file(keystoreProperties['storeFile'])
44+
storePassword keystoreProperties['storePassword']
45+
}
46+
}
47+
}
48+
1949
buildTypes {
2050
release {
2151
minifyEnabled true
52+
signingConfig useReleaseKeys ? signingConfigs.release : signingConfigs.debug
2253
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
2354
}
2455
debug {

‎android/wear/src/main/kotlin/dev/agixt/wear/MainActivity.kt‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package dev.agixt.wear
22

33
import android.content.Intent
44
import android.os.Bundle
5+
import android.util.Log
56
import androidx.activity.ComponentActivity
67
import androidx.activity.compose.setContent
78
import androidx.compose.foundation.background
@@ -23,17 +24,44 @@ import androidx.wear.compose.foundation.lazy.items
2324
import androidx.wear.compose.foundation.lazy.rememberScalingLazyListState
2425

2526
class MainActivity : ComponentActivity() {
27+
28+
companion object {
29+
private const val TAG = "MainActivity"
30+
}
31+
2632
override fun onCreate(savedInstanceState: Bundle?) {
2733
super.onCreate(savedInstanceState)
34+
Log.d(TAG, "onCreate started")
2835

29-
setContent {
30-
AGiXTWearApp()
36+
try {
37+
setContent {
38+
AGiXTWearApp()
39+
}
40+
Log.d(TAG, "setContent completed")
41+
} catch (e: Exception) {
42+
Log.e(TAG, "Error in setContent", e)
3143
}
3244
}
45+
46+
override fun onResume() {
47+
super.onResume()
48+
Log.d(TAG, "onResume")
49+
}
50+
51+
override fun onPause() {
52+
super.onPause()
53+
Log.d(TAG, "onPause")
54+
}
55+
56+
override fun onDestroy() {
57+
super.onDestroy()
58+
Log.d(TAG, "onDestroy")
59+
}
3360
}
3461

3562
@Composable
3663
fun AGiXTWearApp() {
64+
// ViewModel is initialized safely - errors are caught in the ViewModel itself
3765
val viewModel: WearViewModel = viewModel()
3866
val uiState by viewModel.uiState.collectAsState()
3967
val messages by viewModel.messages.collectAsState()

‎android/wear/src/main/kotlin/dev/agixt/wear/WearViewModel.kt‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,18 +37,27 @@ class WearViewModel(application: Application) : AndroidViewModel(application) {
3737
private val _isPhoneConnected = MutableStateFlow(false)
3838
val isPhoneConnected: StateFlow<Boolean> = _isPhoneConnected.asStateFlow()
3939

40-
private val messageClient: MessageClient = Wearable.getMessageClient(application)
41-
private val nodeClient: NodeClient = Wearable.getNodeClient(application)
42-
private val capabilityClient: CapabilityClient = Wearable.getCapabilityClient(application)
40+
private var messageClient: MessageClient? = null
41+
private var nodeClient: NodeClient? = null
42+
private var capabilityClient: CapabilityClient? = null
4343

4444
init {
45-
checkPhoneConnection()
45+
Log.d(TAG, "ViewModel initializing...")
46+
try {
47+
messageClient = Wearable.getMessageClient(application)
48+
nodeClient = Wearable.getNodeClient(application)
49+
capabilityClient = Wearable.getCapabilityClient(application)
50+
Log.d(TAG, "Wearable clients initialized successfully")
51+
checkPhoneConnection()
52+
} catch (e: Exception) {
53+
Log.e(TAG, "Failed to initialize Wearable clients", e)
54+
}
4655
}
4756

4857
private fun checkPhoneConnection() {
4958
viewModelScope.launch {
5059
try {
51-
val nodes = nodeClient.connectedNodes.await()
60+
val nodes = nodeClient?.connectedNodes?.await() ?: emptyList()
5261
_isPhoneConnected.value = nodes.isNotEmpty()
5362
Log.d(TAG, "Connected nodes: ${nodes.size}")
5463
} catch (e: Exception) {
@@ -119,8 +128,11 @@ class WearViewModel(application: Application) : AndroidViewModel(application) {
119128
}
120129

121130
private suspend fun sendMessageToPhone(text: String): Boolean {
131+
val nc = nodeClient ?: return false
132+
val mc = messageClient ?: return false
133+
122134
return try {
123-
val nodes = nodeClient.connectedNodes.await()
135+
val nodes = nc.connectedNodes.await()
124136
if (nodes.isEmpty()) {
125137
Log.w(TAG, "No connected nodes found")
126138
return false
@@ -130,7 +142,7 @@ class WearViewModel(application: Application) : AndroidViewModel(application) {
130142
var sent = false
131143
for (node in nodes) {
132144
try {
133-
messageClient.sendMessage(
145+
mc.sendMessage(
134146
node.id,
135147
VOICE_INPUT_PATH,
136148
text.toByteArray(Charsets.UTF_8)
@@ -149,8 +161,10 @@ class WearViewModel(application: Application) : AndroidViewModel(application) {
149161
}
150162

151163
suspend fun getConnectedPhoneNode(): Node? {
164+
val nc = nodeClient ?: return null
165+
152166
return try {
153-
val nodes = nodeClient.connectedNodes.await()
167+
val nodes = nc.connectedNodes.await()
154168
nodes.firstOrNull { it.isNearby }
155169
} catch (e: Exception) {
156170
Log.e(TAG, "Failed to get phone node", e)

‎lib/services/ai_service.dart‎

Lines changed: 77 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,9 @@ class AIService {
370370
// This uses tts_mode=interleaved to stream both text and audio
371371
final responseBuffer = StringBuffer();
372372
bool audioHeaderSent = false;
373+
bool hasReceivedAudio = false;
374+
DateTime? lastGlassesUpdate;
375+
const glassesUpdateInterval = Duration(milliseconds: 500);
373376

374377
await for (final event in _chatWidget.sendChatMessageStreamingWithTTS(
375378
transcription,
@@ -378,16 +381,23 @@ class AIService {
378381
case ChatStreamEventType.text:
379382
if (event.text != null) {
380383
responseBuffer.write(event.text);
381-
// Stream text to glasses as it arrives
384+
// Stream accumulated text to glasses progressively (rate limited)
382385
if (_bluetoothManager.isConnected) {
383-
// Send partial text updates
384-
await _bluetoothManager.sendText(event.text!);
386+
final now = DateTime.now();
387+
if (lastGlassesUpdate == null ||
388+
now.difference(lastGlassesUpdate!) > glassesUpdateInterval) {
389+
lastGlassesUpdate = now;
390+
// Send full accumulated text so far
391+
await _bluetoothManager.sendAIResponse(
392+
responseBuffer.toString(),
393+
);
394+
}
385395
}
386396
}
387397
break;
388398

389399
case ChatStreamEventType.audioHeader:
390-
// Audio format info - could be used for local playback setup
400+
// Audio format info - used for watch playback
391401
if (event.sampleRate != null) {
392402
debugPrint(
393403
'AIService: Audio header - ${event.sampleRate}Hz, ${event.bitsPerSample}bit, ${event.channels}ch');
@@ -405,10 +415,13 @@ class AIService {
405415

406416
case ChatStreamEventType.audioChunk:
407417
// Stream audio to watch speaker
408-
if (event.audioData != null &&
409-
audioHeaderSent &&
410-
_watchService.isConnected) {
411-
await _watchService.sendAudioChunk(event.audioData!);
418+
if (event.audioData != null && audioHeaderSent) {
419+
// Only mark as received if watch is connected and audio is actually played
420+
if (_watchService.isConnected) {
421+
hasReceivedAudio = true;
422+
await _watchService.sendAudioChunk(event.audioData!);
423+
}
424+
// If watch not connected, audio is not played - fallback TTS will be used
412425
}
413426
break;
414427

@@ -433,14 +446,20 @@ class AIService {
433446
final fullResponse = responseBuffer.toString();
434447

435448
if (fullResponse.isNotEmpty) {
436-
// Display final response on glasses
449+
// Display final complete response on glasses
437450
if (_bluetoothManager.isConnected) {
438451
await _bluetoothManager.sendAIResponse(fullResponse);
439452
}
440453
// Display on watch
441454
if (_watchService.isConnected) {
442455
await _watchService.displayMessage(fullResponse, durationMs: 10000);
443456
}
457+
// If no audio was streamed (no watch connected or AGiXT didn't return audio),
458+
// use phone TTS as fallback
459+
if (!hasReceivedAudio && _ttsService.shouldUseTTS()) {
460+
debugPrint('AIService: No streaming audio received, using phone TTS');
461+
await _ttsService.speak(fullResponse);
462+
}
444463
} else {
445464
await _showErrorMessage('No response from AGiXT');
446465
}
@@ -683,6 +702,9 @@ class AIService {
683702
// Use streaming TTS to send audio to watch (like ESP32 does)
684703
final responseBuffer = StringBuffer();
685704
bool audioHeaderSent = false;
705+
bool hasReceivedAudio = false;
706+
DateTime? lastGlassesUpdate;
707+
const glassesUpdateInterval = Duration(milliseconds: 500);
686708

687709
await for (final event in _chatWidget.sendChatMessageStreamingWithTTS(
688710
message,
@@ -691,9 +713,16 @@ class AIService {
691713
case ChatStreamEventType.text:
692714
if (event.text != null) {
693715
responseBuffer.write(event.text);
694-
// Stream text to glasses as it arrives
716+
// Stream accumulated text to glasses progressively (rate limited)
695717
if (_bluetoothManager.isConnected) {
696-
await _bluetoothManager.sendText(event.text!);
718+
final now = DateTime.now();
719+
if (lastGlassesUpdate == null ||
720+
now.difference(lastGlassesUpdate!) > glassesUpdateInterval) {
721+
lastGlassesUpdate = now;
722+
await _bluetoothManager.sendAIResponse(
723+
responseBuffer.toString(),
724+
);
725+
}
697726
}
698727
}
699728
break;
@@ -716,10 +745,13 @@ class AIService {
716745

717746
case ChatStreamEventType.audioChunk:
718747
// Stream audio to watch speaker
719-
if (event.audioData != null &&
720-
audioHeaderSent &&
721-
_watchService.isConnected) {
722-
await _watchService.sendAudioChunk(event.audioData!);
748+
if (event.audioData != null && audioHeaderSent) {
749+
// Only mark as received if watch is connected and audio is actually played
750+
if (_watchService.isConnected) {
751+
hasReceivedAudio = true;
752+
await _watchService.sendAudioChunk(event.audioData!);
753+
}
754+
// If watch not connected, audio is not played - fallback TTS will be used
723755
}
724756
break;
725757

@@ -750,6 +782,11 @@ class AIService {
750782
if (_watchService.isConnected) {
751783
await _watchService.displayMessage(response, durationMs: 10000);
752784
}
785+
// If no audio was streamed, use phone TTS as fallback
786+
if (!hasReceivedAudio && _ttsService.shouldUseTTS()) {
787+
debugPrint('AIService: No streaming audio received, using phone TTS');
788+
await _ttsService.speak(response);
789+
}
753790
} else {
754791
await _showErrorMessage('No response from AGiXT');
755792
}
@@ -811,6 +848,9 @@ class AIService {
811848
// Use streaming TTS to send audio to watch (like foreground mode)
812849
final responseBuffer = StringBuffer();
813850
bool audioHeaderSent = false;
851+
bool hasReceivedAudio = false;
852+
DateTime? lastGlassesUpdate;
853+
const glassesUpdateInterval = Duration(milliseconds: 500);
814854

815855
await for (final event in _chatWidget.sendChatMessageStreamingWithTTS(
816856
message,
@@ -819,6 +859,17 @@ class AIService {
819859
case ChatStreamEventType.text:
820860
if (event.text != null) {
821861
responseBuffer.write(event.text);
862+
// Stream accumulated text to glasses progressively
863+
if (_bluetoothManager.isConnected) {
864+
final now = DateTime.now();
865+
if (lastGlassesUpdate == null ||
866+
now.difference(lastGlassesUpdate!) > glassesUpdateInterval) {
867+
lastGlassesUpdate = now;
868+
await _bluetoothManager.sendAIResponse(
869+
responseBuffer.toString(),
870+
);
871+
}
872+
}
822873
}
823874
break;
824875

@@ -836,10 +887,13 @@ class AIService {
836887
break;
837888

838889
case ChatStreamEventType.audioChunk:
839-
if (event.audioData != null &&
840-
audioHeaderSent &&
841-
_watchService.isConnected) {
842-
await _watchService.sendAudioChunk(event.audioData!);
890+
if (event.audioData != null && audioHeaderSent) {
891+
// Only mark as received if watch is connected and audio is actually played
892+
if (_watchService.isConnected) {
893+
hasReceivedAudio = true;
894+
await _watchService.sendAudioChunk(event.audioData!);
895+
}
896+
// If watch not connected, audio is not played - fallback TTS will be used
843897
}
844898
break;
845899

@@ -862,6 +916,10 @@ class AIService {
862916
if (_watchService.isConnected) {
863917
await _watchService.displayMessage(response, durationMs: 10000);
864918
}
919+
// Fallback to phone TTS if no streaming audio
920+
if (!hasReceivedAudio && _ttsService.shouldUseTTS()) {
921+
await _ttsService.speak(response);
922+
}
865923
} else {
866924
await _showErrorMessage('No response from AGiXT');
867925
}

0 commit comments

Comments
 (0)