Skip to content

Commit 54eb6a4

Browse files
Josh-XTJosh-XT
committed
more watch stuff
1 parent b60df8c commit 54eb6a4

File tree

10 files changed

+834
-25
lines changed

10 files changed

+834
-25
lines changed

‎android/app/src/main/AndroidManifest.xml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,31 @@
8888
android:name="id.flutter.flutter_background_service.BackgroundService"
8989
android:foregroundServiceType="dataSync|connectedDevice|location"
9090
/>
91+
92+
<!-- Wearable Listener Service - receives messages from Wear OS watch -->
93+
<service
94+
android:name="dev.agixt.agixt.WearableMessageService"
95+
android:exported="true">
96+
<intent-filter>
97+
<action android:name="com.google.android.gms.wearable.MESSAGE_RECEIVED" />
98+
<data
99+
android:host="*"
100+
android:pathPrefix="/voice_input"
101+
android:scheme="wear" />
102+
<data
103+
android:host="*"
104+
android:pathPrefix="/voice_command"
105+
android:scheme="wear" />
106+
<data
107+
android:host="*"
108+
android:pathPrefix="/audio_data"
109+
android:scheme="wear" />
110+
<data
111+
android:host="*"
112+
android:pathPrefix="/connection_status"
113+
android:scheme="wear" />
114+
</intent-filter>
115+
</service>
91116
<service
92117
android:label="notifications"
93118
android:name="notification.listener.service.NotificationListener"

‎android/app/src/main/kotlin/dev/agixt/agixt/MainActivity.kt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class MainActivity: FlutterActivity() {
3131
private val TAG = "MainActivity"
3232
private var methodChannelInitialized = false
3333
private var pendingToken: String? = null
34+
private var pendingVoiceInput: Pair<String, String?>? = null
3435

3536
// Voice & Watch handlers
3637
private var wakeWordHandler: WakeWordHandler? = null
@@ -78,6 +79,21 @@ class MainActivity: FlutterActivity() {
7879
}
7980
}
8081
}
82+
// Handle voice input from WearableMessageService
83+
else if (WearableMessageService.ACTION_VOICE_INPUT == it.action) {
84+
val text = it.getStringExtra(WearableMessageService.EXTRA_TEXT)
85+
val nodeId = it.getStringExtra(WearableMessageService.EXTRA_NODE_ID)
86+
if (!text.isNullOrEmpty()) {
87+
Log.d(TAG, "Voice input from watch intent: $text (from $nodeId)")
88+
if (methodChannelInitialized && watchHandler != null) {
89+
// Forward to WatchHandler which will send to Flutter
90+
watchHandler?.handleVoiceInputFromIntent(text, nodeId)
91+
} else {
92+
// Store for later processing
93+
pendingVoiceInput = Pair(text, nodeId)
94+
}
95+
}
96+
}
8197
}
8298
}
8399

@@ -220,6 +236,13 @@ class MainActivity: FlutterActivity() {
220236
pendingToken = null
221237
}
222238

239+
// Check if we have pending voice input to process
240+
pendingVoiceInput?.let { (text, nodeId) ->
241+
Log.d(TAG, "Processing pending voice input: $text")
242+
watchHandler?.handleVoiceInputFromIntent(text, nodeId)
243+
pendingVoiceInput = null
244+
}
245+
223246
// Setup the new channel for button events
224247
MethodChannel(flutterEngine.dartExecutor.binaryMessenger, BUTTON_EVENTS_CHANNEL).setMethodCallHandler { call, result ->
225248
// Currently no methods expected from Flutter on this channel, but handler is needed

‎android/app/src/main/kotlin/dev/agixt/agixt/WatchHandler.kt‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package dev.agixt.agixt
22

3+
import android.content.BroadcastReceiver
34
import android.content.Context
5+
import android.content.Intent
6+
import android.content.IntentFilter
7+
import android.os.Build
48
import android.util.Log
59
import io.flutter.plugin.common.MethodChannel
610
import io.flutter.plugin.common.BinaryMessenger
@@ -34,6 +38,10 @@ class WatchHandler(
3438
private val PATH_AUDIO_DATA = "/audio_data"
3539
private val PATH_CONNECTION_STATUS = "/connection_status"
3640
private val PATH_ERROR = "/error"
41+
// Audio streaming paths for TTS playback on watch
42+
private val PATH_AUDIO_HEADER = "/audio_header"
43+
private val PATH_AUDIO_CHUNK = "/audio_chunk"
44+
private val PATH_AUDIO_END = "/audio_end"
3745

3846
private lateinit var methodChannel: MethodChannel
3947
private var connectedNodeId: String? = null
@@ -42,9 +50,48 @@ class WatchHandler(
4250

4351
private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob())
4452

53+
// Broadcast receiver for WearableMessageService
54+
private val voiceInputReceiver = object : BroadcastReceiver() {
55+
override fun onReceive(context: Context?, intent: Intent?) {
56+
if (intent?.action == WearableMessageService.ACTION_VOICE_INPUT) {
57+
val text = intent.getStringExtra(WearableMessageService.EXTRA_TEXT)
58+
val nodeId = intent.getStringExtra(WearableMessageService.EXTRA_NODE_ID)
59+
if (!text.isNullOrEmpty()) {
60+
Log.d(TAG, "Received voice input broadcast: $text (from $nodeId)")
61+
sendVoiceInputToFlutter(text, nodeId)
62+
}
63+
}
64+
}
65+
}
66+
67+
/// Handle voice input from MainActivity intent (when app wasn't running)
68+
fun handleVoiceInputFromIntent(text: String, nodeId: String?) {
69+
Log.d(TAG, "Handling voice input from intent: $text (from $nodeId)")
70+
sendVoiceInputToFlutter(text, nodeId)
71+
}
72+
73+
private fun sendVoiceInputToFlutter(text: String, nodeId: String?) {
74+
try {
75+
methodChannel.invokeMethod("onWatchVoiceInput", mapOf(
76+
"text" to text,
77+
"nodeId" to nodeId
78+
))
79+
} catch (e: Exception) {
80+
Log.e(TAG, "Error sending voice input to Flutter", e)
81+
}
82+
}
83+
4584
fun initialize() {
4685
methodChannel = MethodChannel(binaryMessenger, CHANNEL)
4786

87+
// Register broadcast receiver for voice input from WearableMessageService
88+
val intentFilter = IntentFilter(WearableMessageService.ACTION_VOICE_INPUT)
89+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
90+
context.registerReceiver(voiceInputReceiver, intentFilter, Context.RECEIVER_NOT_EXPORTED)
91+
} else {
92+
context.registerReceiver(voiceInputReceiver, intentFilter)
93+
}
94+
4895
methodChannel.setMethodCallHandler { call, result ->
4996
when (call.method) {
5097
"initialize" -> {
@@ -144,6 +191,70 @@ class WatchHandler(
144191
}
145192
}
146193
}
194+
"sendAudioHeader" -> {
195+
// Send audio format header to watch for TTS playback
196+
val sampleRate = call.argument<Int>("sampleRate") ?: 24000
197+
val bitsPerSample = call.argument<Int>("bitsPerSample") ?: 16
198+
val channels = call.argument<Int>("channels") ?: 1
199+
val nodeId = call.argument<String>("nodeId")
200+
201+
// Pack header: 4 bytes sample rate, 2 bytes bits, 2 bytes channels
202+
val header = ByteArray(8)
203+
header[0] = (sampleRate and 0xFF).toByte()
204+
header[1] = ((sampleRate shr 8) and 0xFF).toByte()
205+
header[2] = ((sampleRate shr 16) and 0xFF).toByte()
206+
header[3] = ((sampleRate shr 24) and 0xFF).toByte()
207+
header[4] = (bitsPerSample and 0xFF).toByte()
208+
header[5] = ((bitsPerSample shr 8) and 0xFF).toByte()
209+
header[6] = (channels and 0xFF).toByte()
210+
header[7] = ((channels shr 8) and 0xFF).toByte()
211+
212+
scope.launch {
213+
val success = if (nodeId != null) {
214+
sendMessageToNode(nodeId, PATH_AUDIO_HEADER, header)
215+
} else {
216+
sendMessageToWatch(PATH_AUDIO_HEADER, header)
217+
}
218+
withContext(Dispatchers.Main) {
219+
result.success(success)
220+
}
221+
}
222+
}
223+
"sendAudioChunk" -> {
224+
// Send audio PCM data chunk to watch
225+
val audioData = call.argument<ByteArray>("audioData")
226+
val nodeId = call.argument<String>("nodeId")
227+
228+
if (audioData == null) {
229+
result.success(false)
230+
return@setMethodCallHandler
231+
}
232+
233+
scope.launch {
234+
val success = if (nodeId != null) {
235+
sendMessageToNode(nodeId, PATH_AUDIO_CHUNK, audioData)
236+
} else {
237+
sendMessageToWatch(PATH_AUDIO_CHUNK, audioData)
238+
}
239+
withContext(Dispatchers.Main) {
240+
result.success(success)
241+
}
242+
}
243+
}
244+
"sendAudioEnd" -> {
245+
// Signal end of audio stream
246+
val nodeId = call.argument<String>("nodeId")
247+
scope.launch {
248+
val success = if (nodeId != null) {
249+
sendMessageToNode(nodeId, PATH_AUDIO_END, byteArrayOf())
250+
} else {
251+
sendMessageToWatch(PATH_AUDIO_END, byteArrayOf())
252+
}
253+
withContext(Dispatchers.Main) {
254+
result.success(success)
255+
}
256+
}
257+
}
147258
"checkConnection" -> {
148259
scope.launch {
149260
checkWatchConnection()
@@ -341,6 +452,7 @@ class WatchHandler(
341452

342453
fun destroy() {
343454
try {
455+
context.unregisterReceiver(voiceInputReceiver)
344456
Wearable.getDataClient(context).removeListener(this)
345457
Wearable.getMessageClient(context).removeListener(this)
346458
Wearable.getCapabilityClient(context).removeListener(this)

‎android/app/src/main/kotlin/dev/agixt/agixt/WearableMessageService.kt‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package dev.agixt.agixt
2+
3+
import android.content.Intent
4+
import android.util.Log
5+
import com.google.android.gms.wearable.MessageEvent
6+
import com.google.android.gms.wearable.WearableListenerService
7+
8+
/**
9+
* WearableListenerService that receives messages from the Wear OS app.
10+
* This service runs even when the app is in the background, ensuring
11+
* voice commands from the watch are always received.
12+
*/
13+
class WearableMessageService : WearableListenerService() {
14+
15+
companion object {
16+
private const val TAG = "WearableMessageService"
17+
18+
// Message paths - must match the Wear OS app
19+
const val PATH_VOICE_INPUT = "/voice_input"
20+
const val PATH_VOICE_COMMAND = "/voice_command"
21+
const val PATH_AUDIO_DATA = "/audio_data"
22+
const val PATH_CONNECTION_STATUS = "/connection_status"
23+
24+
// Broadcast actions for local communication
25+
const val ACTION_VOICE_INPUT = "dev.agixt.agixt.VOICE_INPUT"
26+
const val EXTRA_TEXT = "text"
27+
const val EXTRA_NODE_ID = "node_id"
28+
}
29+
30+
override fun onMessageReceived(messageEvent: MessageEvent) {
31+
super.onMessageReceived(messageEvent)
32+
33+
Log.d(TAG, "Message received from watch: ${messageEvent.path}")
34+
35+
when (messageEvent.path) {
36+
PATH_VOICE_INPUT -> handleVoiceInput(messageEvent)
37+
PATH_VOICE_COMMAND -> handleVoiceCommand(messageEvent)
38+
PATH_AUDIO_DATA -> handleAudioData(messageEvent)
39+
PATH_CONNECTION_STATUS -> handleConnectionStatus(messageEvent)
40+
else -> Log.w(TAG, "Unknown message path: ${messageEvent.path}")
41+
}
42+
}
43+
44+
private fun handleVoiceInput(messageEvent: MessageEvent) {
45+
val text = String(messageEvent.data, Charsets.UTF_8)
46+
val nodeId = messageEvent.sourceNodeId
47+
48+
Log.d(TAG, "Voice input from watch ($nodeId): $text")
49+
50+
// Broadcast to the app (WatchHandler will pick this up if running)
51+
val intent = Intent(ACTION_VOICE_INPUT).apply {
52+
putExtra(EXTRA_TEXT, text)
53+
putExtra(EXTRA_NODE_ID, nodeId)
54+
setPackage(packageName)
55+
}
56+
sendBroadcast(intent)
57+
58+
// Also try to launch the main activity if app is not running
59+
// This ensures the voice input gets processed
60+
try {
61+
val launchIntent = Intent(this, MainActivity::class.java).apply {
62+
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP)
63+
putExtra(EXTRA_TEXT, text)
64+
putExtra(EXTRA_NODE_ID, nodeId)
65+
action = ACTION_VOICE_INPUT
66+
}
67+
startActivity(launchIntent)
68+
} catch (e: Exception) {
69+
Log.e(TAG, "Failed to launch activity for voice input", e)
70+
}
71+
}
72+
73+
private fun handleVoiceCommand(messageEvent: MessageEvent) {
74+
// Legacy voice command handling
75+
val transcription = String(messageEvent.data, Charsets.UTF_8)
76+
Log.d(TAG, "Voice command from watch: $transcription")
77+
78+
val intent = Intent(ACTION_VOICE_INPUT).apply {
79+
putExtra(EXTRA_TEXT, transcription)
80+
putExtra(EXTRA_NODE_ID, messageEvent.sourceNodeId)
81+
setPackage(packageName)
82+
}
83+
sendBroadcast(intent)
84+
}
85+
86+
private fun handleAudioData(messageEvent: MessageEvent) {
87+
Log.d(TAG, "Audio data received: ${messageEvent.data.size} bytes")
88+
// Audio data handling would go here if needed
89+
}
90+
91+
private fun handleConnectionStatus(messageEvent: MessageEvent) {
92+
val status = String(messageEvent.data, Charsets.UTF_8)
93+
Log.d(TAG, "Connection status from watch: $status")
94+
}
95+
}

0 commit comments

Comments
 (0)