From 9a5e01af596f157221d928416084f23b9569f628 Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:11:56 -0500 Subject: [PATCH 01/10] Use a callback flow --- .../kotlin/com/google/firebase/ai/common/util/android.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt index 6179c8b52e9..a5d3e892a73 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt @@ -19,6 +19,7 @@ package com.google.firebase.ai.common.util import android.media.AudioRecord import kotlin.time.Duration.Companion.milliseconds import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.flow import kotlinx.coroutines.yield @@ -35,7 +36,7 @@ internal val AudioRecord.minBufferSize: Int * * Will yield when this instance is not recording. */ -internal fun AudioRecord.readAsFlow() = flow { +internal fun AudioRecord.readAsFlow() = callbackFlow { val buffer = ByteArray(minBufferSize) while (true) { @@ -47,7 +48,7 @@ internal fun AudioRecord.readAsFlow() = flow { } val bytesRead = read(buffer, 0, buffer.size) if (bytesRead > 0) { - emit(buffer.copyOf(bytesRead)) + send(buffer.copyOf(bytesRead)) } yield() } From f14a46175dcb1bb80b10e5ced518730bf418ab2c Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:12:07 -0500 Subject: [PATCH 02/10] Listen for cancellation --- .../main/kotlin/com/google/firebase/ai/common/util/android.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt index a5d3e892a73..43dcf4cc115 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt @@ -21,6 +21,7 @@ import kotlin.time.Duration.Companion.milliseconds import kotlinx.coroutines.delay import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.isActive import kotlinx.coroutines.yield /** @@ -39,7 +40,7 @@ internal val AudioRecord.minBufferSize: Int internal fun AudioRecord.readAsFlow() = callbackFlow { val buffer = ByteArray(minBufferSize) - while (true) { + while (isActive) { if (recordingState != AudioRecord.RECORDSTATE_RECORDING) { // TODO(vguthal): Investigate if both yield and delay are required. delay(10.milliseconds) From 6d2428179ea5f63149c50bc4a251b946152102c3 Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:12:26 -0500 Subject: [PATCH 03/10] Change print logging --- .../src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt index f9507f5e8d3..c72d752f0f5 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt @@ -374,7 +374,7 @@ internal constructor( if (it.interrupted) { playBackQueue.clear() } else { - println("Sending audio parts") + println("Queuing audio parts from model") val audioParts = it.content?.parts?.filterIsInstance().orEmpty() for (part in audioParts) { playBackQueue.add(part.inlineData) From 6ac89dc624b0d20451c58fd5c9ecff761719b818 Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:12:35 -0500 Subject: [PATCH 04/10] Log when audio data is played --- .../src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt index c72d752f0f5..7d12ffcfb03 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt @@ -412,6 +412,7 @@ internal constructor( } yield() } else { + println("Playing audio data") /** * We pause the recording while the model is speaking to avoid interrupting it because of * no echo cancellation From fafcd3e4cecc0dab6f05408ec8e9e42f2dca471a Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:13:00 -0500 Subject: [PATCH 05/10] Revert IO change --- .../main/kotlin/com/google/firebase/ai/type/LiveSession.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt index 7d12ffcfb03..bc69ec33d9b 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt @@ -390,7 +390,7 @@ internal constructor( } } } - .launchIn(CoroutineScope(Dispatchers.IO)) + .launchIn(scope) } /** @@ -401,7 +401,7 @@ internal constructor( * Launched asynchronously on [scope]. */ private fun listenForModelPlayback(enableInterruptions: Boolean = false) { - CoroutineScope(Dispatchers.IO).launch { + scope.launch { while (isActive) { val playbackData = playBackQueue.poll() if (playbackData == null) { From d597d48d754cbdc6e2181370ed98f96c6231920c Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:13:30 -0500 Subject: [PATCH 06/10] Add name to coroutine --- .../main/kotlin/com/google/firebase/ai/type/LiveSession.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt index bc69ec33d9b..f299a8c93db 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt @@ -33,6 +33,7 @@ import io.ktor.client.plugins.websocket.DefaultClientWebSocketSession import io.ktor.websocket.Frame import io.ktor.websocket.close import io.ktor.websocket.readBytes +import kotlinx.coroutines.CoroutineName import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicBoolean import kotlin.coroutines.CoroutineContext @@ -137,8 +138,8 @@ internal constructor( ) return@catchAsync } - - scope = CoroutineScope(blockingDispatcher + childJob()) + // TODO: maybe it should be THREAD_PRIORITY_AUDIO anyways for playback and recording (not network though) + scope = CoroutineScope(blockingDispatcher + childJob() + CoroutineName("LiveSession Scope")) audioHelper = AudioHelper.build() recordUserAudio() From ace97d235e1219c4892d2cc2e08892fd274be7d5 Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:13:51 -0500 Subject: [PATCH 07/10] Use delay instead of yield --- .../kotlin/com/google/firebase/ai/common/util/android.kt | 6 ++---- .../main/kotlin/com/google/firebase/ai/type/LiveSession.kt | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt index 43dcf4cc115..e9b1736977c 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt @@ -42,15 +42,13 @@ internal fun AudioRecord.readAsFlow() = callbackFlow { while (isActive) { if (recordingState != AudioRecord.RECORDSTATE_RECORDING) { - // TODO(vguthal): Investigate if both yield and delay are required. - delay(10.milliseconds) - yield() + delay(0) continue } val bytesRead = read(buffer, 0, buffer.size) if (bytesRead > 0) { send(buffer.copyOf(bytesRead)) } - yield() + delay(0) } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt index f299a8c93db..104cddfca2f 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt @@ -41,6 +41,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.buffer import kotlinx.coroutines.flow.catch @@ -411,7 +412,7 @@ internal constructor( if (!enableInterruptions) { audioHelper?.resumeRecording() } - yield() + delay(0) } else { println("Playing audio data") /** From 031c38d6ffe0f9e53b6e0a9f186d2806fd7dac90 Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:14:00 -0500 Subject: [PATCH 08/10] Add delay to sending audio data --- .../main/kotlin/com/google/firebase/ai/type/LiveSession.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt index 104cddfca2f..3e3cc6063eb 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt @@ -328,7 +328,10 @@ internal constructor( ?.listenToRecording() ?.buffer(UNLIMITED) ?.accumulateUntil(MIN_BUFFER_SIZE) - ?.onEach { sendMediaStream(listOf(MediaData(it, "audio/pcm"))) } + ?.onEach { + sendMediaStream(listOf(MediaData(it, "audio/pcm"))) + delay(0) + } ?.catch { throw FirebaseAIException.from(it) } ?.launchIn(scope) } From e7dd7fcee5687ccd37122147084aaa7cf812fada Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:14:10 -0500 Subject: [PATCH 09/10] Bump coroutines --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 74be10aa2ad..9f760b20104 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,7 +16,7 @@ benchmarkMacro = "1.3.4" browser = "1.3.0" cardview = "1.0.0" constraintlayout = "2.1.4" -coroutines = "1.9.0" +coroutines = "1.10.2" dagger = "2.51" # Don't bump above 2.51 as it causes a bug in AppDistro FeedbackSender JPEG code datastore = "1.1.7" dexmaker = "2.28.1" From ffa0ce9e05a6dbbeef9c8c22164b6cf97deedcd5 Mon Sep 17 00:00:00 2001 From: Daymon Date: Wed, 15 Oct 2025 13:35:38 -0500 Subject: [PATCH 10/10] Update missed yield --- .../main/kotlin/com/google/firebase/ai/type/LiveSession.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt index 3e3cc6063eb..a5b169d12aa 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveSession.kt @@ -51,7 +51,6 @@ import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -import kotlinx.coroutines.yield import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.Serializable import kotlinx.serialization.encodeToString @@ -122,7 +121,6 @@ internal constructor( functionCallHandler: ((FunctionCallPart) -> FunctionResponsePart)? = null, enableInterruptions: Boolean = false, ) { - val context = firebaseApp.applicationContext if ( ContextCompat.checkSelfPermission(context, RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED @@ -203,7 +201,7 @@ internal constructor( ) } ?.let { emit(it.toPublic()) } - yield() + delay(0) } } .onCompletion { stopAudioConversation() }