Audio Conversation user interrupt support and threading fixes

firebase-ai/CHANGELOG.md

@@ -1,6 +1,8 @@
 # Unreleased
 
 - [changed] **Breaking Change**: Removed the `candidateCount` option from `LiveGenerationConfig`
+- [changed] Added support for user interrupts for the `startAudioConversation` method in the
+  `LiveSession` class.
 - [changed] Added support for the URL context tool, which allows the model to access content from
   provided public web URLs to inform and enhance its responses. (#7382)
 - [changed] Added better error messages to `ServiceConnectionHandshakeFailedException` (#7412)

firebase-ai/api.txt

@@ -27,10 +27,10 @@ package com.google.firebase.ai {
     method public static com.google.firebase.ai.FirebaseAI getInstance(com.google.firebase.FirebaseApp app);
     method public static com.google.firebase.ai.FirebaseAI getInstance(com.google.firebase.FirebaseApp app = Firebase.app, com.google.firebase.ai.type.GenerativeBackend backend);
     method public static com.google.firebase.ai.FirebaseAI getInstance(com.google.firebase.FirebaseApp app = Firebase.app, com.google.firebase.ai.type.GenerativeBackend backend, boolean useLimitedUseAppCheckTokens);
-    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.ImagenModel imagenModel(String modelName);
-    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.ImagenModel imagenModel(String modelName, com.google.firebase.ai.type.ImagenGenerationConfig? generationConfig = null);
-    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.ImagenModel imagenModel(String modelName, com.google.firebase.ai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.ai.type.ImagenSafetySettings? safetySettings = null);
-    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.ImagenModel imagenModel(String modelName, com.google.firebase.ai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.ai.type.ImagenSafetySettings? safetySettings = null, com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions());
+    method public com.google.firebase.ai.ImagenModel imagenModel(String modelName);
+    method public com.google.firebase.ai.ImagenModel imagenModel(String modelName, com.google.firebase.ai.type.ImagenGenerationConfig? generationConfig = null);
+    method public com.google.firebase.ai.ImagenModel imagenModel(String modelName, com.google.firebase.ai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.ai.type.ImagenSafetySettings? safetySettings = null);
+    method public com.google.firebase.ai.ImagenModel imagenModel(String modelName, com.google.firebase.ai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.ai.type.ImagenSafetySettings? safetySettings = null, com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions());
     method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName);
     method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null);
     method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.ai.type.Tool>? tools = null);
@@ -72,11 +72,11 @@ package com.google.firebase.ai {
     method public com.google.firebase.ai.Chat startChat(java.util.List<com.google.firebase.ai.type.Content> history = emptyList());
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenModel {
-    method public suspend Object? editImage(java.util.List<? extends com.google.firebase.ai.type.ImagenReferenceImage> referenceImages, String prompt, com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
+  public final class ImagenModel {
+    method @com.google.firebase.ai.type.PublicPreviewAPI public suspend Object? editImage(java.util.List<? extends com.google.firebase.ai.type.ImagenReferenceImage> referenceImages, String prompt, com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
     method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
-    method public suspend Object? inpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, String prompt, com.google.firebase.ai.type.ImagenMaskReference mask, com.google.firebase.ai.type.ImagenEditingConfig config, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
-    method public suspend Object? outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
+    method @com.google.firebase.ai.type.PublicPreviewAPI public suspend Object? inpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, String prompt, com.google.firebase.ai.type.ImagenMaskReference mask, com.google.firebase.ai.type.ImagenEditingConfig config, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
+    method @com.google.firebase.ai.type.PublicPreviewAPI public suspend Object? outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
   }
 
   @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveGenerativeModel {
@@ -148,7 +148,9 @@ package com.google.firebase.ai.java {
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendFunctionResponse(java.util.List<com.google.firebase.ai.type.FunctionResponsePart> functionList);
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendMediaStream(java.util.List<com.google.firebase.ai.type.MediaData> mediaChunks);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation();
-    method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler);
+    method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(boolean enableInterruptions);
+    method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler);
+    method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler, boolean enableInterruptions);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> stopAudioConversation();
     method public abstract void stopReceiving();
     field public static final com.google.firebase.ai.java.LiveSessionFutures.Companion Companion;
@@ -581,7 +583,7 @@ package com.google.firebase.ai.type {
     property public boolean isThought;
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenAspectRatio {
+  public final class ImagenAspectRatio {
     field public static final com.google.firebase.ai.type.ImagenAspectRatio.Companion Companion;
     field public static final com.google.firebase.ai.type.ImagenAspectRatio LANDSCAPE_16x9;
     field public static final com.google.firebase.ai.type.ImagenAspectRatio LANDSCAPE_4x3;
@@ -630,7 +632,7 @@ package com.google.firebase.ai.type {
     ctor public ImagenForegroundMask(Double? dilation = null);
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenGenerationConfig {
+  public final class ImagenGenerationConfig {
     ctor public ImagenGenerationConfig(String? negativePrompt = null, Integer? numberOfImages = 1, com.google.firebase.ai.type.ImagenAspectRatio? aspectRatio = null, com.google.firebase.ai.type.ImagenImageFormat? imageFormat = null, Boolean? addWatermark = null);
     method public Boolean? getAddWatermark();
     method public com.google.firebase.ai.type.ImagenAspectRatio? getAspectRatio();
@@ -665,17 +667,17 @@ package com.google.firebase.ai.type {
   }
 
   public final class ImagenGenerationConfigKt {
-    method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.ImagenGenerationConfig imagenGenerationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.ImagenGenerationConfig.Builder,kotlin.Unit> init);
+    method public static com.google.firebase.ai.type.ImagenGenerationConfig imagenGenerationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.ImagenGenerationConfig.Builder,kotlin.Unit> init);
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenGenerationResponse<T> {
+  public final class ImagenGenerationResponse<T> {
     method public String? getFilteredReason();
     method public java.util.List<T> getImages();
     property public final String? filteredReason;
     property public final java.util.List<T> images;
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenImageFormat {
+  public final class ImagenImageFormat {
     method public Integer? getCompressionQuality();
     method public String getMimeType();
     method public static com.google.firebase.ai.type.ImagenImageFormat jpeg(Integer? compressionQuality = null);
@@ -712,7 +714,7 @@ package com.google.firebase.ai.type {
     method public com.google.firebase.ai.type.ImagenImagePlacement fromCoordinate(int x, int y);
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenInlineImage {
+  public final class ImagenInlineImage {
     method public android.graphics.Bitmap asBitmap();
     method public byte[] getData();
     method public String getMimeType();
@@ -737,7 +739,7 @@ package com.google.firebase.ai.type {
     method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, double dilation = 0.01);
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {
+  public final class ImagenPersonFilterLevel {
     field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ADULT;
     field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ALL;
     field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel BLOCK_ALL;
@@ -762,7 +764,7 @@ package com.google.firebase.ai.type {
     property public final Integer? referenceId;
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenSafetyFilterLevel {
+  public final class ImagenSafetyFilterLevel {
     field public static final com.google.firebase.ai.type.ImagenSafetyFilterLevel BLOCK_LOW_AND_ABOVE;
     field public static final com.google.firebase.ai.type.ImagenSafetyFilterLevel BLOCK_MEDIUM_AND_ABOVE;
     field public static final com.google.firebase.ai.type.ImagenSafetyFilterLevel BLOCK_NONE;
@@ -773,7 +775,7 @@ package com.google.firebase.ai.type {
   public static final class ImagenSafetyFilterLevel.Companion {
   }
 
-  @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenSafetySettings {
+  public final class ImagenSafetySettings {
     ctor public ImagenSafetySettings(com.google.firebase.ai.type.ImagenSafetyFilterLevel safetyFilterLevel, com.google.firebase.ai.type.ImagenPersonFilterLevel personFilterLevel);
   }
 
@@ -891,7 +893,7 @@ package com.google.firebase.ai.type {
     method public suspend Object? send(String text, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? sendFunctionResponse(java.util.List<com.google.firebase.ai.type.FunctionResponsePart> functionList, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? sendMediaStream(java.util.List<com.google.firebase.ai.type.MediaData> mediaChunks, kotlin.coroutines.Continuation<? super kotlin.Unit>);
-    method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public suspend Object? startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler = null, kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public suspend Object? startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler = null, Boolean? enableInterruptions = null, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public void stopAudioConversation();
     method public void stopReceiving();
   }

firebase-ai/src/main/kotlin/com/google/firebase/ai/common/util/android.kt

@@ -17,6 +17,7 @@
 package com.google.firebase.ai.common.util
 
 import android.media.AudioRecord
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.yield
 
@@ -38,13 +39,14 @@ internal fun AudioRecord.readAsFlow() = flow {
 
   while (true) {
     if (recordingState != AudioRecord.RECORDSTATE_RECORDING) {
+      delay(10)
       yield()
       continue
     }
-
     val bytesRead = read(buffer, 0, buffer.size)
     if (bytesRead > 0) {
       emit(buffer.copyOf(bytesRead))
     }
+    yield()
   }
 }

firebase-ai/src/main/kotlin/com/google/firebase/ai/java/LiveSessionFutures.kt

@@ -47,6 +47,7 @@ public abstract class LiveSessionFutures internal constructor() {
    * @param functionCallHandler A callback function that is invoked whenever the model receives a
    * function call.
    */
+  @RequiresPermission(RECORD_AUDIO)
   public abstract fun startAudioConversation(
     functionCallHandler: ((FunctionCallPart) -> FunctionResponsePart)?
   ): ListenableFuture<Unit>
@@ -58,6 +59,36 @@ public abstract class LiveSessionFutures internal constructor() {
   @RequiresPermission(RECORD_AUDIO)
   public abstract fun startAudioConversation(): ListenableFuture<Unit>
 
+  /**
+   * Starts an audio conversation with the model, which can only be stopped using
+   * [stopAudioConversation] or [close].
+   *
+   * @param enableInterruptions Boolean to enable user to interrupt the model. Setting this variable
+   * would allow the user to talk while the model is responding.
+   *
+   * **WARNING**: User interruption might not work reliably across all devices.
+   */
+  @RequiresPermission(RECORD_AUDIO)
+  public abstract fun startAudioConversation(enableInterruptions: Boolean): ListenableFuture<Unit>
+
+  /**
+   * Starts an audio conversation with the model, which can only be stopped using
+   * [stopAudioConversation] or [close].
+   *
+   * @param functionCallHandler A callback function that is invoked whenever the model receives a
+   * function call.
+   *
+   * @param enableInterruptions Boolean to enable user to interrupt the model. Setting this variable
+   * would allow the user to talk while the model is responding.
+   *
+   * **WARNING**: User interruption might not work reliably across all devices.
+   */
+  @RequiresPermission(RECORD_AUDIO)
+  public abstract fun startAudioConversation(
+    functionCallHandler: ((FunctionCallPart) -> FunctionResponsePart)?,
+    enableInterruptions: Boolean
+  ): ListenableFuture<Unit>
+
   /**
    * Stops the audio conversation with the Gemini Server.
    *
@@ -169,6 +200,24 @@ public abstract class LiveSessionFutures internal constructor() {
     override fun startAudioConversation() =
       SuspendToFutureAdapter.launchFuture { session.startAudioConversation() }
 
+    @RequiresPermission(RECORD_AUDIO)
+    override fun startAudioConversation(enableInterruptions: Boolean) =
+      SuspendToFutureAdapter.launchFuture {
+        session.startAudioConversation(enableInterruptions = enableInterruptions)
+      }
+
+    @RequiresPermission(RECORD_AUDIO)
+    override fun startAudioConversation(
+      functionCallHandler: ((FunctionCallPart) -> FunctionResponsePart)?,
+      enableInterruptions: Boolean
+    ) =
+      SuspendToFutureAdapter.launchFuture {
+        session.startAudioConversation(
+          functionCallHandler,
+          enableInterruptions = enableInterruptions
+        )
+      }
+
     override fun stopAudioConversation() =
       SuspendToFutureAdapter.launchFuture { session.stopAudioConversation() }
 

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/AudioHelper.kt

@@ -141,7 +141,6 @@ internal class AudioHelper(
    */
   fun listenToRecording(): Flow<ByteArray> {
     if (released) return emptyFlow()
-
     resumeRecording()
 
     return recorder.readAsFlow()