Merge branch 'main' into bugfix/Issue.7296.Fix.ClassNotFoundException

Author: mrober

firebase-ai/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Unreleased
 
+- [changed] Added `LiveAudioConversationConfig` to control different aspects of the conversation
+  while using the `startAudioConversation` function.
+- [fixed] Fixed an issue causing streaming chat interactions to drop thought signatures. (#7562)
+
+# 17.6.0
+
 - [feature] Added support for server templates via `TemplateGenerativeModel` and
   `TemplateImagenModel`. (#7503)
 

firebase-ai/api.txt

@@ -165,6 +165,7 @@ package com.google.firebase.ai.java {
     method public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> sendVideoRealtime(com.google.firebase.ai.type.InlineData video);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation();
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(boolean enableInterruptions);
+    method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(com.google.firebase.ai.type.LiveAudioConversationConfig liveAudioConversationConfig);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler, boolean enableInterruptions);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public abstract com.google.common.util.concurrent.ListenableFuture<kotlin.Unit> startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler, kotlin.jvm.functions.Function2<? super com.google.firebase.ai.type.Transcription?,? super com.google.firebase.ai.type.Transcription?,kotlin.Unit>? transcriptHandler, boolean enableInterruptions);
@@ -874,6 +875,31 @@ package com.google.firebase.ai.type {
   public final class InvalidStateException extends com.google.firebase.ai.type.FirebaseAIException {
   }
 
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveAudioConversationConfig {
+    field public static final com.google.firebase.ai.type.LiveAudioConversationConfig.Companion Companion;
+  }
+
+  public static final class LiveAudioConversationConfig.Builder {
+    ctor public LiveAudioConversationConfig.Builder();
+    method public com.google.firebase.ai.type.LiveAudioConversationConfig build();
+    method public com.google.firebase.ai.type.LiveAudioConversationConfig.Builder setEnableInterruptions(boolean enableInterruptions);
+    method public com.google.firebase.ai.type.LiveAudioConversationConfig.Builder setFunctionCallHandler(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler);
+    method public com.google.firebase.ai.type.LiveAudioConversationConfig.Builder setInitializationHandler(kotlin.jvm.functions.Function2<? super android.media.AudioRecord.Builder,? super android.media.AudioTrack.Builder,kotlin.Unit>? initializationHandler);
+    method public com.google.firebase.ai.type.LiveAudioConversationConfig.Builder setTranscriptHandler(kotlin.jvm.functions.Function2<? super com.google.firebase.ai.type.Transcription?,? super com.google.firebase.ai.type.Transcription?,kotlin.Unit>? transcriptHandler);
+    field public boolean enableInterruptions;
+    field public kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler;
+    field public kotlin.jvm.functions.Function2<? super android.media.AudioRecord.Builder,? super android.media.AudioTrack.Builder,kotlin.Unit>? initializationHandler;
+    field public kotlin.jvm.functions.Function2<? super com.google.firebase.ai.type.Transcription?,? super com.google.firebase.ai.type.Transcription?,kotlin.Unit>? transcriptHandler;
+  }
+
+  public static final class LiveAudioConversationConfig.Companion {
+    method public com.google.firebase.ai.type.LiveAudioConversationConfig.Builder builder();
+  }
+
+  public final class LiveAudioConversationConfigKt {
+    method public static com.google.firebase.ai.type.LiveAudioConversationConfig liveAudioConversationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.LiveAudioConversationConfig.Builder,kotlin.Unit> init);
+  }
+
   @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveGenerationConfig {
     field public static final com.google.firebase.ai.type.LiveGenerationConfig.Companion Companion;
   }
@@ -958,6 +984,7 @@ package com.google.firebase.ai.type {
     method @Deprecated public suspend Object? sendMediaStream(java.util.List<com.google.firebase.ai.type.MediaData> mediaChunks, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? sendTextRealtime(String text, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? sendVideoRealtime(com.google.firebase.ai.type.InlineData video, kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public suspend Object? startAudioConversation(com.google.firebase.ai.type.LiveAudioConversationConfig liveAudioConversationConfig, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public suspend Object? startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler = null, boolean enableInterruptions = false, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public suspend Object? startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler = null, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method @RequiresPermission(android.Manifest.permission.RECORD_AUDIO) public suspend Object? startAudioConversation(kotlin.jvm.functions.Function1<? super com.google.firebase.ai.type.FunctionCallPart,com.google.firebase.ai.type.FunctionResponsePart>? functionCallHandler = null, kotlin.jvm.functions.Function2<? super com.google.firebase.ai.type.Transcription?,? super com.google.firebase.ai.type.Transcription?,kotlin.Unit>? transcriptHandler = null, boolean enableInterruptions = false, kotlin.coroutines.Continuation<? super kotlin.Unit>);

firebase-ai/gradle.properties

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.6.0
-latestReleasedVersion=17.5.0
+version=17.7.0
+latestReleasedVersion=17.6.0

firebase-ai/src/main/kotlin/com/google/firebase/ai/Chat.kt

@@ -22,6 +22,7 @@ import com.google.firebase.ai.type.GenerateContentResponse
 import com.google.firebase.ai.type.ImagePart
 import com.google.firebase.ai.type.InlineDataPart
 import com.google.firebase.ai.type.InvalidStateException
+import com.google.firebase.ai.type.Part
 import com.google.firebase.ai.type.TextPart
 import com.google.firebase.ai.type.content
 import java.util.LinkedList
@@ -133,6 +134,7 @@ public class Chat(
     val bitmaps = LinkedList<Bitmap>()
     val inlineDataParts = LinkedList<InlineDataPart>()
     val text = StringBuilder()
+    val parts = mutableListOf<Part>()
 
     /**
      * TODO: revisit when images and inline data are returned. This will cause issues with how
@@ -147,22 +149,17 @@ public class Chat(
             is ImagePart -> bitmaps.add(part.image)
             is InlineDataPart -> inlineDataParts.add(part)
           }
+          parts.add(part)
         }
       }
       .onCompletion {
         lock.release()
         if (it == null) {
           val content =
             content("model") {
-              for (bitmap in bitmaps) {
-                image(bitmap)
-              }
-              for (inlineDataPart in inlineDataParts) {
-                inlineData(inlineDataPart.inlineData, inlineDataPart.mimeType)
-              }
-              if (text.isNotBlank()) {
-                text(text.toString())
-              }
+              setParts(
+                parts.filterNot { part -> part is TextPart && !part.hasContent() }.toMutableList()
+              )
             }
 
           history.add(prompt)
@@ -224,3 +221,12 @@ public class Chat(
     }
   }
 }
+
+/**
+ * Returns true if the [TextPart] contains any content, either in its [TextPart.text] property or
+ * its [TextPart.thoughtSignature] property.
+ */
+private fun TextPart.hasContent(): Boolean {
+  if (text.isNotEmpty()) return true
+  return !thoughtSignature.isNullOrBlank()
+}

firebase-ai/src/main/kotlin/com/google/firebase/ai/java/LiveSessionFutures.kt

@@ -24,6 +24,7 @@ import com.google.firebase.ai.type.Content
 import com.google.firebase.ai.type.FunctionCallPart
 import com.google.firebase.ai.type.FunctionResponsePart
 import com.google.firebase.ai.type.InlineData
+import com.google.firebase.ai.type.LiveAudioConversationConfig
 import com.google.firebase.ai.type.LiveServerMessage
 import com.google.firebase.ai.type.LiveSession
 import com.google.firebase.ai.type.MediaData
@@ -49,6 +50,18 @@ public abstract class LiveSessionFutures internal constructor() {
   @RequiresPermission(RECORD_AUDIO)
   public abstract fun startAudioConversation(): ListenableFuture<Unit>
 
+  /**
+   * Starts an audio conversation with the model, which can only be stopped using
+   * [stopAudioConversation].
+   *
+   * @param liveAudioConversationConfig A [LiveAudioConversationConfig] provided by the user to
+   * control the various aspects of the conversation.
+   */
+  @RequiresPermission(RECORD_AUDIO)
+  public abstract fun startAudioConversation(
+    liveAudioConversationConfig: LiveAudioConversationConfig
+  ): ListenableFuture<Unit>
+
   /**
    * Starts an audio conversation with the model, which can only be stopped using
    * [stopAudioConversation] or [close].
@@ -197,9 +210,12 @@ public abstract class LiveSessionFutures internal constructor() {
   public abstract fun sendVideoRealtime(video: InlineData): ListenableFuture<Unit>
 
   /**
-   * Sends text data to the server in realtime. Check
-   * https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for details about the realtime
-   * input usage.
+   * For details about the realtime input usage, see the `BidiGenerateContentRealtimeInput`
+   * documentation (
+   * [Gemini Developer API](https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput) or
+   * [Vertex AI Gemini API](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-live#bidigeneratecontentrealtimeinput)
+   * ).
+   *
    * @param text The text data to send.
    */
   public abstract fun sendTextRealtime(text: String): ListenableFuture<Unit>
@@ -211,7 +227,7 @@ public abstract class LiveSessionFutures internal constructor() {
    *
    * @param mediaChunks The list of [MediaData] instances representing the media data to be sent.
    */
-  @Deprecated("Use sendAudioRealtime, sendVideoRealtime, or sendTextRealtime instead")
+  @Deprecated("Use `sendAudioRealtime`, `sendVideoRealtime`, or `sendTextRealtime` instead")
   public abstract fun sendMediaStream(mediaChunks: List<MediaData>): ListenableFuture<Unit>
 
   /**
@@ -294,6 +310,12 @@ public abstract class LiveSessionFutures internal constructor() {
         session.startAudioConversation(transcriptHandler = transcriptHandler)
       }
 
+    @RequiresPermission(RECORD_AUDIO)
+    override fun startAudioConversation(liveAudioConversationConfig: LiveAudioConversationConfig) =
+      SuspendToFutureAdapter.launchFuture {
+        session.startAudioConversation(liveAudioConversationConfig)
+      }
+
     @RequiresPermission(RECORD_AUDIO)
     override fun startAudioConversation() =
       SuspendToFutureAdapter.launchFuture { session.startAudioConversation() }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/AudioHelper.kt

@@ -19,7 +19,6 @@ package com.google.firebase.ai.type
 import android.Manifest
 import android.media.AudioAttributes
 import android.media.AudioFormat
-import android.media.AudioManager
 import android.media.AudioRecord
 import android.media.AudioTrack
 import android.media.MediaRecorder
@@ -157,28 +156,39 @@ internal class AudioHelper(
      *
      * It also makes it easier to read, since the long initialization is separate from the
      * constructor.
+     *
+     * @param initializationHandler A callback that is invoked immediately following the successful
+     * initialization of the associated [AudioRecord.Builder] and [AudioTrack.Builder] objects. This
+     * offers a final opportunity to configure these objects, which will remain valid and effective
+     * for the duration of the current audio session.
      */
     @RequiresPermission(Manifest.permission.RECORD_AUDIO)
-    fun build(): AudioHelper {
-      val playbackTrack =
-        AudioTrack(
-          AudioAttributes.Builder()
-            .setUsage(AudioAttributes.USAGE_MEDIA)
-            .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
-            .build(),
+    fun build(
+      initializationHandler: ((AudioRecord.Builder, AudioTrack.Builder) -> Unit)? = null
+    ): AudioHelper {
+      val playTrackBuilder = AudioTrack.Builder()
+      playTrackBuilder
+        .setAudioFormat(
           AudioFormat.Builder()
             .setSampleRate(24000)
             .setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
             .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
-            .build(),
+            .build()
+        )
+        .setAudioAttributes(
+          AudioAttributes.Builder()
+            .setUsage(AudioAttributes.USAGE_MEDIA)
+            .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
+            .build()
+        )
+        .setBufferSizeInBytes(
           AudioTrack.getMinBufferSize(
             24000,
             AudioFormat.CHANNEL_OUT_MONO,
             AudioFormat.ENCODING_PCM_16BIT
           ),
-          AudioTrack.MODE_STREAM,
-          AudioManager.AUDIO_SESSION_ID_GENERATE
         )
+        .setTransferMode(AudioTrack.MODE_STREAM)
 
       val bufferSize =
         AudioRecord.getMinBufferSize(
@@ -191,15 +201,22 @@ internal class AudioHelper(
         throw AudioRecordInitializationFailedException(
           "Audio Record buffer size is invalid ($bufferSize)"
         )
-
-      val recorder =
-        AudioRecord(
-          MediaRecorder.AudioSource.VOICE_COMMUNICATION,
-          16000,
-          AudioFormat.CHANNEL_IN_MONO,
-          AudioFormat.ENCODING_PCM_16BIT,
-          bufferSize
-        )
+      val recorderBuilder =
+        AudioRecord.Builder()
+          .setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION)
+          .setAudioFormat(
+            AudioFormat.Builder()
+              .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+              .setSampleRate(16000)
+              .setChannelMask(AudioFormat.CHANNEL_IN_MONO)
+              .build()
+          )
+          .setBufferSizeInBytes(bufferSize)
+      if (initializationHandler != null) {
+        initializationHandler(recorderBuilder, playTrackBuilder)
+      }
+      val recorder = recorderBuilder.build()
+      val playbackTrack = playTrackBuilder.build()
       if (recorder.state != AudioRecord.STATE_INITIALIZED)
         throw AudioRecordInitializationFailedException(
           "Audio Record initialization has failed. State: ${recorder.state}"

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt

@@ -84,7 +84,7 @@ internal constructor(
  *
  * The rating will be restricted to a particular [category].
  *
- * @property category The category of harm being assessed (e.g., Hate speech).
+ * @property category The category of harm being assessed (for example, Hate speech).
  * @property probability The likelihood of the content causing harm.
  * @property probabilityScore A numerical score representing the probability of harm, between `0`
  * and `1`.

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ContentModality.kt

@@ -71,7 +71,7 @@ public class ContentModality private constructor(public val ordinal: Int) {
     /** Audio. */
     @JvmField public val AUDIO: ContentModality = ContentModality(4)
 
-    /** Document, e.g. PDF. */
+    /** Document (for example, PDF). */
     @JvmField public val DOCUMENT: ContentModality = ContentModality(5)
   }
 }