Merge branch 'main' into daymxn-patch-1

Author: daymxn

.github/workflows/ai-nightlies.yml renamed to .github/workflows/ai-daily-tests.yml

@@ -1,19 +1,21 @@
-name: Firebase AI Nightlies
+name: Firebase AI Daily Tests
 
 on:
   schedule:
     - cron: 2 7 * * *  # Runs automatically once a day
   workflow_dispatch:   # Allow triggering the workflow manually
 
+permissions:
+  contents: read
+
 jobs:
-  nightlies:
-    name: "Nightlies"
+  dailies:
+    name: "Daily Tests"
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          fetch-depth: 2
           submodules: true
 
       - name: Enable KVM
@@ -48,7 +50,7 @@ jobs:
           FTL_RESULTS_DIR: ${{ format('logs/{0}/{1}_{2}/artifacts/', github.workflow, github.run_id, github.run_attempt) }}
           FIREBASE_APP_CHECK_DEBUG_SECRET: ${{ secrets.FIREBASE_APP_CHECK_DEBUG_SECRET }}
         with:
-          api-level: 31
+          api-level: 34
           arch: x86_64
           ram-size: 4096M
           heap-size: 4096M

.github/workflows/dataconnect.yml

@@ -34,7 +34,7 @@ env:
   FDC_JAVA_VERSION: ${{ inputs.javaVersion || '17' }}
   FDC_ANDROID_EMULATOR_API_LEVEL: ${{ inputs.androidEmulatorApiLevel || '34' }}
   FDC_NODEJS_VERSION: ${{ inputs.nodeJsVersion || '20' }}
-  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.12.0' }}
+  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.15.1' }}
   FDC_FIREBASE_TOOLS_DIR: /tmp/firebase-tools
   FDC_FIREBASE_COMMAND: /tmp/firebase-tools/node_modules/.bin/firebase
   FDC_PYTHON_VERSION: ${{ inputs.pythonVersion || '3.13' }}
@@ -134,6 +134,7 @@ jobs:
 
           ./gradlew \
             --profile \
+            --warning-mode all \
             ${{ (inputs.gradleInfoLog && '--info') || '' }} \
             :firebase-dataconnect:assembleDebugAndroidTest
 
@@ -222,7 +223,7 @@ jobs:
           emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
           disable-animations: true
           script: |
-            set -eux && ./gradlew ${{ (inputs.gradleInfoLog && '--info') || '' }} :firebase-dataconnect:connectedCheck :firebase-dataconnect:connectors:connectedCheck
+            set -eux && ./gradlew --warning-mode all ${{ (inputs.gradleInfoLog && '--info') || '' }} :firebase-dataconnect:connectedCheck :firebase-dataconnect:connectors:connectedCheck
 
       - name: Upload Log Files
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1

.github/workflows/dataconnect_demo_app.yml

@@ -18,7 +18,7 @@ on:
 
 env:
   FDC_NODE_VERSION: ${{ inputs.nodeVersion || '20' }}
-  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.12.0' }}
+  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.15.1' }}
   FDC_JAVA_VERSION: ${{ inputs.javaVersion || '17' }}
   FDC_FIREBASE_TOOLS_DIR: ${{ github.workspace }}/firebase-tools
   FDC_FIREBASE_COMMAND: ${{ github.workspace }}/firebase-tools/node_modules/.bin/firebase
@@ -108,13 +108,15 @@ jobs:
           --project-dir firebase-dataconnect/demo \
           ${{ (inputs.gradleInfoLog && '--info') || '' }} \
           --profile \
+          --warning-mode all \
           -PdataConnect.demo.firebaseCommand=${{ env.FDC_FIREBASE_COMMAND }} \
           assemble test
 
       - name: gradle dokkaGeneratePublicationHtml
         run: |
           set -x
           firebase-dataconnect/demo/gradlew \
+          --warning-mode all \
           --project-dir firebase-dataconnect/demo \
           ${{ (inputs.gradleInfoLog && '--info') || '' }} \
           dokkaGeneratePublicationHtml
@@ -176,6 +178,7 @@ jobs:
         run: |
           set -x
           firebase-dataconnect/demo/gradlew \
+          --warning-mode all \
           --project-dir firebase-dataconnect/demo \
           --no-daemon \
           ${{ (inputs.gradleInfoLog && '--info') || '' }} \

firebase-ai/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Unreleased
 
+- [feature] Introduced `MissingPermissionsException`, which is thrown when the necessary permissions
+  have not been granted by the user.
+- [feature] Added helper functions to `LiveSession` to allow developers to track the status of the
+  audio session and the underlying websocket connection.
+- [changed] Added new values to `HarmCategory` (#7324)
+- [fixed] Fixed an issue that caused unknown or empty `Part`s to throw an exception. Instead, we now
+  log them and filter them from the response (#7333)
+
 # 17.2.0
 
 - [feature] Added support for returning thought summaries, which are synthesized versions of a

firebase-ai/api.txt

@@ -531,6 +531,10 @@ package com.google.firebase.ai.type {
     field public static final com.google.firebase.ai.type.HarmCategory DANGEROUS_CONTENT;
     field public static final com.google.firebase.ai.type.HarmCategory HARASSMENT;
     field public static final com.google.firebase.ai.type.HarmCategory HATE_SPEECH;
+    field public static final com.google.firebase.ai.type.HarmCategory IMAGE_DANGEROUS_CONTENT;
+    field public static final com.google.firebase.ai.type.HarmCategory IMAGE_HARASSMENT;
+    field public static final com.google.firebase.ai.type.HarmCategory IMAGE_HATE;
+    field public static final com.google.firebase.ai.type.HarmCategory IMAGE_SEXUALLY_EXPLICIT;
     field public static final com.google.firebase.ai.type.HarmCategory SEXUALLY_EXPLICIT;
     field public static final com.google.firebase.ai.type.HarmCategory UNKNOWN;
   }
@@ -879,6 +883,8 @@ package com.google.firebase.ai.type {
 
   @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveSession {
     method public suspend Object? close(kotlin.coroutines.Continuation<? super kotlin.Unit>);
+    method public boolean isAudioConversationActive();
+    method public boolean isClosed();
     method public kotlinx.coroutines.flow.Flow<com.google.firebase.ai.type.LiveServerMessage> receive();
     method public suspend Object? send(com.google.firebase.ai.type.Content content, kotlin.coroutines.Continuation<? super kotlin.Unit>);
     method public suspend Object? send(String text, kotlin.coroutines.Continuation<? super kotlin.Unit>);
@@ -918,6 +924,10 @@ package com.google.firebase.ai.type {
     method public static String? asTextOrNull(com.google.firebase.ai.type.Part);
   }
 
+  public final class PermissionMissingException extends com.google.firebase.ai.type.FirebaseAIException {
+    ctor public PermissionMissingException(String message, Throwable? cause = null);
+  }
+
   public final class PromptBlockedException extends com.google.firebase.ai.type.FirebaseAIException {
     method public com.google.firebase.ai.type.GenerateContentResponse? getResponse();
     property public final com.google.firebase.ai.type.GenerateContentResponse? response;

firebase-ai/gradle.properties

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.2.1
+version=17.3.0
 latestReleasedVersion=17.2.0

firebase-ai/src/main/kotlin/com/google/firebase/ai/LiveGenerativeModel.kt

@@ -55,7 +55,8 @@ internal constructor(
   private val tools: List<Tool>? = null,
   private val systemInstruction: Content? = null,
   private val location: String,
-  private val controller: APIController,
+  private val firebaseApp: FirebaseApp,
+  private val controller: APIController
 ) {
   internal constructor(
     modelName: String,
@@ -78,6 +79,7 @@ internal constructor(
     tools,
     systemInstruction,
     location,
+    firebaseApp,
     APIController(
       apiKey,
       modelName,
@@ -119,7 +121,11 @@ internal constructor(
       val receivedJson = JSON.parseToJsonElement(receivedJsonStr)
 
       return if (receivedJson is JsonObject && "setupComplete" in receivedJson) {
-        LiveSession(session = webSession, blockingDispatcher = blockingDispatcher)
+        LiveSession(
+          session = webSession,
+          blockingDispatcher = blockingDispatcher,
+          firebaseApp = firebaseApp
+        )
       } else {
         webSession.close()
         throw ServiceConnectionHandshakeFailedException("Unable to connect to the server")

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt

@@ -17,6 +17,7 @@
 package com.google.firebase.ai.type
 
 import android.graphics.Bitmap
+import kotlin.collections.filterNot
 import kotlinx.serialization.EncodeDefault
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.Serializable
@@ -90,14 +91,21 @@ constructor(public val role: String? = "user", public val parts: List<Part>) {
   @Serializable
   internal data class Internal(
     @EncodeDefault val role: String? = "user",
-    val parts: List<InternalPart>
+    val parts: List<InternalPart>? = null
   ) {
     internal fun toPublic(): Content {
+      // Return empty if none of the parts is a known part
+      if (parts == null || parts.filterNot { it is UnknownPart.Internal }.isEmpty()) {
+        return Content(role, emptyList())
+      }
+      // From all the known parts, if they are all text and empty, we coalesce them into a single
+      // one-character string part so the backend doesn't fail if we send this back as part of a
+      // multi-turn interaction.
       val returnedParts =
-        parts.map { it.toPublic() }.filterNot { it is TextPart && it.text.isEmpty() }
-      // If all returned parts were text and empty, we coalesce them into a single one-character
-      // string
-      // part so the backend doesn't fail if we send this back as part of a multi-turn interaction.
+        parts
+          .filterNot { it is UnknownPart.Internal }
+          .map { it.toPublic() }
+          .filterNot { it is TextPart && it.text.isEmpty() }
       return Content(role, returnedParts.ifEmpty { listOf(TextPart(" ")) })
     }
   }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Exceptions.kt

@@ -231,6 +231,10 @@ public class AudioRecordInitializationFailedException(message: String) :
 public class ServiceConnectionHandshakeFailedException(message: String, cause: Throwable? = null) :
   FirebaseAIException(message, cause)
 
+/** The request is missing a permission that is required to perform the requested operation. */
+public class PermissionMissingException(message: String, cause: Throwable? = null) :
+  FirebaseAIException(message, cause)
+
 /** Catch all case for exceptions not explicitly expected. */
 public class UnknownException internal constructor(message: String, cause: Throwable? = null) :
   FirebaseAIException(message, cause)

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/HarmCategory.kt

@@ -31,6 +31,10 @@ public class HarmCategory private constructor(public val ordinal: Int) {
       SEXUALLY_EXPLICIT -> Internal.SEXUALLY_EXPLICIT
       DANGEROUS_CONTENT -> Internal.DANGEROUS_CONTENT
       CIVIC_INTEGRITY -> Internal.CIVIC_INTEGRITY
+      IMAGE_HATE -> Internal.IMAGE_HATE
+      IMAGE_DANGEROUS_CONTENT -> Internal.IMAGE_DANGEROUS_CONTENT
+      IMAGE_HARASSMENT -> Internal.IMAGE_HARASSMENT
+      IMAGE_SEXUALLY_EXPLICIT -> Internal.IMAGE_SEXUALLY_EXPLICIT
       UNKNOWN -> Internal.UNKNOWN
       else -> throw makeMissingCaseException("HarmCategory", ordinal)
     }
@@ -41,7 +45,11 @@ public class HarmCategory private constructor(public val ordinal: Int) {
     @SerialName("HARM_CATEGORY_HATE_SPEECH") HATE_SPEECH,
     @SerialName("HARM_CATEGORY_SEXUALLY_EXPLICIT") SEXUALLY_EXPLICIT,
     @SerialName("HARM_CATEGORY_DANGEROUS_CONTENT") DANGEROUS_CONTENT,
-    @SerialName("HARM_CATEGORY_CIVIC_INTEGRITY") CIVIC_INTEGRITY;
+    @SerialName("HARM_CATEGORY_CIVIC_INTEGRITY") CIVIC_INTEGRITY,
+    @SerialName("HARM_CATEGORY_IMAGE_HATE") IMAGE_HATE,
+    @SerialName("HARM_CATEGORY_IMAGE_DANGEROUS_CONTENT") IMAGE_DANGEROUS_CONTENT,
+    @SerialName("HARM_CATEGORY_IMAGE_HARASSMENT") IMAGE_HARASSMENT,
+    @SerialName("HARM_CATEGORY_IMAGE_SEXUALLY_EXPLICIT") IMAGE_SEXUALLY_EXPLICIT;
 
     internal object Serializer : KSerializer<Internal> by FirstOrdinalSerializer(Internal::class)
 
@@ -52,6 +60,10 @@ public class HarmCategory private constructor(public val ordinal: Int) {
         SEXUALLY_EXPLICIT -> HarmCategory.SEXUALLY_EXPLICIT
         DANGEROUS_CONTENT -> HarmCategory.DANGEROUS_CONTENT
         CIVIC_INTEGRITY -> HarmCategory.CIVIC_INTEGRITY
+        IMAGE_HATE -> HarmCategory.IMAGE_HATE
+        IMAGE_DANGEROUS_CONTENT -> HarmCategory.IMAGE_DANGEROUS_CONTENT
+        IMAGE_HARASSMENT -> HarmCategory.IMAGE_HARASSMENT
+        IMAGE_SEXUALLY_EXPLICIT -> HarmCategory.IMAGE_SEXUALLY_EXPLICIT
         else -> HarmCategory.UNKNOWN
       }
   }
@@ -73,5 +85,17 @@ public class HarmCategory private constructor(public val ordinal: Int) {
 
     /** Content that may be used to harm civic integrity. */
     @JvmField public val CIVIC_INTEGRITY: HarmCategory = HarmCategory(5)
+
+    /** Content that is image hate. */
+    @JvmField public val IMAGE_HATE: HarmCategory = HarmCategory(6)
+
+    /** Image dangerous content. */
+    @JvmField public val IMAGE_DANGEROUS_CONTENT: HarmCategory = HarmCategory(7)
+
+    /** Content is image harassment. */
+    @JvmField public val IMAGE_HARASSMENT: HarmCategory = HarmCategory(8)
+
+    /** Image sexually explicit content. */
+    @JvmField public val IMAGE_SEXUALLY_EXPLICIT: HarmCategory = HarmCategory(9)
   }
 }