Merge branch 'main' into dl/grounding

Author: dlarocque

.github/workflows/dataconnect.yml

@@ -25,7 +25,7 @@ env:
   FDC_JAVA_VERSION: ${{ inputs.javaVersion || '17' }}
   FDC_ANDROID_EMULATOR_API_LEVEL: ${{ inputs.androidEmulatorApiLevel || '34' }}
   FDC_NODEJS_VERSION: ${{ inputs.nodeJsVersion || '20' }}
-  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.2.0' }}
+  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.5.1' }}
   FDC_FIREBASE_TOOLS_DIR: /tmp/firebase-tools
   FDC_FIREBASE_COMMAND: /tmp/firebase-tools/node_modules/.bin/firebase
   FDC_PYTHON_VERSION: ${{ inputs.pythonVersion || '3.13' }}

.github/workflows/dataconnect_demo_app.yml

@@ -18,7 +18,7 @@ on:
 
 env:
   FDC_NODE_VERSION: ${{ inputs.nodeVersion || '20' }}
-  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.2.0' }}
+  FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '14.5.1' }}
   FDC_JAVA_VERSION: ${{ inputs.javaVersion || '17' }}
   FDC_FIREBASE_TOOLS_DIR: ${{ github.workspace }}/firebase-tools
   FDC_FIREBASE_COMMAND: ${{ github.workspace }}/firebase-tools/node_modules/.bin/firebase
@@ -109,7 +109,7 @@ jobs:
           --no-daemon \
           ${{ (inputs.gradleInfoLog && '--info') || '' }} \
           --profile \
-          -PdataConnect.minimalApp.firebaseCommand=${{ env.FDC_FIREBASE_COMMAND }} \
+          -PdataConnect.demo.firebaseCommand=${{ env.FDC_FIREBASE_COMMAND }} \
           assemble test
 
       - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1

.github/workflows/sessions-e2e.yml

@@ -23,7 +23,7 @@ jobs:
       - name: set up JDK 17
         uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0
         with:
-          java-version: '11'
+          java-version: '17'
           distribution: 'temurin'
           cache: gradle
 
@@ -39,4 +39,4 @@ jobs:
         env:
           FTL_RESULTS_BUCKET: fireescape
         run: |
-          ./gradlew :firebase-sessions:test-app:deviceCheck withErrorProne -PtargetBackend="prod" -PtriggerCrashes
+          ./gradlew :firebase-sessions:test-app:deviceCheck withErrorProne -PtargetBackend="prod"

firebase-ai/CHANGELOG.md

@@ -1,8 +1,17 @@
 # Unreleased
-
 * [feature] Add support for Grounding with Google Search (#7042).
+* [changed] Deprecate the `totalBillableCharacters` field (only usable with pre-2.0 models). (#7042)
+* [feature] Added support for extra schema properties like `title`, `minItems`, `maxItems`, `minimum`
+ and `maximum`. As well as support for the `anyOf` schema. (#7013)
+
+# 16.1.0
 * [fixed] Fixed `FirebaseAI.getInstance` StackOverflowException (#6971)
-* [fixed] Fixed an issue that was causing the SDK to send empty `FunctionDeclaration` descriptions to the API. 
+* [fixed] Fixed an issue that was causing the SDK to send empty `FunctionDeclaration` descriptions to the API.
+* [changed] Introduced the `Voice` class, which accepts a voice name, and deprecated the `Voices` class.
+* [changed] **Breaking Change**: Updated `SpeechConfig` to take in `Voice` class instead of `Voices` class.
+    * **Action Required:** Update all references of `SpeechConfig` initialization to use `Voice` class.
+* [fixed] Fix incorrect model name in count token requests to the developer API backend
+
 
 # 16.0.0
 * [feature] Initial release of the Firebase AI SDK (`firebase-ai`). This SDK *replaces* the previous
@@ -21,4 +30,3 @@
 
  Note: This feature is in Public Preview, which means that it is not subject to any SLA or
  deprecation policy and could change in backwards-incompatible ways.
-

firebase-ai/api.txt

@@ -28,7 +28,7 @@ firebaseLibrary {
   testLab.enabled = false
   publishJavadoc = true
   releaseNotes {
-    name.set("{{firebase_ai}}")
+    name.set("{{firebase_ai_logic}}")
     versionName.set("ai")
     hasKTX.set(false)
   }
@@ -67,7 +67,10 @@ android {
     targetSdk = targetSdkVersion
     baseline = file("lint-baseline.xml")
   }
-  sourceSets { getByName("test").java.srcDirs("src/testUtil") }
+  sourceSets {
+    // getByName("test").java.srcDirs("src/testUtil")
+    getByName("androidTest") { kotlin.srcDirs("src/testUtil") }
+  }
 }
 
 // Enable Kotlin "Explicit API Mode". This causes the Kotlin compiler to fail if any

firebase-ai/firebase-ai.gradle.kts

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=16.0.1
-latestReleasedVersion=16.0.0
+version=16.2.0
+latestReleasedVersion=16.1.0

firebase-ai/gradle.properties

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.firebase.ai
+
+import androidx.test.platform.app.InstrumentationRegistry
+import com.google.firebase.FirebaseApp
+import com.google.firebase.FirebaseOptions
+import com.google.firebase.ai.type.GenerativeBackend
+
+class AIModels {
+
+  companion object {
+    private val API_KEY: String = ""
+    private val APP_ID: String = ""
+    private val PROJECT_ID: String = "fireescape-integ-tests"
+    // General purpose models
+    var app: FirebaseApp? = null
+    var flash2Model: GenerativeModel? = null
+    var flash2LiteModel: GenerativeModel? = null
+
+    /** Returns a list of general purpose models to test */
+    fun getModels(): List<GenerativeModel> {
+      if (flash2Model == null) {
+        setup()
+      }
+      return listOf(flash2Model!!, flash2LiteModel!!)
+    }
+
+    fun app(): FirebaseApp {
+      if (app == null) {
+        setup()
+      }
+      return app!!
+    }
+
+    fun setup() {
+      val context = InstrumentationRegistry.getInstrumentation().context
+      app =
+        FirebaseApp.initializeApp(
+          context,
+          FirebaseOptions.Builder()
+            .setApiKey(API_KEY)
+            .setApplicationId(APP_ID)
+            .setProjectId(PROJECT_ID)
+            .build()
+        )
+      flash2Model =
+        FirebaseAI.getInstance(app!!, GenerativeBackend.vertexAI())
+          .generativeModel(
+            modelName = "gemini-2.0-flash",
+          )
+      flash2LiteModel =
+        FirebaseAI.getInstance(app!!, GenerativeBackend.vertexAI())
+          .generativeModel(
+            modelName = "gemini-2.0-flash-lite",
+          )
+    }
+  }
+}

firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/AIModels.kt

@@ -0,0 +1,204 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.firebase.ai
+
+import android.graphics.Bitmap
+import com.google.firebase.ai.AIModels.Companion.getModels
+import com.google.firebase.ai.type.Content
+import com.google.firebase.ai.type.ContentModality
+import com.google.firebase.ai.type.CountTokensResponse
+import java.io.ByteArrayOutputStream
+import kotlinx.coroutines.runBlocking
+import org.junit.Test
+
+class CountTokensTests {
+
+  /** Ensures that the token count is expected for simple words. */
+  @Test
+  fun testCountTokensAmount() {
+    for (model in getModels()) {
+      runBlocking {
+        val response = model.countTokens("this is five different words")
+        assert(response.totalTokens == 5)
+        assert(response.promptTokensDetails.size == 1)
+        assert(response.promptTokensDetails[0].modality == ContentModality.TEXT)
+        assert(response.promptTokensDetails[0].tokenCount == 5)
+      }
+    }
+  }
+
+  /** Ensures that the model returns token counts in the correct modality for text. */
+  @Test
+  fun testCountTokensTextModality() {
+    for (model in getModels()) {
+      runBlocking {
+        val response = model.countTokens("this is a text prompt")
+        checkTokenCountsMatch(response)
+        assert(response.promptTokensDetails.size == 1)
+        assert(containsModality(response, ContentModality.TEXT))
+      }
+    }
+  }
+
+  /** Ensures that the model returns token counts in the correct modality for bitmap images. */
+  @Test
+  fun testCountTokensImageModality() {
+    for (model in getModels()) {
+      runBlocking {
+        val bitmap = Bitmap.createBitmap(10, 10, Bitmap.Config.ARGB_8888)
+        val response = model.countTokens(bitmap)
+        checkTokenCountsMatch(response)
+        assert(response.promptTokensDetails.size == 1)
+        assert(containsModality(response, ContentModality.IMAGE))
+      }
+    }
+  }
+
+  /**
+   * Ensures the model can count tokens for multiple modalities at once, and return the
+   * corresponding token modalities correctly.
+   */
+  @Test
+  fun testCountTokensTextAndImageModality() {
+    for (model in getModels()) {
+      runBlocking {
+        val bitmap = Bitmap.createBitmap(10, 10, Bitmap.Config.ARGB_8888)
+        val response =
+          model.countTokens(
+            Content.Builder().text("this is text").build(),
+            Content.Builder().image(bitmap).build()
+          )
+        checkTokenCountsMatch(response)
+        assert(response.promptTokensDetails.size == 2)
+        assert(containsModality(response, ContentModality.TEXT))
+        assert(containsModality(response, ContentModality.IMAGE))
+      }
+    }
+  }
+
+  /**
+   * Ensures the model can count the tokens for a sent file. Additionally, ensures that the model
+   * treats this sent file as the modality of the mime type, in this case, a plaintext file has its
+   * tokens counted as `ContentModality.TEXT`.
+   */
+  @Test
+  fun testCountTokensTextFileModality() {
+    for (model in getModels()) {
+      runBlocking {
+        val response =
+          model.countTokens(
+            Content.Builder().inlineData("this is text".toByteArray(), "text/plain").build()
+          )
+        checkTokenCountsMatch(response)
+        assert(response.totalTokens == 3)
+        assert(response.promptTokensDetails.size == 1)
+        assert(containsModality(response, ContentModality.TEXT))
+      }
+    }
+  }
+
+  /**
+   * Ensures the model can count the tokens for a sent file. Additionally, ensures that the model
+   * treats this sent file as the modality of the mime type, in this case, a PNG encoded bitmap has
+   * its tokens counted as `ContentModality.IMAGE`.
+   */
+  @Test
+  fun testCountTokensImageFileModality() {
+    for (model in getModels()) {
+      runBlocking {
+        val bitmap = Bitmap.createBitmap(10, 10, Bitmap.Config.ARGB_8888)
+        val stream = ByteArrayOutputStream()
+        bitmap.compress(Bitmap.CompressFormat.PNG, 1, stream)
+        val array = stream.toByteArray()
+        val response = model.countTokens(Content.Builder().inlineData(array, "image/png").build())
+        checkTokenCountsMatch(response)
+        assert(response.promptTokensDetails.size == 1)
+        assert(containsModality(response, ContentModality.IMAGE))
+      }
+    }
+  }
+
+  /**
+   * Ensures that nothing is free, that is, empty content contains no tokens. For some reason, this
+   * is treated as `ContentModality.TEXT`.
+   */
+  @Test
+  fun testCountTokensNothingIsFree() {
+    for (model in getModels()) {
+      runBlocking {
+        val response = model.countTokens(Content.Builder().build())
+        checkTokenCountsMatch(response)
+        assert(response.totalTokens == 0)
+        assert(response.promptTokensDetails.size == 1)
+        assert(containsModality(response, ContentModality.TEXT))
+      }
+    }
+  }
+
+  /**
+   * Checks if the model can count the tokens for a sent file. Additionally, ensures that the model
+   * treats this sent file as the modality of the mime type, in this case, a JSON file is not
+   * recognized, and no tokens are counted. This ensures if/when the model can handle JSON, our
+   * testing makes us aware.
+   */
+  @Test
+  fun testCountTokensJsonFileModality() {
+    for (model in getModels()) {
+      runBlocking {
+        val json =
+          """
+          {
+            "foo": "bar",
+            "baz": 3,
+            "qux": [
+              {
+                "quux": [
+                  1,
+                  2
+                ]
+              }
+            ]
+          }
+        """
+            .trimIndent()
+        val response =
+          model.countTokens(
+            Content.Builder().inlineData(json.toByteArray(), "application/json").build()
+          )
+        checkTokenCountsMatch(response)
+        assert(response.promptTokensDetails.isEmpty())
+        assert(response.totalTokens == 0)
+      }
+    }
+  }
+
+  fun checkTokenCountsMatch(response: CountTokensResponse) {
+    assert(sumTokenCount(response) == response.totalTokens)
+  }
+
+  fun sumTokenCount(response: CountTokensResponse): Int {
+    return response.promptTokensDetails.sumOf { it.tokenCount }
+  }
+
+  fun containsModality(response: CountTokensResponse, modality: ContentModality): Boolean {
+    for (token in response.promptTokensDetails) {
+      if (token.modality == modality) {
+        return true
+      }
+    }
+    return false
+  }
+}