Merge branch 'main' into code_execution_support

Author: VinayGuthal

.github/workflows/ai-nightlies.yml

@@ -0,0 +1,66 @@
+name: Firebase AI Nightlies
+
+on:
+  schedule:
+    - cron: 2 7 * * *  # Runs automatically once a day
+  workflow_dispatch:   # Allow triggering the workflow manually
+
+jobs:
+  nightlies:
+    name: "Nightlies"
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 2
+          submodules: true
+
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 # v4.7.0
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: gradle
+
+      - name: Add google-services.json
+        env:
+          INTEG_TESTS_GOOGLE_SERVICES: ${{ secrets.INTEG_TESTS_GOOGLE_SERVICES }}
+        run: |
+          echo $INTEG_TESTS_GOOGLE_SERVICES | base64 -d > google-services.json
+      - uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+      - uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4
+
+      - name: Run tests
+        uses: reactivecircus/android-emulator-runner@62dbb605bba737720e10b196cb4220d374026a6d #v2.33.0
+        env:
+          FIREBASE_CI: 1
+          FTL_RESULTS_BUCKET: android-ci
+          FTL_RESULTS_DIR: ${{ format('logs/{0}/{1}_{2}/artifacts/', github.workflow, github.run_id, github.run_attempt) }}
+          FIREBASE_APP_CHECK_DEBUG_SECRET: ${{ secrets.FIREBASE_APP_CHECK_DEBUG_SECRET }}
+        with:
+          api-level: 31
+          arch: x86_64
+          ram-size: 4096M
+          heap-size: 4096M
+          script: |
+            adb logcat -v time > logcat.txt &
+            ./gradlew firebase-ai:connectedCheck withErrorProne -PtargetBackend="prod"
+
+      - name: Upload logs
+        if: failure()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: logcat.txt
+          path: logcat.txt
+          retention-days: 7
+          if-no-files-found: ignore

firebase-ai/CHANGELOG.md

@@ -1,6 +1,18 @@
 # Unreleased
-* [feature] added support for Imagen Editing, including inpainting, outpainting, control, style 
+* [feature] Added support for returning thought summaries, which are synthesized
+  versions of a model's internal reasoning process.
+* [fixed] Fixed an issue causing the accessor methods in `GenerateContentResponse` to throw an exception
+  when the response contained no candidates.
+* [changed] Added better description for requests which fail due to the Gemini API not being
+  configured.
+* [changed] Added a `dilation` parameter to `ImagenMaskReference.generateMaskAndPadForOutpainting`
+  (#7260)
+
+# 17.1.0
+=======
+* [feature] added support for Imagen Editing, including inpainting, outpainting, control, style
   transfer, and subject references (#7075)
+* [feature] **Preview:** Added support for bidirectional streaming in Gemini Developer Api
 
 # 17.0.0
 * [feature] Added support for configuring the "thinking" budget when using Gemini
@@ -43,4 +55,3 @@
 
  Note: This feature is in Public Preview, which means that it is not subject to any SLA or
  deprecation policy and could change in backwards-incompatible ways.
-

firebase-ai/api.txt

@@ -157,6 +157,9 @@ package com.google.firebase.ai.java {
 
 package com.google.firebase.ai.type {
 
+  public final class APINotConfiguredException extends com.google.firebase.ai.type.FirebaseAIException {
+  }
+
   public final class AudioRecordInitializationFailedException extends com.google.firebase.ai.type.FirebaseAIException {
     ctor public AudioRecordInitializationFailedException(String message);
   }
@@ -214,6 +217,8 @@ package com.google.firebase.ai.type {
     ctor public CodeExecutionResultPart(String outcome, String output);
     method public String getOutcome();
     method public String getOutput();
+    method public boolean isThought();
+    property public boolean isThought;
     property public final String outcome;
     property public final String output;
   }
@@ -289,14 +294,18 @@ package com.google.firebase.ai.type {
     ctor public ExecutableCodePart(String language, String code);
     method public String getCode();
     method public String getLanguage();
+    method public boolean isThought();
     property public final String code;
+    property public boolean isThought;
     property public final String language;
   }
 
   public final class FileDataPart implements com.google.firebase.ai.type.Part {
     ctor public FileDataPart(String uri, String mimeType);
     method public String getMimeType();
     method public String getUri();
+    method public boolean isThought();
+    property public boolean isThought;
     property public final String mimeType;
     property public final String uri;
   }
@@ -331,8 +340,10 @@ package com.google.firebase.ai.type {
     method public java.util.Map<java.lang.String,kotlinx.serialization.json.JsonElement> getArgs();
     method public String? getId();
     method public String getName();
+    method public boolean isThought();
     property public final java.util.Map<java.lang.String,kotlinx.serialization.json.JsonElement> args;
     property public final String? id;
+    property public boolean isThought;
     property public final String name;
   }
 
@@ -361,7 +372,9 @@ package com.google.firebase.ai.type {
     method public String? getId();
     method public String getName();
     method public kotlinx.serialization.json.JsonObject getResponse();
+    method public boolean isThought();
     property public final String? id;
+    property public boolean isThought;
     property public final String name;
     property public final kotlinx.serialization.json.JsonObject response;
   }
@@ -375,6 +388,7 @@ package com.google.firebase.ai.type {
     method public java.util.List<com.google.firebase.ai.type.InlineDataPart> getInlineDataParts();
     method public com.google.firebase.ai.type.PromptFeedback? getPromptFeedback();
     method public String? getText();
+    method public String? getThoughtSummary();
     method public com.google.firebase.ai.type.UsageMetadata? getUsageMetadata();
     property public final java.util.List<com.google.firebase.ai.type.Candidate> candidates;
     property public final java.util.List<com.google.firebase.ai.type.CodeExecutionResultPart> codeExecutionResults;
@@ -383,6 +397,7 @@ package com.google.firebase.ai.type {
     property public final java.util.List<com.google.firebase.ai.type.InlineDataPart> inlineDataParts;
     property public final com.google.firebase.ai.type.PromptFeedback? promptFeedback;
     property public final String? text;
+    property public final String? thoughtSummary;
     property public final com.google.firebase.ai.type.UsageMetadata? usageMetadata;
   }
 
@@ -553,7 +568,9 @@ package com.google.firebase.ai.type {
   public final class ImagePart implements com.google.firebase.ai.type.Part {
     ctor public ImagePart(android.graphics.Bitmap image);
     method public android.graphics.Bitmap getImage();
+    method public boolean isThought();
     property public final android.graphics.Bitmap image;
+    property public boolean isThought;
   }
 
   @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenAspectRatio {
@@ -702,12 +719,14 @@ package com.google.firebase.ai.type {
   @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenMaskReference extends com.google.firebase.ai.type.ImagenReferenceImage {
     method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions);
     method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER);
+    method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, double dilation = 0.01);
     field public static final com.google.firebase.ai.type.ImagenMaskReference.Companion Companion;
   }
 
   public static final class ImagenMaskReference.Companion {
     method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions);
     method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER);
+    method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, double dilation = 0.01);
   }
 
   @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {
@@ -776,7 +795,9 @@ package com.google.firebase.ai.type {
     ctor public InlineDataPart(byte[] inlineData, String mimeType);
     method public byte[] getInlineData();
     method public String getMimeType();
+    method public boolean isThought();
     property public final byte[] inlineData;
+    property public boolean isThought;
     property public final String mimeType;
   }
 
@@ -885,6 +906,8 @@ package com.google.firebase.ai.type {
   }
 
   public interface Part {
+    method public boolean isThought();
+    property public abstract boolean isThought;
   }
 
   public final class PartKt {
@@ -1145,6 +1168,8 @@ package com.google.firebase.ai.type {
   public final class TextPart implements com.google.firebase.ai.type.Part {
     ctor public TextPart(String text);
     method public String getText();
+    method public boolean isThought();
+    property public boolean isThought;
     property public final String text;
   }
 
@@ -1154,6 +1179,7 @@ package com.google.firebase.ai.type {
   public static final class ThinkingConfig.Builder {
     ctor public ThinkingConfig.Builder();
     method public com.google.firebase.ai.type.ThinkingConfig build();
+    method public com.google.firebase.ai.type.ThinkingConfig.Builder setIncludeThoughts(boolean includeThoughts);
     method public com.google.firebase.ai.type.ThinkingConfig.Builder setThinkingBudget(int thinkingBudget);
   }
 

firebase-ai/firebase-ai.gradle.kts

@@ -21,6 +21,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 plugins {
   id("firebase-library")
   id("kotlin-android")
+  id("copy-google-services")
   alias(libs.plugins.kotlinx.serialization)
 }
 

firebase-ai/gradle.properties

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.1.0
-latestReleasedVersion=17.0.0
+version=17.2.0
+latestReleasedVersion=17.1.0

firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/AIModels.kt

@@ -17,7 +17,6 @@ package com.google.firebase.ai
 
 import androidx.test.platform.app.InstrumentationRegistry
 import com.google.firebase.FirebaseApp
-import com.google.firebase.FirebaseOptions
 import com.google.firebase.ai.type.GenerativeBackend
 
 class AIModels {
@@ -48,15 +47,7 @@ class AIModels {
 
     fun setup() {
       val context = InstrumentationRegistry.getInstrumentation().context
-      app =
-        FirebaseApp.initializeApp(
-          context,
-          FirebaseOptions.Builder()
-            .setApiKey(API_KEY)
-            .setApplicationId(APP_ID)
-            .setProjectId(PROJECT_ID)
-            .build()
-        )
+      app = FirebaseApp.initializeApp(context)
       flash2Model =
         FirebaseAI.getInstance(app!!, GenerativeBackend.vertexAI())
           .generativeModel(

firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt

@@ -22,10 +22,13 @@ import com.google.firebase.ai.type.ImagenEditingConfig
 import com.google.firebase.ai.type.ImagenRawImage
 import com.google.firebase.ai.type.PublicPreviewAPI
 import kotlinx.coroutines.runBlocking
+import org.junit.Ignore
 import org.junit.Test
 
 @OptIn(PublicPreviewAPI::class)
 class ImagenTests {
+
+  @Ignore("Currently not supported by backend model")
   @Test
   fun testGenerateAndEditImage() {
     val imageGenerationModel = FirebaseAI.getInstance(app()).imagenModel("imagen-3.0-generate-002")

firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt

@@ -334,6 +334,9 @@ private suspend fun validateResponse(response: HttpResponse) {
   if (message.contains("The prompt could not be submitted")) {
     throw PromptBlockedException(message)
   }
+  if (message.contains("genai config not found")) {
+    throw APINotConfiguredException()
+  }
   getServiceDisabledErrorDetailsOrNull(error)?.let {
     val errorMessage =
       if (it.metadata?.get("service") == "firebasevertexai.googleapis.com") {

firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Exceptions.kt

@@ -89,6 +89,16 @@ internal constructor(
 internal class UnsupportedUserLocationException(cause: Throwable? = null) :
   FirebaseCommonAIException("User location is not supported for the API use.", cause)
 
+/**
+ * The user's project does not have the Gemini Developer API enabled in the Firebase Console.
+ *
+ * See the Firebase documentation for the
+ * [steps](https://firebase.google.com/docs/ai-logic/faq-and-troubleshooting?api=dev#error-genai-config-not-found)
+ * to enable the Gemini Developer API.
+ */
+internal class APINotConfiguredException(cause: Throwable? = null) :
+  FirebaseCommonAIException("Gemini Developer API not enabled in Firebase console.", cause)
+
 /**
  * Some form of state occurred that shouldn't have.
  *

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Exceptions.kt

@@ -61,6 +61,8 @@ internal constructor(message: String, cause: Throwable? = null) : RuntimeExcepti
               ContentBlockedException(cause.message ?: "", cause.cause)
             is com.google.firebase.ai.common.QuotaExceededException ->
               QuotaExceededException(cause.message ?: "", cause.cause)
+            is com.google.firebase.ai.common.APINotConfiguredException ->
+              APINotConfiguredException(cause.cause)
             else -> UnknownException(cause.message ?: "", cause)
           }
         is TimeoutCancellationException ->
@@ -149,6 +151,16 @@ internal constructor(message: String, cause: Throwable? = null) :
 public class UnsupportedUserLocationException internal constructor(cause: Throwable? = null) :
   FirebaseAIException("User location is not supported for the API use.", cause)
 
+/**
+ * The user's project does not have the Gemini Developer API enabled in the Firebase Console.
+ *
+ * See the Firebase documentation for the
+ * [steps](https://firebase.google.com/docs/ai-logic/faq-and-troubleshooting?api=dev#error-genai-config-not-found)
+ * to enable the Gemini Developer API.
+ */
+public class APINotConfiguredException internal constructor(cause: Throwable? = null) :
+  FirebaseAIException("Gemini Developer API not enabled in Firebase console.", cause)
+
 /**
  * Some form of state occurred that shouldn't have.
  *