Merge branch 'main' into td/app-start

Author: tejasd

firebase-ai/CHANGELOG.md

@@ -1,8 +1,16 @@
 # Unreleased
+* [fixed] Fixed an issue causing the accessor methods in `GenerateContentResponse` to throw an exception
+  when the response contained no candidates.
 * [changed] Added better description for requests which fail due to the Gemini API not being
   configured.
-* [feature] added support for Imagen Editing, including inpainting, outpainting, control, style 
+* [changed] Added a `dilation` parameter to `ImagenMaskReference.generateMaskAndPadForOutpainting`
+  (#7260)
+
+# 17.1.0
+=======
+* [feature] added support for Imagen Editing, including inpainting, outpainting, control, style
   transfer, and subject references (#7075)
+* [feature] **Preview:** Added support for bidirectional streaming in Gemini Developer Api
 
 # 17.0.0
 * [feature] Added support for configuring the "thinking" budget when using Gemini
@@ -45,4 +53,3 @@
 
  Note: This feature is in Public Preview, which means that it is not subject to any SLA or
  deprecation policy and could change in backwards-incompatible ways.
-

firebase-ai/api.txt

@@ -157,6 +157,9 @@ package com.google.firebase.ai.java {
 
 package com.google.firebase.ai.type {
 
+  public final class APINotConfiguredException extends com.google.firebase.ai.type.FirebaseAIException {
+  }
+
   public final class AudioRecordInitializationFailedException extends com.google.firebase.ai.type.FirebaseAIException {
     ctor public AudioRecordInitializationFailedException(String message);
   }
@@ -698,12 +701,14 @@ package com.google.firebase.ai.type {
   @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenMaskReference extends com.google.firebase.ai.type.ImagenReferenceImage {
     method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions);
     method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER);
+    method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, double dilation = 0.01);
     field public static final com.google.firebase.ai.type.ImagenMaskReference.Companion Companion;
   }
 
   public static final class ImagenMaskReference.Companion {
     method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions);
     method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER);
+    method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, double dilation = 0.01);
   }
 
   @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {

firebase-ai/gradle.properties

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.1.0
-latestReleasedVersion=17.0.0
+version=17.2.0
+latestReleasedVersion=17.1.0

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Exceptions.kt

@@ -158,7 +158,7 @@ public class UnsupportedUserLocationException internal constructor(cause: Throwa
  * [steps](https://firebase.google.com/docs/ai-logic/faq-and-troubleshooting?api=dev#error-genai-config-not-found)
  * to enable the Gemini Developer API.
  */
-internal class APINotConfiguredException internal constructor(cause: Throwable? = null) :
+public class APINotConfiguredException internal constructor(cause: Throwable? = null) :
   FirebaseAIException("Gemini Developer API not enabled in Firebase console.", cause)
 
 /**

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/GenerateContentResponse.kt

@@ -32,28 +32,42 @@ public class GenerateContentResponse(
   public val usageMetadata: UsageMetadata?,
 ) {
   /**
-   * Convenience field representing all the text parts in the response as a single string, if they
-   * exists.
+   * Convenience field representing all the text parts in the response as a single string.
+   *
+   * The value is null if the response contains no [candidates].
    */
   public val text: String? by lazy {
-    candidates.first().content.parts.filterIsInstance<TextPart>().joinToString(" ") { it.text }
+    candidates.firstOrNull()?.content?.parts?.filterIsInstance<TextPart>()?.joinToString(" ") {
+      it.text
+    }
   }
 
-  /** Convenience field to list all the [FunctionCallPart]s in the response, if they exist. */
+  /**
+   * Convenience field to list all the [FunctionCallPart]s in the response.
+   *
+   * The value is an empty list if the response contains no [candidates].
+   */
   public val functionCalls: List<FunctionCallPart> by lazy {
-    candidates.first().content.parts.filterIsInstance<FunctionCallPart>()
+    candidates.firstOrNull()?.content?.parts?.filterIsInstance<FunctionCallPart>().orEmpty()
   }
 
   /**
    * Convenience field representing all the [InlineDataPart]s in the first candidate, if they exist.
    *
    * This also includes any [ImagePart], but they will be represented as [InlineDataPart] instead.
+   *
+   * The value is an empty list if the response contains no [candidates].
    */
   public val inlineDataParts: List<InlineDataPart> by lazy {
-    candidates.first().content.parts.let { parts ->
-      parts.filterIsInstance<ImagePart>().map { it.toInlineDataPart() } +
-        parts.filterIsInstance<InlineDataPart>()
-    }
+    candidates
+      .firstOrNull()
+      ?.content
+      ?.parts
+      ?.let { parts ->
+        parts.filterIsInstance<ImagePart>().map { it.toInlineDataPart() } +
+          parts.filterIsInstance<InlineDataPart>()
+      }
+      .orEmpty()
   }
 
   @Serializable

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt

@@ -19,18 +19,26 @@ package com.google.firebase.ai.type
 public class ImagenControlType internal constructor(internal val value: String) {
   public companion object {
 
-    /** Use edge detection to ensure the new image follow the same outlines */
+    /**
+     * Use edge detection to ensure the new image follows the same outlines as the reference image.
+     */
     @JvmField public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY")
 
-    /** Use enhanced edge detection to ensure the new image follow similar outlines */
+    /**
+     * Use enhanced edge detection to ensure the new image follows the same outlines as the
+     * reference image.
+     */
     @JvmField public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE")
 
-    /** Use face mesh control to ensure that the new image has the same facial expressions */
+    /**
+     * Use face mesh control to ensure that the new image has the same facial expressions as the
+     * reference image.
+     */
     @JvmField public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH")
 
     /**
      * Use color superpixels to ensure that the new image is similar in shape and color to the
-     * original
+     * reference image.
      */
     @JvmField
     public val COLOR_SUPERPIXEL: ImagenControlType =

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt

@@ -25,7 +25,7 @@ public class ImagenEditMode private constructor(internal val value: String) {
     /** Removes an element from an image */
     @JvmField
     public val INPAINT_REMOVAL: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_REMOVAL")
-    /** Extend the borders of an image outwards */
+    /** Extends the borders of an image outwards */
     @JvmField public val OUTPAINT: ImagenEditMode = ImagenEditMode("EDIT_MODE_OUTPAINT")
   }
 }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt

@@ -80,10 +80,11 @@ internal constructor(
  * @param referenceId the reference ID for this image, to be referenced in the prompt
  * @param enableComputation requests that the reference image be generated serverside instead of
  * provided
- * @param superpixelRegionSize if type is COLOR_SUPERPIXEL and [enableComputation] is true, this
- * will control the size of each superpixel region in pixels for the generated referenced image
- * @param superpixelRuler if type is COLOR_SUPERPIXEL and [enableComputation] is true, this will
- * control the superpixel smoothness factor for the generated referenced image
+ * @param superpixelRegionSize if type is [ImagenControlType.COLOR_SUPERPIXEL] and
+ * [enableComputation] is true, this will control the size of each superpixel region in pixels for
+ * the generated referenced image
+ * @param superpixelRuler if type is [ImagenControlType.COLOR_SUPERPIXEL] and [enableComputation] is
+ * true, this will control the superpixel smoothness factor for the generated referenced image
  */
 @PublicPreviewAPI
 public class ImagenControlReference(
@@ -102,9 +103,8 @@ public class ImagenControlReference(
   ) {}
 
 /**
- * Represents a reference image for Imagen editing which will mask of a region to be edited. This
- * image (generated or provided) should contain only black and white pixels, with black representing
- * parts of the image which should not change.
+ * Represents a mask for Imagen editing. This image (generated or provided) should contain only
+ * black and white pixels, with black representing parts of the image which should not change.
  */
 @PublicPreviewAPI
 public abstract class ImagenMaskReference
@@ -113,17 +113,16 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n
 
   public companion object {
     /**
-     * Generates these two reference images in order:
+     * Generates two reference images of [ImagenRawImage] and [ImagenRawMask]. These images are
+     * generated in this order:
      * * One [ImagenRawImage] containing the original image, padded out to the new dimensions with
      * black pixels, with the original image placed at the given placement
      * * One [ImagenRawMask] of the same dimensions containing white everywhere except at the
-     * placement original image.
-     *
-     * This is the format expected by Imagen for outpainting requests.
+     * placement original image. This is the format expected by Imagen for outpainting requests.
      *
      * @param image the original image
-     * @param newDimensions the new dimensions for outpainting. This *must* be more than the
-     * original image.
+     * @param newDimensions the new dimensions for outpainting. These new dimensions *must* be more
+     * than the original image.
      * @param newPosition the placement of the original image within the new outpainted image.
      */
     @JvmOverloads
@@ -132,6 +131,29 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n
       image: ImagenInlineImage,
       newDimensions: Dimensions,
       newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER,
+    ): List<ImagenReferenceImage> =
+      generateMaskAndPadForOutpainting(image, newDimensions, newPosition, 0.01)
+
+    /**
+     * Generates two reference images of [ImagenRawImage] and [ImagenRawMask]. These images are
+     * generated in this order:
+     * * One [ImagenRawImage] containing the original image, padded out to the new dimensions with
+     * black pixels, with the original image placed at the given placement
+     * * One [ImagenRawMask] of the same dimensions containing white everywhere except at the
+     * placement original image. This is the format expected by Imagen for outpainting requests.
+     *
+     * @param image the original image
+     * @param newDimensions the new dimensions for outpainting. These new dimensions *must* be more
+     * than the original image.
+     * @param newPosition the placement of the original image within the new outpainted image.
+     * @param dilation the dilation for the outpainting mask. See: [ImagenRawMask].
+     */
+    @JvmStatic
+    public fun generateMaskAndPadForOutpainting(
+      image: ImagenInlineImage,
+      newDimensions: Dimensions,
+      newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER,
+      dilation: Double = 0.01
     ): List<ImagenReferenceImage> {
       val originalBitmap = image.asBitmap()
       if (
@@ -181,7 +203,7 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n
       newImageCanvas.drawBitmap(originalBitmap, null, normalizedImageRectangle, null)
       return listOf(
         ImagenRawImage(newImageBitmap.toImagenInlineImage()),
-        ImagenRawMask(maskBitmap.toImagenInlineImage()),
+        ImagenRawMask(maskBitmap.toImagenInlineImage(), dilation),
       )
     }
   }
@@ -190,8 +212,8 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n
 /**
  * A generated mask image which will auto-detect and mask out the background. The background will be
  * white, and the foreground black
- * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and
- * make it seem more convincing. For example, 0.05 would dilate the mask 5%.
+ * @param dilation the amount to dilate the mask. This can help smooth the borders of an edit and
+ * make it seem more convincing. For example, `0.05` will dilate the mask 5%.
  */
 @PublicPreviewAPI
 public class ImagenBackgroundMask(dilation: Double? = null) :
@@ -200,21 +222,20 @@ public class ImagenBackgroundMask(dilation: Double? = null) :
 /**
  * A generated mask image which will auto-detect and mask out the foreground. The background will be
  * black, and the foreground white
- * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and
- * make it seem more convincing. For example, 0.05 would dilate the mask 5%.
+ * @param dilation the amount to dilate the mask. This can help smooth the borders of an edit and
+ * make it seem more convincing. For example, `0.05` will dilate the mask 5%.
  */
 @PublicPreviewAPI
 public class ImagenForegroundMask(dilation: Double? = null) :
   ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.FOREGROUND, dilation)) {}
 
 /**
- * Represents a reference image for Imagen editing which will mask of a region to be edited. This
- * image should contain only black and white pixels, with black representing parts of the image
- * which should not change.
+ * Represents a mask for Imagen editing. This image should contain only black and white pixels, with
+ * black representing parts of the image which should not change.
  *
  * @param mask the mask image
- * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and
- * make it seem more convincing. For example, 0.05 would dilate the mask 5%.
+ * @param dilation the amount to dilate the mask. This can help smooth the borders of an edit and
+ * make it seem more convincing. For example, `0.05` will dilate the mask 5%.
  */
 @PublicPreviewAPI
 public class ImagenRawMask(mask: ImagenInlineImage, dilation: Double? = null) :
@@ -226,11 +247,11 @@ public class ImagenRawMask(mask: ImagenInlineImage, dilation: Double? = null) :
 /**
  * Represents a generated mask for Imagen editing which masks out certain objects using object
  * detection.
- * @param classes the list of segmentation IDs for objects to detect and mask out. See
- * [here](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api-edit#segment-ids)
- * for a list of segmentation IDs
- * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and
- * make it seem more convincing. For example, 0.05 would dilate the mask 5%.
+ * @param classes the list of segmentation IDs for objects to detect and mask out. Find a
+ * [list of segmentation IDs](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api-edit#segment-ids)
+ * in the Vertex AI documentation.
+ * @param dilation the amount to dilate the mask. This can help smooth the borders of an edit and
+ * make it seem more convincing. For example, `0.05` will dilate the mask 5%.
  */
 @PublicPreviewAPI
 public class ImagenSemanticMask(classes: List<Int>, dilation: Double? = null) :

firebase-ai/src/test/java/com/google/firebase/ai/DevAPIUnarySnapshotTests.kt

@@ -22,12 +22,12 @@ import com.google.firebase.ai.type.ResponseStoppedException
 import com.google.firebase.ai.type.ServerException
 import com.google.firebase.ai.util.goldenDevAPIUnaryFile
 import io.kotest.assertions.throwables.shouldThrow
+import io.kotest.matchers.collections.shouldBeEmpty
 import io.kotest.matchers.collections.shouldNotBeEmpty
 import io.kotest.matchers.nulls.shouldBeNull
 import io.kotest.matchers.nulls.shouldNotBeNull
 import io.kotest.matchers.should
 import io.kotest.matchers.shouldBe
-import io.kotest.matchers.shouldNotBe
 import io.ktor.http.HttpStatusCode
 import kotlin.time.Duration.Companion.seconds
 import kotlinx.coroutines.withTimeout
@@ -42,9 +42,24 @@ internal class DevAPIUnarySnapshotTests {
       withTimeout(testTimeout) {
         val response = model.generateContent("prompt")
 
-        response.candidates.isEmpty() shouldBe false
+        response.candidates.shouldNotBeEmpty()
         response.candidates.first().finishReason shouldBe FinishReason.STOP
-        response.candidates.first().content.parts.isEmpty() shouldBe false
+        response.candidates.first().content.parts.shouldNotBeEmpty()
+      }
+    }
+
+  @Test
+  fun `only prompt feedback reply`() =
+    goldenDevAPIUnaryFile("unary-failure-only-prompt-feedback.json") {
+      withTimeout(testTimeout) {
+        val response = model.generateContent("prompt")
+
+        response.candidates.shouldBeEmpty()
+
+        // Check response from accessors
+        response.text.shouldBeNull()
+        response.functionCalls.shouldBeEmpty()
+        response.inlineDataParts.shouldBeEmpty()
       }
     }
 
@@ -54,9 +69,9 @@ internal class DevAPIUnarySnapshotTests {
       withTimeout(testTimeout) {
         val response = model.generateContent("prompt")
 
-        response.candidates.isEmpty() shouldBe false
+        response.candidates.shouldNotBeEmpty()
         response.candidates.first().finishReason shouldBe FinishReason.STOP
-        response.candidates.first().content.parts.isEmpty() shouldBe false
+        response.candidates.first().content.parts.shouldNotBeEmpty()
       }
     }
 
@@ -66,11 +81,11 @@ internal class DevAPIUnarySnapshotTests {
       withTimeout(testTimeout) {
         val response = model.generateContent("prompt")
 
-        response.candidates.isEmpty() shouldBe false
+        response.candidates.shouldNotBeEmpty()
         response.candidates.first().citationMetadata?.citations?.size shouldBe 4
         response.candidates.first().citationMetadata?.citations?.forEach {
-          it.startIndex shouldNotBe null
-          it.endIndex shouldNotBe null
+          it.startIndex.shouldNotBeNull()
+          it.endIndex.shouldNotBeNull()
         }
       }
     }