Merge branch 'main' into rl.thinking

Author: rlazo

firebase-ai/CHANGELOG.md

@@ -1,8 +1,14 @@
 # Unreleased
 * [changed] Added better description for requests which fail due to the Gemini API not being
   configured.
+* [changed] Added a `dilation` parameter to `ImagenMaskReference.generateMaskAndPadForOutpainting`
+  (#7260)
+
+# 17.1.0
+=======
 * [feature] added support for Imagen Editing, including inpainting, outpainting, control, style 
   transfer, and subject references (#7075)
+* [feature] **Preview:** Added support for bidirectional streaming in Gemini Developer Api
 
 # 17.0.0
 * [feature] Added support for configuring the "thinking" budget when using Gemini

firebase-ai/api.txt

@@ -698,12 +698,14 @@ package com.google.firebase.ai.type {
   @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenMaskReference extends com.google.firebase.ai.type.ImagenReferenceImage {
     method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions);
     method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER);
+    method public static final java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, double dilation = 0.01);
     field public static final com.google.firebase.ai.type.ImagenMaskReference.Companion Companion;
   }
 
   public static final class ImagenMaskReference.Companion {
     method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions);
     method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER);
+    method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, double dilation = 0.01);
   }
 
   @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {

firebase-ai/gradle.properties

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.1.0
-latestReleasedVersion=17.0.0
+version=17.2.0
+latestReleasedVersion=17.1.0

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt

@@ -19,18 +19,26 @@ package com.google.firebase.ai.type
 public class ImagenControlType internal constructor(internal val value: String) {
   public companion object {
 
-    /** Use edge detection to ensure the new image follow the same outlines */
+    /**
+     * Use edge detection to ensure the new image follows the same outlines as the reference image.
+     */
     @JvmField public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY")
 
-    /** Use enhanced edge detection to ensure the new image follow similar outlines */
+    /**
+     * Use enhanced edge detection to ensure the new image follows the same outlines as the
+     * reference image.
+     */
     @JvmField public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE")
 
-    /** Use face mesh control to ensure that the new image has the same facial expressions */
+    /**
+     * Use face mesh control to ensure that the new image has the same facial expressions as the
+     * reference image.
+     */
     @JvmField public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH")
 
     /**
      * Use color superpixels to ensure that the new image is similar in shape and color to the
-     * original
+     * reference image.
      */
     @JvmField
     public val COLOR_SUPERPIXEL: ImagenControlType =

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt

@@ -25,7 +25,7 @@ public class ImagenEditMode private constructor(internal val value: String) {
     /** Removes an element from an image */
     @JvmField
     public val INPAINT_REMOVAL: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_REMOVAL")
-    /** Extend the borders of an image outwards */
+    /** Extends the borders of an image outwards */
     @JvmField public val OUTPAINT: ImagenEditMode = ImagenEditMode("EDIT_MODE_OUTPAINT")
   }
 }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt

@@ -80,10 +80,11 @@ internal constructor(
  * @param referenceId the reference ID for this image, to be referenced in the prompt
  * @param enableComputation requests that the reference image be generated serverside instead of
  * provided
- * @param superpixelRegionSize if type is COLOR_SUPERPIXEL and [enableComputation] is true, this
- * will control the size of each superpixel region in pixels for the generated referenced image
- * @param superpixelRuler if type is COLOR_SUPERPIXEL and [enableComputation] is true, this will
- * control the superpixel smoothness factor for the generated referenced image
+ * @param superpixelRegionSize if type is [ImagenControlType.COLOR_SUPERPIXEL] and
+ * [enableComputation] is true, this will control the size of each superpixel region in pixels for
+ * the generated referenced image
+ * @param superpixelRuler if type is [ImagenControlType.COLOR_SUPERPIXEL] and [enableComputation] is
+ * true, this will control the superpixel smoothness factor for the generated referenced image
  */
 @PublicPreviewAPI
 public class ImagenControlReference(
@@ -102,9 +103,8 @@ public class ImagenControlReference(
   ) {}
 
 /**
- * Represents a reference image for Imagen editing which will mask of a region to be edited. This
- * image (generated or provided) should contain only black and white pixels, with black representing
- * parts of the image which should not change.
+ * Represents a mask for Imagen editing. This image (generated or provided) should contain only
+ * black and white pixels, with black representing parts of the image which should not change.
  */
 @PublicPreviewAPI
 public abstract class ImagenMaskReference
@@ -113,17 +113,16 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n
 
   public companion object {
     /**
-     * Generates these two reference images in order:
+     * Generates two reference images of [ImagenRawImage] and [ImagenRawMask]. These images are
+     * generated in this order:
      * * One [ImagenRawImage] containing the original image, padded out to the new dimensions with
      * black pixels, with the original image placed at the given placement
      * * One [ImagenRawMask] of the same dimensions containing white everywhere except at the
-     * placement original image.
-     *
-     * This is the format expected by Imagen for outpainting requests.
+     * placement original image. This is the format expected by Imagen for outpainting requests.
      *
      * @param image the original image
-     * @param newDimensions the new dimensions for outpainting. This *must* be more than the
-     * original image.
+     * @param newDimensions the new dimensions for outpainting. These new dimensions *must* be more
+     * than the original image.
      * @param newPosition the placement of the original image within the new outpainted image.
      */
     @JvmOverloads
@@ -132,6 +131,29 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n
       image: ImagenInlineImage,
       newDimensions: Dimensions,
       newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER,
+    ): List<ImagenReferenceImage> =
+      generateMaskAndPadForOutpainting(image, newDimensions, newPosition, 0.01)
+
+    /**
+     * Generates two reference images of [ImagenRawImage] and [ImagenRawMask]. These images are
+     * generated in this order:
+     * * One [ImagenRawImage] containing the original image, padded out to the new dimensions with
+     * black pixels, with the original image placed at the given placement
+     * * One [ImagenRawMask] of the same dimensions containing white everywhere except at the
+     * placement original image. This is the format expected by Imagen for outpainting requests.
+     *
+     * @param image the original image
+     * @param newDimensions the new dimensions for outpainting. These new dimensions *must* be more
+     * than the original image.
+     * @param newPosition the placement of the original image within the new outpainted image.
+     * @param dilation the dilation for the outpainting mask. See: [ImagenRawMask].
+     */
+    @JvmStatic
+    public fun generateMaskAndPadForOutpainting(
+      image: ImagenInlineImage,
+      newDimensions: Dimensions,
+      newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER,
+      dilation: Double = 0.01
     ): List<ImagenReferenceImage> {
       val originalBitmap = image.asBitmap()
       if (
@@ -181,7 +203,7 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n
       newImageCanvas.drawBitmap(originalBitmap, null, normalizedImageRectangle, null)
       return listOf(
         ImagenRawImage(newImageBitmap.toImagenInlineImage()),
-        ImagenRawMask(maskBitmap.toImagenInlineImage()),
+        ImagenRawMask(maskBitmap.toImagenInlineImage(), dilation),
       )
     }
   }
@@ -190,8 +212,8 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n
 /**
  * A generated mask image which will auto-detect and mask out the background. The background will be
  * white, and the foreground black
- * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and
- * make it seem more convincing. For example, 0.05 would dilate the mask 5%.
+ * @param dilation the amount to dilate the mask. This can help smooth the borders of an edit and
+ * make it seem more convincing. For example, `0.05` will dilate the mask 5%.
  */
 @PublicPreviewAPI
 public class ImagenBackgroundMask(dilation: Double? = null) :
@@ -200,21 +222,20 @@ public class ImagenBackgroundMask(dilation: Double? = null) :
 /**
  * A generated mask image which will auto-detect and mask out the foreground. The background will be
  * black, and the foreground white
- * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and
- * make it seem more convincing. For example, 0.05 would dilate the mask 5%.
+ * @param dilation the amount to dilate the mask. This can help smooth the borders of an edit and
+ * make it seem more convincing. For example, `0.05` will dilate the mask 5%.
  */
 @PublicPreviewAPI
 public class ImagenForegroundMask(dilation: Double? = null) :
   ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.FOREGROUND, dilation)) {}
 
 /**
- * Represents a reference image for Imagen editing which will mask of a region to be edited. This
- * image should contain only black and white pixels, with black representing parts of the image
- * which should not change.
+ * Represents a mask for Imagen editing. This image should contain only black and white pixels, with
+ * black representing parts of the image which should not change.
  *
  * @param mask the mask image
- * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and
- * make it seem more convincing. For example, 0.05 would dilate the mask 5%.
+ * @param dilation the amount to dilate the mask. This can help smooth the borders of an edit and
+ * make it seem more convincing. For example, `0.05` will dilate the mask 5%.
  */
 @PublicPreviewAPI
 public class ImagenRawMask(mask: ImagenInlineImage, dilation: Double? = null) :
@@ -226,11 +247,11 @@ public class ImagenRawMask(mask: ImagenInlineImage, dilation: Double? = null) :
 /**
  * Represents a generated mask for Imagen editing which masks out certain objects using object
  * detection.
- * @param classes the list of segmentation IDs for objects to detect and mask out. See
- * [here](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api-edit#segment-ids)
- * for a list of segmentation IDs
- * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and
- * make it seem more convincing. For example, 0.05 would dilate the mask 5%.
+ * @param classes the list of segmentation IDs for objects to detect and mask out. Find a
+ * [list of segmentation IDs](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api-edit#segment-ids)
+ * in the Vertex AI documentation.
+ * @param dilation the amount to dilate the mask. This can help smooth the borders of an edit and
+ * make it seem more convincing. For example, `0.05` will dilate the mask 5%.
  */
 @PublicPreviewAPI
 public class ImagenSemanticMask(classes: List<Int>, dilation: Double? = null) :

firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutableLauncher.kt

@@ -21,10 +21,12 @@ import org.gradle.api.Task
 interface DataConnectExecutableConfig {
   var outputDirectory: File?
   var connectors: Collection<String>
+  var connectorId: Collection<String>
   var listen: String?
   var localConnectionString: String?
   var logFile: File?
   var schemaExtensionsOutputEnabled: Boolean?
+  var platform: String?
 }
 
 fun Task.runDataConnectExecutable(
@@ -37,10 +39,12 @@ fun Task.runDataConnectExecutable(
     object : DataConnectExecutableConfig {
         override var outputDirectory: File? = null
         override var connectors: Collection<String> = emptyList()
+        override var connectorId: Collection<String> = emptyList()
         override var listen: String? = null
         override var localConnectionString: String? = null
         override var logFile: File? = null
         override var schemaExtensionsOutputEnabled: Boolean? = null
+        override var platform: String? = null
       }
       .apply(configure)
 
@@ -76,7 +80,13 @@ fun Task.runDataConnectExecutable(
             args("-connectors=${it.joinToString(",")}")
           }
         }
+        config.connectorId.let {
+          if (it.isNotEmpty()) {
+            args("-connector_id=${it.joinToString(",")}")
+          }
+        }
         config.listen?.let { args("-listen=${it}") }
+        config.platform?.let { args("-platform=${it}") }
         config.localConnectionString?.let { args("-local_connection_string=${it}") }
         config.schemaExtensionsOutputEnabled?.let { args("-enable_output_schema_extensions=${it}") }
       }