Merge branch 'main' into common_dep

Author: rlazo

firebase-ai/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Unreleased
 
+- [feature] Added support for code execution.
+- [changed] Marked the public constructors for `ExecutableCodePart` and `CodeExecutionResultPart` as
+  deprecated.
 - [feature] Introduced `MissingPermissionsException`, which is thrown when the necessary permissions
   have not been granted by the user.
 - [feature] Added helper functions to `LiveSession` to allow developers to track the status of the
@@ -19,9 +22,12 @@
 
 * [changed] Added a `dilation` parameter to `ImagenMaskReference.generateMaskAndPadForOutpainting`
   (#7260)
-* [feature] Added support for limited-use tokens with Firebase App Check. These short-lived tokens
-  provide greater protection for the APIs that give you access to Gemini and Imagen models. Learn
-  how to [enable usage of limited-use tokens](https://firebase.google.com/docs/ai-logic/app-check).
+* [feature] Added support for limited-use tokens with Firebase App Check.
+  These limited-use tokens are required for an upcoming optional feature called
+  _replay protection_. We recommend
+  [enabling the usage of limited-use tokens](https://firebase.google.com/docs/ai-logic/app-check)
+  now so that when replay protection becomes available, you can enable it sooner
+  because more of your users will be on versions of your app that send limited-use tokens.
   (#7285)
 
 # 17.1.0

firebase-ai/api.txt

@@ -219,7 +219,8 @@ package com.google.firebase.ai.type {
   }
 
   public final class CodeExecutionResultPart implements com.google.firebase.ai.type.Part {
-    ctor public CodeExecutionResultPart(String outcome, String output);
+    ctor @Deprecated public CodeExecutionResultPart(String outcome, String output);
+    method public boolean executionSucceeded();
     method public String getOutcome();
     method public String getOutput();
     method public boolean isThought();
@@ -296,7 +297,7 @@ package com.google.firebase.ai.type {
   }
 
   public final class ExecutableCodePart implements com.google.firebase.ai.type.Part {
-    ctor public ExecutableCodePart(String language, String code);
+    ctor @Deprecated public ExecutableCodePart(String language, String code);
     method public String getCode();
     method public String getLanguage();
     method public boolean isThought();
@@ -1199,12 +1200,14 @@ package com.google.firebase.ai.type {
   }
 
   public final class Tool {
+    method public static com.google.firebase.ai.type.Tool codeExecution();
     method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
     method public static com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
     field public static final com.google.firebase.ai.type.Tool.Companion Companion;
   }
 
   public static final class Tool.Companion {
+    method public com.google.firebase.ai.type.Tool codeExecution();
     method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
     method public com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
   }

firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt

@@ -177,6 +177,7 @@ internal constructor(
       GenerateImageRequest.ImagenParameters(
         sampleCount = generationConfig?.numberOfImages ?: 1,
         includeRaiReason = true,
+        includeSafetyAttributes = true,
         addWatermark = generationConfig?.addWatermark,
         personGeneration = safetySettings?.personFilterLevel?.internalVal,
         negativePrompt = generationConfig?.negativePrompt,
@@ -206,6 +207,7 @@ internal constructor(
       GenerateImageRequest.ImagenParameters(
         sampleCount = generationConfig?.numberOfImages ?: 1,
         includeRaiReason = true,
+        includeSafetyAttributes = true,
         addWatermark = generationConfig?.addWatermark,
         personGeneration = safetySettings?.personFilterLevel?.internalVal,
         negativePrompt = generationConfig?.negativePrompt,

firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt

@@ -93,6 +93,7 @@ internal data class GenerateImageRequest(
   internal data class ImagenParameters(
     val sampleCount: Int,
     val includeRaiReason: Boolean,
+    val includeSafetyAttributes: Boolean,
     val storageUri: String?,
     val negativePrompt: String?,
     val aspectRatio: String?,

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt

@@ -42,7 +42,7 @@ internal constructor(public val images: List<T>, public val filteredReason: Stri
     internal fun toPublicInline() =
       ImagenGenerationResponse(
         images = predictions.filter { it.mimeType != null }.map { it.toPublicInline() },
-        null,
+        predictions.firstNotNullOfOrNull { it.raiFilteredReason },
       )
   }
 
@@ -52,10 +52,24 @@ internal constructor(public val images: List<T>, public val filteredReason: Stri
     val gcsUri: String? = null,
     val mimeType: String? = null,
     val raiFilteredReason: String? = null,
+    val safetyAttributes: ImagenSafetyAttributes? = null,
   ) {
     internal fun toPublicInline() =
       ImagenInlineImage(Base64.decode(bytesBase64Encoded!!, Base64.NO_WRAP), mimeType!!)
 
     internal fun toPublicGCS() = ImagenGCSImage(gcsUri!!, mimeType!!)
   }
+
+  @Serializable
+  internal data class ImagenSafetyAttributes(
+    val categories: List<String>? = null,
+    val scores: List<Double>? = null
+  ) {
+    internal fun toPublic(): Map<String, Double> {
+      if (categories == null || scores == null) {
+        return emptyMap()
+      }
+      return categories.zip(scores).toMap()
+    }
+  }
 }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt

@@ -31,7 +31,10 @@ import kotlinx.serialization.Serializable
  */
 @PublicPreviewAPI
 public class ImagenInlineImage
-internal constructor(public val data: ByteArray, public val mimeType: String) {
+internal constructor(
+  public val data: ByteArray,
+  public val mimeType: String,
+) {
 
   /**
    * Returns the image as an Android OS native [Bitmap] so that it can be saved or sent to the UI.

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt

@@ -54,6 +54,12 @@ internal constructor(
   ) : InternalPart
 }
 
+/**
+ * Represents the code execution result from the model.
+ * @property outcome The result of the execution.
+ * @property output The stdout from the code execution, or an error message if it failed.
+ * @property isThought Indicates whether the response is a thought.
+ */
 public class CodeExecutionResultPart
 internal constructor(
   public val outcome: String,
@@ -62,23 +68,29 @@ internal constructor(
   internal val thoughtSignature: String?
 ) : Part {
 
+  @Deprecated("Part of the model response. Do not instantiate directly.")
   public constructor(outcome: String, output: String) : this(outcome, output, false, null)
 
+  /** Indicates if the code execution was successful */
+  public fun executionSucceeded(): Boolean = (outcome.lowercase() == "outcome_ok")
+
   @Serializable
   internal data class Internal(
     @SerialName("codeExecutionResult") val codeExecutionResult: CodeExecutionResult,
     val thought: Boolean? = null,
     val thoughtSignature: String? = null
   ) : InternalPart {
 
-    @Serializable
-    internal data class CodeExecutionResult(
-      @SerialName("outcome") val outcome: String,
-      val output: String
-    )
+    @Serializable internal data class CodeExecutionResult(val outcome: String, val output: String)
   }
 }
 
+/**
+ * Represents the code that was executed by the model.
+ * @property language The programming language of the code.
+ * @property code The source code to be executed.
+ * @property isThought Indicates whether the response is a thought.
+ */
 public class ExecutableCodePart
 internal constructor(
   public val language: String,
@@ -87,6 +99,7 @@ internal constructor(
   internal val thoughtSignature: String?
 ) : Part {
 
+  @Deprecated("Part of the model response. Do not instantiate directly.")
   public constructor(language: String, code: String) : this(language, code, false, null)
 
   @Serializable

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt

@@ -26,12 +26,14 @@ import kotlinx.serialization.json.JsonObject
 public class Tool
 internal constructor(
   internal val functionDeclarations: List<FunctionDeclaration>?,
-  internal val googleSearch: GoogleSearch?
+  internal val googleSearch: GoogleSearch?,
+  internal val codeExecution: JsonObject?,
 ) {
   internal fun toInternal() =
     Internal(
       functionDeclarations?.map { it.toInternal() } ?: emptyList(),
-      googleSearch = this.googleSearch?.toInternal()
+      googleSearch = this.googleSearch?.toInternal(),
+      codeExecution = this.codeExecution
     )
   @Serializable
   internal data class Internal(
@@ -42,14 +44,22 @@ internal constructor(
   )
   public companion object {
 
+    private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap())) }
+
     /**
      * Creates a [Tool] instance that provides the model with access to the [functionDeclarations].
      *
      * @param functionDeclarations The list of functions that this tool allows the model access to.
      */
     @JvmStatic
     public fun functionDeclarations(functionDeclarations: List<FunctionDeclaration>): Tool {
-      return Tool(functionDeclarations, null)
+      return Tool(functionDeclarations, null, null)
+    }
+
+    /** Creates a [Tool] instance that allows the model to use Code Execution. */
+    @JvmStatic
+    public fun codeExecution(): Tool {
+      return codeExecutionInstance
     }
 
     /**
@@ -70,7 +80,7 @@ internal constructor(
      */
     @JvmStatic
     public fun googleSearch(googleSearch: GoogleSearch = GoogleSearch()): Tool {
-      return Tool(null, googleSearch)
+      return Tool(null, googleSearch, null)
     }
   }
 }

firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt

@@ -609,6 +609,15 @@ internal class VertexAIUnarySnapshotTests {
       }
     }
 
+  @Test
+  fun `generateImages should contain safety data`() =
+    goldenVertexUnaryFile("unary-success-generate-images-safety_info.json") {
+      withTimeout(testTimeout) {
+        val response = imagenModel.generateImages("prompt")
+        // There is no public API, but if it parses then success
+      }
+    }
+
   @Test
   fun `google search grounding metadata is parsed correctly`() =
     goldenVertexUnaryFile("unary-success-google-search-grounding.json") {

firebase-ai/src/test/java/com/google/firebase/ai/type/ToolTest.kt

@@ -27,6 +27,7 @@ internal class ToolTest {
 
     tool.googleSearch.shouldNotBeNull()
     tool.functionDeclarations.shouldBeNull()
+    tool.codeExecution.shouldBeNull()
   }
 
   @Test
@@ -36,5 +37,14 @@ internal class ToolTest {
 
     tool.functionDeclarations?.first() shouldBe functionDeclaration
     tool.googleSearch.shouldBeNull()
+    tool.codeExecution.shouldBeNull()
+  }
+
+  @Test
+  fun `codeExecution() creates a tool with code execution`() {
+    val tool = Tool.codeExecution()
+    tool.codeExecution.shouldNotBeNull()
+    tool.functionDeclarations.shouldBeNull()
+    tool.googleSearch.shouldBeNull()
   }
 }