[FirebaseAI] Add support for URL context

Author: dlarocque

firebase-ai/CHANGELOG.md

@@ -1,6 +1,7 @@
 # Unreleased
 
 - [changed] **Breaking Change**: Removed the `candidateCount` option from `LiveGenerationConfig`
+- [changed] Added support for URL context.
 
 # 17.3.0
 

firebase-ai/api.txt

@@ -191,11 +191,13 @@ package com.google.firebase.ai.type {
     method public com.google.firebase.ai.type.FinishReason? getFinishReason();
     method public com.google.firebase.ai.type.GroundingMetadata? getGroundingMetadata();
     method public java.util.List<com.google.firebase.ai.type.SafetyRating> getSafetyRatings();
+    method public com.google.firebase.ai.type.UrlContextMetadata? getUrlContextMetadata();
     property public final com.google.firebase.ai.type.CitationMetadata? citationMetadata;
     property public final com.google.firebase.ai.type.Content content;
     property public final com.google.firebase.ai.type.FinishReason? finishReason;
     property public final com.google.firebase.ai.type.GroundingMetadata? groundingMetadata;
     property public final java.util.List<com.google.firebase.ai.type.SafetyRating> safetyRatings;
+    property public final com.google.firebase.ai.type.UrlContextMetadata? urlContextMetadata;
   }
 
   public final class Citation {
@@ -1201,13 +1203,15 @@ package com.google.firebase.ai.type {
     method public static com.google.firebase.ai.type.Tool codeExecution();
     method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
     method public static com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
+    method public static com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext());
     field public static final com.google.firebase.ai.type.Tool.Companion Companion;
   }
 
   public static final class Tool.Companion {
     method public com.google.firebase.ai.type.Tool codeExecution();
     method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
     method public com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
+    method public com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext());
   }
 
   public final class ToolConfig {
@@ -1220,19 +1224,57 @@ package com.google.firebase.ai.type {
   public final class UnsupportedUserLocationException extends com.google.firebase.ai.type.FirebaseAIException {
   }
 
+  public final class UrlContext {
+    ctor public UrlContext();
+  }
+
+  public final class UrlContextMetadata {
+    ctor public UrlContextMetadata(java.util.List<com.google.firebase.ai.type.UrlMetadata> urlMetadata);
+    method public java.util.List<com.google.firebase.ai.type.UrlMetadata> getUrlMetadata();
+    property public final java.util.List<com.google.firebase.ai.type.UrlMetadata> urlMetadata;
+  }
+
+  public final class UrlMetadata {
+    ctor public UrlMetadata(String? retrievedUrl, com.google.firebase.ai.type.UrlRetrievalStatus urlRetrievalStatus);
+    method public String? getRetrievedUrl();
+    method public com.google.firebase.ai.type.UrlRetrievalStatus getUrlRetrievalStatus();
+    property public final String? retrievedUrl;
+    property public final com.google.firebase.ai.type.UrlRetrievalStatus urlRetrievalStatus;
+  }
+
+  public final class UrlRetrievalStatus {
+    method public String getName();
+    method public int getOrdinal();
+    property public final String name;
+    property public final int ordinal;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus.Companion Companion;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus ERROR;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus PAYWALL;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus SUCCESS;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus UNSAFE;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus UNSPECIFIED;
+  }
+
+  public static final class UrlRetrievalStatus.Companion {
+  }
+
   public final class UsageMetadata {
-    ctor public UsageMetadata(int promptTokenCount, Integer? candidatesTokenCount, int totalTokenCount, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails, int thoughtsTokenCount);
+    ctor public UsageMetadata(int promptTokenCount, Integer? candidatesTokenCount, int totalTokenCount, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> toolUsePromptTokensDetails, int thoughtsTokenCount, int toolUsePromptTokenCount);
     method public Integer? getCandidatesTokenCount();
     method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getCandidatesTokensDetails();
     method public int getPromptTokenCount();
     method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getPromptTokensDetails();
     method public int getThoughtsTokenCount();
+    method public int getToolUsePromptTokenCount();
+    method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getToolUsePromptTokensDetails();
     method public int getTotalTokenCount();
     property public final Integer? candidatesTokenCount;
     property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails;
     property public final int promptTokenCount;
     property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails;
     property public final int thoughtsTokenCount;
+    property public final int toolUsePromptTokenCount;
+    property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> toolUsePromptTokensDetails;
     property public final int totalTokenCount;
   }
 

firebase-ai/gradle.properties

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.3.1
+version=17.4.0
 latestReleasedVersion=17.3.0

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt

@@ -33,15 +33,18 @@ import kotlinx.serialization.json.JsonNames
  * @property safetyRatings A list of [SafetyRating]s describing the generated content.
  * @property citationMetadata Metadata about the sources used to generate this content.
  * @property finishReason The reason the model stopped generating content, if it exist.
- * @property groundingMetadata Metadata returned to the client when grounding is enabled.
+ * @property groundingMetadata Metadata returned to the client when the model grounds its response.
+ * @property urlContextMetadata Metadata returned to the client when the [UrlContext] tool is
+ * enabled.
  */
 public class Candidate
 internal constructor(
   public val content: Content,
   public val safetyRatings: List<SafetyRating>,
   public val citationMetadata: CitationMetadata?,
   public val finishReason: FinishReason?,
-  public val groundingMetadata: GroundingMetadata?
+  public val groundingMetadata: GroundingMetadata?,
+  public val urlContextMetadata: UrlContextMetadata?
 ) {
 
   @Serializable
@@ -50,20 +53,23 @@ internal constructor(
     val finishReason: FinishReason.Internal? = null,
     val safetyRatings: List<SafetyRating.Internal>? = null,
     val citationMetadata: CitationMetadata.Internal? = null,
-    val groundingMetadata: GroundingMetadata.Internal? = null
+    val groundingMetadata: GroundingMetadata.Internal? = null,
+    val urlContextMetadata: UrlContextMetadata.Internal? = null
   ) {
     internal fun toPublic(): Candidate {
       val safetyRatings = safetyRatings?.mapNotNull { it.toPublic() }.orEmpty()
       val citations = citationMetadata?.toPublic()
       val finishReason = finishReason?.toPublic()
       val groundingMetadata = groundingMetadata?.toPublic()
+      val urlContextMetadata = urlContextMetadata?.toPublic()
 
       return Candidate(
         this.content?.toPublic() ?: content("model") {},
         safetyRatings,
         citations,
         finishReason,
-        groundingMetadata
+        groundingMetadata,
+        urlContextMetadata
       )
     }
   }
@@ -372,8 +378,7 @@ public class SearchEntryPoint(
 }
 
 /**
- * Represents a chunk of retrieved data that supports a claim in the model's response. This is part
- * of the grounding information provided when grounding is enabled.
+ * Represents a chunk of retrieved data that supports a claim in the model's response.
  *
  * @property web Contains details if the grounding chunk is from a web source.
  */
@@ -492,3 +497,81 @@ public class Segment(
       )
   }
 }
+
+/**
+ * Metadata related to the [UrlContext] tool.
+ *
+ * @property urlMetadata List of [UrlMetadata] used to provide context to the Gemini model.
+ */
+public class UrlContextMetadata(public val urlMetadata: List<UrlMetadata>) {
+  @Serializable
+  internal data class Internal(val urlMetadata: List<UrlMetadata.Internal>) {
+    internal fun toPublic() = UrlContextMetadata(urlMetadata.map { it.toPublic() })
+  }
+}
+
+/**
+ * Metadata for a single URL retrieved by the [UrlContext] tool.
+ *
+ * @property retrievedUrl The retrieved URL.
+ * @property urlRetrievalStatus The status of the URL retrieval.
+ */
+public class UrlMetadata(
+  public val retrievedUrl: String?,
+  public val urlRetrievalStatus: UrlRetrievalStatus
+) {
+  @Serializable
+  internal data class Internal(
+    val retrievedUrl: String?,
+    val urlRetrievalStatus: UrlRetrievalStatus.Internal
+  ) {
+    internal fun toPublic() = UrlMetadata(retrievedUrl, urlRetrievalStatus.toPublic())
+  }
+}
+
+/**
+ * The status of a URL retrieval.
+ *
+ * @property name The name of the retrieval status.
+ * @property ordinal The ordinal value of the retrieval status.
+ */
+public class UrlRetrievalStatus
+private constructor(public val name: String, public val ordinal: Int) {
+
+  @Serializable(Internal.Serializer::class)
+  internal enum class Internal {
+    @SerialName("URL_RETRIEVAL_STATUS_UNSPECIFIED") UNSPECIFIED,
+    @SerialName("URL_RETRIEVAL_STATUS_SUCCESS") SUCCESS,
+    @SerialName("URL_RETRIEVAL_STATUS_ERROR") ERROR,
+    @SerialName("URL_RETRIEVAL_STATUS_PAYWALL") PAYWALL,
+    @SerialName("URL_RETRIEVAL_STATUS_UNSAFE") UNSAFE;
+
+    internal object Serializer : KSerializer<Internal> by FirstOrdinalSerializer(Internal::class)
+
+    internal fun toPublic() =
+      when (this) {
+        SUCCESS -> UrlRetrievalStatus.SUCCESS
+        ERROR -> UrlRetrievalStatus.ERROR
+        PAYWALL -> UrlRetrievalStatus.PAYWALL
+        UNSAFE -> UrlRetrievalStatus.UNSAFE
+        else -> UrlRetrievalStatus.UNSPECIFIED
+      }
+  }
+
+  public companion object {
+    /** Unspecified retrieval status. */
+    @JvmField public val UNSPECIFIED: UrlRetrievalStatus = UrlRetrievalStatus("UNSPECIFIED", 0)
+
+    /** The URL retrieval was successful. */
+    @JvmField public val SUCCESS: UrlRetrievalStatus = UrlRetrievalStatus("SUCCESS", 1)
+
+    /** The URL retrieval failed. */
+    @JvmField public val ERROR: UrlRetrievalStatus = UrlRetrievalStatus("ERROR", 2)
+
+    /** The URL retrieval failed because the content is behind a paywall. */
+    @JvmField public val PAYWALL: UrlRetrievalStatus = UrlRetrievalStatus("PAYWALL", 3)
+
+    /** The URL retrieval failed because the content is unsafe. */
+    @JvmField public val UNSAFE: UrlRetrievalStatus = UrlRetrievalStatus("UNSAFE", 4)
+  }
+}

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt

@@ -28,23 +28,26 @@ internal constructor(
   internal val functionDeclarations: List<FunctionDeclaration>?,
   internal val googleSearch: GoogleSearch?,
   internal val codeExecution: JsonObject?,
+  internal val urlContext: UrlContext?,
 ) {
   internal fun toInternal() =
     Internal(
       functionDeclarations?.map { it.toInternal() } ?: emptyList(),
       googleSearch = this.googleSearch?.toInternal(),
-      codeExecution = this.codeExecution
+      codeExecution = this.codeExecution,
+      urlContext = this.urlContext?.toInternal()
     )
   @Serializable
   internal data class Internal(
     val functionDeclarations: List<FunctionDeclaration.Internal>? = null,
     val googleSearch: GoogleSearch.Internal? = null,
     // This is a json object because it is not possible to make a data class with no parameters.
     val codeExecution: JsonObject? = null,
+    val urlContext: UrlContext.Internal? = null,
   )
   public companion object {
 
-    private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap())) }
+    private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap()), null) }
 
     /**
      * Creates a [Tool] instance that provides the model with access to the [functionDeclarations].
@@ -53,7 +56,7 @@ internal constructor(
      */
     @JvmStatic
     public fun functionDeclarations(functionDeclarations: List<FunctionDeclaration>): Tool {
-      return Tool(functionDeclarations, null, null)
+      return Tool(functionDeclarations, null, null, null)
     }
 
     /** Creates a [Tool] instance that allows the model to use Code Execution. */
@@ -62,6 +65,19 @@ internal constructor(
       return codeExecutionInstance
     }
 
+    /**
+     * Creates a [Tool] instance that allows you to provide additional context to the models in the
+     * form of public web URLs. By including URLs in your request, the Gemini model will access the
+     * content from those pages to inform and enhance its response.
+     *
+     * @param urlContext Specifies the URL context configuration.
+     * @return A [Tool] configured for URL context
+     */
+    @JvmStatic
+    public fun urlContext(urlContext: UrlContext = UrlContext()): Tool {
+      return Tool(null, null, null, urlContext)
+    }
+
     /**
      * Creates a [Tool] instance that allows the model to use Grounding with Google Search.
      *
@@ -80,7 +96,7 @@ internal constructor(
      */
     @JvmStatic
     public fun googleSearch(googleSearch: GoogleSearch = GoogleSearch()): Tool {
-      return Tool(null, googleSearch, null)
+      return Tool(null, googleSearch, null, null)
     }
   }
 }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UrlContext.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.firebase.ai.type
+
+import kotlinx.serialization.Serializable
+
+/** Specifies the URL context configuration. */
+public class UrlContext {
+  @Serializable internal class Internal()
+
+  internal fun toInternal() = Internal()
+}

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UsageMetadata.kt

@@ -28,15 +28,20 @@ import kotlinx.serialization.Serializable
  * prompt.
  * @param candidatesTokensDetails The breakdown, by modality, of how many tokens are consumed by the
  * candidates.
+ * @param toolUsePromptTokensDetails The breakdown, by modality, of how many tokens are consumed by
+ * tools.
  * @param thoughtsTokenCount The number of tokens used by the model's internal "thinking" process.
+ * @param toolUsePromptTokenCount The number of tokens used by tools.
  */
 public class UsageMetadata(
   public val promptTokenCount: Int,
   public val candidatesTokenCount: Int?,
   public val totalTokenCount: Int,
   public val promptTokensDetails: List<ModalityTokenCount>,
   public val candidatesTokensDetails: List<ModalityTokenCount>,
+  public val toolUsePromptTokensDetails: List<ModalityTokenCount>,
   public val thoughtsTokenCount: Int,
+  public val toolUsePromptTokenCount: Int,
 ) {
 
   @Serializable
@@ -46,7 +51,9 @@ public class UsageMetadata(
     val totalTokenCount: Int? = null,
     val promptTokensDetails: List<ModalityTokenCount.Internal>? = null,
     val candidatesTokensDetails: List<ModalityTokenCount.Internal>? = null,
+    val toolUsePromptTokensDetails: List<ModalityTokenCount.Internal>? = null,
     val thoughtsTokenCount: Int? = null,
+    val toolUsePromptTokenCount: Int? = null,
   ) {
 
     internal fun toPublic(): UsageMetadata =
@@ -56,7 +63,10 @@ public class UsageMetadata(
         totalTokenCount ?: 0,
         promptTokensDetails = promptTokensDetails?.map { it.toPublic() } ?: emptyList(),
         candidatesTokensDetails = candidatesTokensDetails?.map { it.toPublic() } ?: emptyList(),
-        thoughtsTokenCount ?: 0
+        toolUsePromptTokensDetails = toolUsePromptTokensDetails?.map { it.toPublic() }
+            ?: emptyList(),
+        thoughtsTokenCount ?: 0,
+        toolUsePromptTokenCount ?: 0,
       )
   }
 }