[FirebaseAI] Add support for URL context

firebase-ai/CHANGELOG.md

@@ -1,7 +1,8 @@
 # Unreleased
 
 - [changed] **Breaking Change**: Removed the `candidateCount` option from `LiveGenerationConfig`
-  (#7382)
+- [changed] Added support for the URL context tool, which allows the model to access content from
+  provided public web URLs to inform and enhance its responses. (#7382)
 - [changed] Added better error messages to `ServiceConnectionHandshakeFailedException` (#7412)
 - [changed] Marked the public constructor for `UsageMetadata` as deprecated (#7420)
 

firebase-ai/api.txt

@@ -191,11 +191,13 @@ package com.google.firebase.ai.type {
     method public com.google.firebase.ai.type.FinishReason? getFinishReason();
     method public com.google.firebase.ai.type.GroundingMetadata? getGroundingMetadata();
     method public java.util.List<com.google.firebase.ai.type.SafetyRating> getSafetyRatings();
+    method public com.google.firebase.ai.type.UrlContextMetadata? getUrlContextMetadata();
     property public final com.google.firebase.ai.type.CitationMetadata? citationMetadata;
     property public final com.google.firebase.ai.type.Content content;
     property public final com.google.firebase.ai.type.FinishReason? finishReason;
     property public final com.google.firebase.ai.type.GroundingMetadata? groundingMetadata;
     property public final java.util.List<com.google.firebase.ai.type.SafetyRating> safetyRatings;
+    property public final com.google.firebase.ai.type.UrlContextMetadata? urlContextMetadata;
   }
 
   public final class Citation {
@@ -942,7 +944,7 @@ package com.google.firebase.ai.type {
     property public final java.util.List<com.google.firebase.ai.type.SafetyRating> safetyRatings;
   }
 
-  @kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) public @interface PublicPreviewAPI {
+  @kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) @kotlin.annotation.Target(allowedTargets={kotlin.annotation.AnnotationTarget.CLASS, kotlin.annotation.AnnotationTarget.FUNCTION, kotlin.annotation.AnnotationTarget.PROPERTY}) public @interface PublicPreviewAPI {
   }
 
   public final class QuotaExceededException extends com.google.firebase.ai.type.FirebaseAIException {
@@ -1201,13 +1203,15 @@ package com.google.firebase.ai.type {
     method public static com.google.firebase.ai.type.Tool codeExecution();
     method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
     method public static com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
+    method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext());
     field public static final com.google.firebase.ai.type.Tool.Companion Companion;
   }
 
   public static final class Tool.Companion {
     method public com.google.firebase.ai.type.Tool codeExecution();
     method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
     method public com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
+    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext());
   }
 
   public final class ToolConfig {
@@ -1220,19 +1224,55 @@ package com.google.firebase.ai.type {
   public final class UnsupportedUserLocationException extends com.google.firebase.ai.type.FirebaseAIException {
   }
 
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class UrlContext {
+    ctor public UrlContext();
+  }
+
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class UrlContextMetadata {
+    method public java.util.List<com.google.firebase.ai.type.UrlMetadata> getUrlMetadata();
+    property public final java.util.List<com.google.firebase.ai.type.UrlMetadata> urlMetadata;
+  }
+
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class UrlMetadata {
+    method public String? getRetrievedUrl();
+    method public com.google.firebase.ai.type.UrlRetrievalStatus getUrlRetrievalStatus();
+    property public final String? retrievedUrl;
+    property public final com.google.firebase.ai.type.UrlRetrievalStatus urlRetrievalStatus;
+  }
+
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class UrlRetrievalStatus {
+    method public String getName();
+    method public int getOrdinal();
+    property public final String name;
+    property public final int ordinal;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus.Companion Companion;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus ERROR;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus PAYWALL;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus SUCCESS;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus UNSAFE;
+    field public static final com.google.firebase.ai.type.UrlRetrievalStatus UNSPECIFIED;
+  }
+
+  public static final class UrlRetrievalStatus.Companion {
+  }
+
   public final class UsageMetadata {
-    ctor public UsageMetadata(int promptTokenCount, Integer? candidatesTokenCount, int totalTokenCount, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails, int thoughtsTokenCount);
+    ctor @Deprecated public UsageMetadata(int promptTokenCount, Integer? candidatesTokenCount, int totalTokenCount, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails, int thoughtsTokenCount);
     method public Integer? getCandidatesTokenCount();
     method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getCandidatesTokensDetails();
     method public int getPromptTokenCount();
     method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getPromptTokensDetails();
     method public int getThoughtsTokenCount();
+    method public int getToolUsePromptTokenCount();
+    method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getToolUsePromptTokensDetails();
     method public int getTotalTokenCount();
     property public final Integer? candidatesTokenCount;
     property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails;
     property public final int promptTokenCount;
     property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails;
     property public final int thoughtsTokenCount;
+    property public final int toolUsePromptTokenCount;
+    property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> toolUsePromptTokensDetails;
     property public final int totalTokenCount;
   }
 

firebase-ai/gradle.properties

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.3.1
+version=17.4.0
 latestReleasedVersion=17.3.0

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt

@@ -33,37 +33,47 @@ import kotlinx.serialization.json.JsonNames
  * @property safetyRatings A list of [SafetyRating]s describing the generated content.
  * @property citationMetadata Metadata about the sources used to generate this content.
  * @property finishReason The reason the model stopped generating content, if it exist.
- * @property groundingMetadata Metadata returned to the client when grounding is enabled.
+ * @property groundingMetadata Metadata returned to the client when the model grounds its response.
+ * @property urlContextMetadata Metadata returned to the client when the [UrlContext] tool is
+ * enabled.
  */
 public class Candidate
+@OptIn(PublicPreviewAPI::class)
 internal constructor(
   public val content: Content,
   public val safetyRatings: List<SafetyRating>,
   public val citationMetadata: CitationMetadata?,
   public val finishReason: FinishReason?,
-  public val groundingMetadata: GroundingMetadata?
+  public val groundingMetadata: GroundingMetadata?,
+  @property:PublicPreviewAPI public val urlContextMetadata: UrlContextMetadata?
 ) {
 
+  @OptIn(PublicPreviewAPI::class)
   @Serializable
   internal data class Internal(
     val content: Content.Internal? = null,
     val finishReason: FinishReason.Internal? = null,
     val safetyRatings: List<SafetyRating.Internal>? = null,
     val citationMetadata: CitationMetadata.Internal? = null,
-    val groundingMetadata: GroundingMetadata.Internal? = null
+    val groundingMetadata: GroundingMetadata.Internal? = null,
+    val urlContextMetadata: UrlContextMetadata.Internal? = null
   ) {
+
+    @OptIn(PublicPreviewAPI::class)
     internal fun toPublic(): Candidate {
       val safetyRatings = safetyRatings?.mapNotNull { it.toPublic() }.orEmpty()
       val citations = citationMetadata?.toPublic()
       val finishReason = finishReason?.toPublic()
       val groundingMetadata = groundingMetadata?.toPublic()
+      val urlContextMetadata = urlContextMetadata?.toPublic()
 
       return Candidate(
         this.content?.toPublic() ?: content("model") {},
         safetyRatings,
         citations,
         finishReason,
-        groundingMetadata
+        groundingMetadata,
+        urlContextMetadata
       )
     }
   }
@@ -372,8 +382,7 @@ public class SearchEntryPoint(
 }
 
 /**
- * Represents a chunk of retrieved data that supports a claim in the model's response. This is part
- * of the grounding information provided when grounding is enabled.
+ * Represents a chunk of retrieved data that supports a claim in the model's response.
  *
  * @property web Contains details if the grounding chunk is from a web source.
  */
@@ -492,3 +501,87 @@ public class Segment(
       )
   }
 }
+
+/**
+ * Metadata related to the [UrlContext] tool.
+ *
+ * @property urlMetadata List of [UrlMetadata] used to provide context to the Gemini model.
+ */
+@PublicPreviewAPI
+public class UrlContextMetadata internal constructor(public val urlMetadata: List<UrlMetadata>) {
+
+  @Serializable
+  @PublicPreviewAPI
+  internal data class Internal(val urlMetadata: List<UrlMetadata.Internal>?) {
+    internal fun toPublic() = UrlContextMetadata(urlMetadata?.map { it.toPublic() } ?: emptyList())
+  }
+}
+
+/**
+ * Metadata for a single URL retrieved by the [UrlContext] tool.
+ *
+ * @property retrievedUrl The retrieved URL.
+ * @property urlRetrievalStatus The status of the URL retrieval.
+ */
+@PublicPreviewAPI
+public class UrlMetadata
+internal constructor(
+  public val retrievedUrl: String?,
+  public val urlRetrievalStatus: UrlRetrievalStatus
+) {
+  @Serializable
+  internal data class Internal(
+    val retrievedUrl: String?,
+    val urlRetrievalStatus: UrlRetrievalStatus.Internal
+  ) {
+    internal fun toPublic() = UrlMetadata(retrievedUrl, urlRetrievalStatus.toPublic())
+  }
+}
+
+/**
+ * The status of a URL retrieval.
+ *
+ * @property name The name of the retrieval status.
+ * @property ordinal The ordinal value of the retrieval status.
+ */
+@PublicPreviewAPI
+public class UrlRetrievalStatus
+private constructor(public val name: String, public val ordinal: Int) {
+
+  @Serializable(Internal.Serializer::class)
+  internal enum class Internal {
+    @SerialName("URL_RETRIEVAL_STATUS_UNSPECIFIED") UNSPECIFIED,
+    @SerialName("URL_RETRIEVAL_STATUS_SUCCESS") SUCCESS,
+    @SerialName("URL_RETRIEVAL_STATUS_ERROR") ERROR,
+    @SerialName("URL_RETRIEVAL_STATUS_PAYWALL") PAYWALL,
+    @SerialName("URL_RETRIEVAL_STATUS_UNSAFE") UNSAFE;
+
+    internal object Serializer : KSerializer<Internal> by FirstOrdinalSerializer(Internal::class)
+
+    internal fun toPublic() =
+      when (this) {
+        SUCCESS -> UrlRetrievalStatus.SUCCESS
+        ERROR -> UrlRetrievalStatus.ERROR
+        PAYWALL -> UrlRetrievalStatus.PAYWALL
+        UNSAFE -> UrlRetrievalStatus.UNSAFE
+        else -> UrlRetrievalStatus.UNSPECIFIED
+      }
+  }
+
+  public companion object {
+    /** Unspecified retrieval status. */
+    @JvmField public val UNSPECIFIED: UrlRetrievalStatus = UrlRetrievalStatus("UNSPECIFIED", 0)
+
+    /** The URL retrieval was successful. */
+    @JvmField public val SUCCESS: UrlRetrievalStatus = UrlRetrievalStatus("SUCCESS", 1)
+
+    /** The URL retrieval failed. */
+    @JvmField public val ERROR: UrlRetrievalStatus = UrlRetrievalStatus("ERROR", 2)
+
+    /** The URL retrieval failed because the content is behind a paywall. */
+    @JvmField public val PAYWALL: UrlRetrievalStatus = UrlRetrievalStatus("PAYWALL", 3)
+
+    /** The URL retrieval failed because the content is unsafe. */
+    @JvmField public val UNSAFE: UrlRetrievalStatus = UrlRetrievalStatus("UNSAFE", 4)
+  }
+}

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/PublicPreviewAPI.kt

@@ -23,4 +23,5 @@ package com.google.firebase.ai.type
     "This API is part of an experimental public preview and may change in " +
       "backwards-incompatible ways without notice.",
 )
+@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY)
 public annotation class PublicPreviewAPI()

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt

@@ -24,27 +24,36 @@ import kotlinx.serialization.json.JsonObject
  * can be used to gather information or complete tasks.
  */
 public class Tool
+@OptIn(PublicPreviewAPI::class)
 internal constructor(
   internal val functionDeclarations: List<FunctionDeclaration>?,
   internal val googleSearch: GoogleSearch?,
   internal val codeExecution: JsonObject?,
+  @property:PublicPreviewAPI internal val urlContext: UrlContext?,
 ) {
+
+  @OptIn(PublicPreviewAPI::class)
   internal fun toInternal() =
     Internal(
       functionDeclarations?.map { it.toInternal() } ?: emptyList(),
       googleSearch = this.googleSearch?.toInternal(),
-      codeExecution = this.codeExecution
+      codeExecution = this.codeExecution,
+      urlContext = this.urlContext?.toInternal()
     )
+
+  @OptIn(PublicPreviewAPI::class)
   @Serializable
   internal data class Internal(
     val functionDeclarations: List<FunctionDeclaration.Internal>? = null,
     val googleSearch: GoogleSearch.Internal? = null,
     // This is a json object because it is not possible to make a data class with no parameters.
     val codeExecution: JsonObject? = null,
+    val urlContext: UrlContext.Internal? = null,
   )
   public companion object {
 
-    private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap())) }
+    @OptIn(PublicPreviewAPI::class)
+    private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap()), null) }
 
     /**
      * Creates a [Tool] instance that provides the model with access to the [functionDeclarations].
@@ -53,7 +62,7 @@ internal constructor(
      */
     @JvmStatic
     public fun functionDeclarations(functionDeclarations: List<FunctionDeclaration>): Tool {
-      return Tool(functionDeclarations, null, null)
+      @OptIn(PublicPreviewAPI::class) return Tool(functionDeclarations, null, null, null)
     }
 
     /** Creates a [Tool] instance that allows the model to use code execution. */
@@ -62,6 +71,20 @@ internal constructor(
       return codeExecutionInstance
     }
 
+    /**
+     * Creates a [Tool] instance that allows you to provide additional context to the models in the
+     * form of public web URLs. By including URLs in your request, the Gemini model will access the
+     * content from those pages to inform and enhance its response.
+     *
+     * @param urlContext Specifies the URL context configuration.
+     * @return A [Tool] configured for URL context.
+     */
+    @PublicPreviewAPI
+    @JvmStatic
+    public fun urlContext(urlContext: UrlContext = UrlContext()): Tool {
+      return Tool(null, null, null, urlContext)
+    }
+
     /**
      * Creates a [Tool] instance that allows the model to use grounding with Google Search.
      *
@@ -80,7 +103,7 @@ internal constructor(
      */
     @JvmStatic
     public fun googleSearch(googleSearch: GoogleSearch = GoogleSearch()): Tool {
-      return Tool(null, googleSearch, null)
+      @OptIn(PublicPreviewAPI::class) return Tool(null, googleSearch, null, null)
     }
   }
 }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UrlContext.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.firebase.ai.type
+
+import kotlinx.serialization.Serializable
+
+/** Specifies the URL context configuration. */
+@PublicPreviewAPI
+public class UrlContext {
+  @Serializable internal class Internal()
+
+  internal fun toInternal() = Internal()
+}