Skip to content

Commit 1654b31

Browse files
dlarocquerlazo
dlarocque
and
rlazo
authored
[FirebaseAI] Add support for URL context (#7401)
API proposal: [go/alf-urlcontext-api](http://go/alf-urlcontext-api) (internal) Changes: - Added `UrlContext` and `Tool.urlContext()` static initializer - Added `UrlContextMetadata` to `Candidate`. - Added `toolUsePromptTokenCount` and `toolUsePromptTokensDetails`, which will be populated when using Google Search, Code Execution, or URL Context. - Adjusted `GroundingMetadata` documentation. It used to only be used by Google Search, now it can be used by either Google Search or URL context- reworded a couple things to reflect this. --------- Co-authored-by: Rodrigo Lazo <[email protected]>
1 parent 6974f49 commit 1654b31

File tree

15 files changed

+513
-22
lines changed

15 files changed

+513
-22
lines changed

‎firebase-ai/CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
# Unreleased
22

33
- [changed] **Breaking Change**: Removed the `candidateCount` option from `LiveGenerationConfig`
4-
(#7382)
4+
- [changed] Added support for the URL context tool, which allows the model to access content from
5+
provided public web URLs to inform and enhance its responses. (#7382)
56
- [changed] Added better error messages to `ServiceConnectionHandshakeFailedException` (#7412)
67
- [changed] Marked the public constructor for `UsageMetadata` as deprecated (#7420)
78

‎firebase-ai/api.txt‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,11 +191,13 @@ package com.google.firebase.ai.type {
191191
method public com.google.firebase.ai.type.FinishReason? getFinishReason();
192192
method public com.google.firebase.ai.type.GroundingMetadata? getGroundingMetadata();
193193
method public java.util.List<com.google.firebase.ai.type.SafetyRating> getSafetyRatings();
194+
method public com.google.firebase.ai.type.UrlContextMetadata? getUrlContextMetadata();
194195
property public final com.google.firebase.ai.type.CitationMetadata? citationMetadata;
195196
property public final com.google.firebase.ai.type.Content content;
196197
property public final com.google.firebase.ai.type.FinishReason? finishReason;
197198
property public final com.google.firebase.ai.type.GroundingMetadata? groundingMetadata;
198199
property public final java.util.List<com.google.firebase.ai.type.SafetyRating> safetyRatings;
200+
property public final com.google.firebase.ai.type.UrlContextMetadata? urlContextMetadata;
199201
}
200202

201203
public final class Citation {
@@ -942,7 +944,7 @@ package com.google.firebase.ai.type {
942944
property public final java.util.List<com.google.firebase.ai.type.SafetyRating> safetyRatings;
943945
}
944946

945-
@kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) public @interface PublicPreviewAPI {
947+
@kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) @kotlin.annotation.Target(allowedTargets={kotlin.annotation.AnnotationTarget.CLASS, kotlin.annotation.AnnotationTarget.FUNCTION, kotlin.annotation.AnnotationTarget.PROPERTY}) public @interface PublicPreviewAPI {
946948
}
947949

948950
public final class QuotaExceededException extends com.google.firebase.ai.type.FirebaseAIException {
@@ -1201,13 +1203,15 @@ package com.google.firebase.ai.type {
12011203
method public static com.google.firebase.ai.type.Tool codeExecution();
12021204
method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
12031205
method public static com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
1206+
method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext());
12041207
field public static final com.google.firebase.ai.type.Tool.Companion Companion;
12051208
}
12061209

12071210
public static final class Tool.Companion {
12081211
method public com.google.firebase.ai.type.Tool codeExecution();
12091212
method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
12101213
method public com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
1214+
method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext());
12111215
}
12121216

12131217
public final class ToolConfig {
@@ -1220,19 +1224,55 @@ package com.google.firebase.ai.type {
12201224
public final class UnsupportedUserLocationException extends com.google.firebase.ai.type.FirebaseAIException {
12211225
}
12221226

1227+
@com.google.firebase.ai.type.PublicPreviewAPI public final class UrlContext {
1228+
ctor public UrlContext();
1229+
}
1230+
1231+
@com.google.firebase.ai.type.PublicPreviewAPI public final class UrlContextMetadata {
1232+
method public java.util.List<com.google.firebase.ai.type.UrlMetadata> getUrlMetadata();
1233+
property public final java.util.List<com.google.firebase.ai.type.UrlMetadata> urlMetadata;
1234+
}
1235+
1236+
@com.google.firebase.ai.type.PublicPreviewAPI public final class UrlMetadata {
1237+
method public String? getRetrievedUrl();
1238+
method public com.google.firebase.ai.type.UrlRetrievalStatus getUrlRetrievalStatus();
1239+
property public final String? retrievedUrl;
1240+
property public final com.google.firebase.ai.type.UrlRetrievalStatus urlRetrievalStatus;
1241+
}
1242+
1243+
@com.google.firebase.ai.type.PublicPreviewAPI public final class UrlRetrievalStatus {
1244+
method public String getName();
1245+
method public int getOrdinal();
1246+
property public final String name;
1247+
property public final int ordinal;
1248+
field public static final com.google.firebase.ai.type.UrlRetrievalStatus.Companion Companion;
1249+
field public static final com.google.firebase.ai.type.UrlRetrievalStatus ERROR;
1250+
field public static final com.google.firebase.ai.type.UrlRetrievalStatus PAYWALL;
1251+
field public static final com.google.firebase.ai.type.UrlRetrievalStatus SUCCESS;
1252+
field public static final com.google.firebase.ai.type.UrlRetrievalStatus UNSAFE;
1253+
field public static final com.google.firebase.ai.type.UrlRetrievalStatus UNSPECIFIED;
1254+
}
1255+
1256+
public static final class UrlRetrievalStatus.Companion {
1257+
}
1258+
12231259
public final class UsageMetadata {
1224-
ctor public UsageMetadata(int promptTokenCount, Integer? candidatesTokenCount, int totalTokenCount, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails, int thoughtsTokenCount);
1260+
ctor @Deprecated public UsageMetadata(int promptTokenCount, Integer? candidatesTokenCount, int totalTokenCount, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails, java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails, int thoughtsTokenCount);
12251261
method public Integer? getCandidatesTokenCount();
12261262
method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getCandidatesTokensDetails();
12271263
method public int getPromptTokenCount();
12281264
method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getPromptTokensDetails();
12291265
method public int getThoughtsTokenCount();
1266+
method public int getToolUsePromptTokenCount();
1267+
method public java.util.List<com.google.firebase.ai.type.ModalityTokenCount> getToolUsePromptTokensDetails();
12301268
method public int getTotalTokenCount();
12311269
property public final Integer? candidatesTokenCount;
12321270
property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> candidatesTokensDetails;
12331271
property public final int promptTokenCount;
12341272
property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> promptTokensDetails;
12351273
property public final int thoughtsTokenCount;
1274+
property public final int toolUsePromptTokenCount;
1275+
property public final java.util.List<com.google.firebase.ai.type.ModalityTokenCount> toolUsePromptTokensDetails;
12361276
property public final int totalTokenCount;
12371277
}
12381278

‎firebase-ai/gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
version=17.3.1
15+
version=17.4.0
1616
latestReleasedVersion=17.3.0

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt‎

Lines changed: 99 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,37 +33,47 @@ import kotlinx.serialization.json.JsonNames
3333
* @property safetyRatings A list of [SafetyRating]s describing the generated content.
3434
* @property citationMetadata Metadata about the sources used to generate this content.
3535
* @property finishReason The reason the model stopped generating content, if it exist.
36-
* @property groundingMetadata Metadata returned to the client when grounding is enabled.
36+
* @property groundingMetadata Metadata returned to the client when the model grounds its response.
37+
* @property urlContextMetadata Metadata returned to the client when the [UrlContext] tool is
38+
* enabled.
3739
*/
3840
public class Candidate
41+
@OptIn(PublicPreviewAPI::class)
3942
internal constructor(
4043
public val content: Content,
4144
public val safetyRatings: List<SafetyRating>,
4245
public val citationMetadata: CitationMetadata?,
4346
public val finishReason: FinishReason?,
44-
public val groundingMetadata: GroundingMetadata?
47+
public val groundingMetadata: GroundingMetadata?,
48+
@property:PublicPreviewAPI public val urlContextMetadata: UrlContextMetadata?
4549
) {
4650

51+
@OptIn(PublicPreviewAPI::class)
4752
@Serializable
4853
internal data class Internal(
4954
val content: Content.Internal? = null,
5055
val finishReason: FinishReason.Internal? = null,
5156
val safetyRatings: List<SafetyRating.Internal>? = null,
5257
val citationMetadata: CitationMetadata.Internal? = null,
53-
val groundingMetadata: GroundingMetadata.Internal? = null
58+
val groundingMetadata: GroundingMetadata.Internal? = null,
59+
val urlContextMetadata: UrlContextMetadata.Internal? = null
5460
) {
61+
62+
@OptIn(PublicPreviewAPI::class)
5563
internal fun toPublic(): Candidate {
5664
val safetyRatings = safetyRatings?.mapNotNull { it.toPublic() }.orEmpty()
5765
val citations = citationMetadata?.toPublic()
5866
val finishReason = finishReason?.toPublic()
5967
val groundingMetadata = groundingMetadata?.toPublic()
68+
val urlContextMetadata = urlContextMetadata?.toPublic()
6069

6170
return Candidate(
6271
this.content?.toPublic() ?: content("model") {},
6372
safetyRatings,
6473
citations,
6574
finishReason,
66-
groundingMetadata
75+
groundingMetadata,
76+
urlContextMetadata
6777
)
6878
}
6979
}
@@ -372,8 +382,7 @@ public class SearchEntryPoint(
372382
}
373383

374384
/**
375-
* Represents a chunk of retrieved data that supports a claim in the model's response. This is part
376-
* of the grounding information provided when grounding is enabled.
385+
* Represents a chunk of retrieved data that supports a claim in the model's response.
377386
*
378387
* @property web Contains details if the grounding chunk is from a web source.
379388
*/
@@ -492,3 +501,87 @@ public class Segment(
492501
)
493502
}
494503
}
504+
505+
/**
506+
* Metadata related to the [UrlContext] tool.
507+
*
508+
* @property urlMetadata List of [UrlMetadata] used to provide context to the Gemini model.
509+
*/
510+
@PublicPreviewAPI
511+
public class UrlContextMetadata internal constructor(public val urlMetadata: List<UrlMetadata>) {
512+
513+
@Serializable
514+
@PublicPreviewAPI
515+
internal data class Internal(val urlMetadata: List<UrlMetadata.Internal>?) {
516+
internal fun toPublic() = UrlContextMetadata(urlMetadata?.map { it.toPublic() } ?: emptyList())
517+
}
518+
}
519+
520+
/**
521+
* Metadata for a single URL retrieved by the [UrlContext] tool.
522+
*
523+
* @property retrievedUrl The retrieved URL.
524+
* @property urlRetrievalStatus The status of the URL retrieval.
525+
*/
526+
@PublicPreviewAPI
527+
public class UrlMetadata
528+
internal constructor(
529+
public val retrievedUrl: String?,
530+
public val urlRetrievalStatus: UrlRetrievalStatus
531+
) {
532+
@Serializable
533+
internal data class Internal(
534+
val retrievedUrl: String?,
535+
val urlRetrievalStatus: UrlRetrievalStatus.Internal
536+
) {
537+
internal fun toPublic() = UrlMetadata(retrievedUrl, urlRetrievalStatus.toPublic())
538+
}
539+
}
540+
541+
/**
542+
* The status of a URL retrieval.
543+
*
544+
* @property name The name of the retrieval status.
545+
* @property ordinal The ordinal value of the retrieval status.
546+
*/
547+
@PublicPreviewAPI
548+
public class UrlRetrievalStatus
549+
private constructor(public val name: String, public val ordinal: Int) {
550+
551+
@Serializable(Internal.Serializer::class)
552+
internal enum class Internal {
553+
@SerialName("URL_RETRIEVAL_STATUS_UNSPECIFIED") UNSPECIFIED,
554+
@SerialName("URL_RETRIEVAL_STATUS_SUCCESS") SUCCESS,
555+
@SerialName("URL_RETRIEVAL_STATUS_ERROR") ERROR,
556+
@SerialName("URL_RETRIEVAL_STATUS_PAYWALL") PAYWALL,
557+
@SerialName("URL_RETRIEVAL_STATUS_UNSAFE") UNSAFE;
558+
559+
internal object Serializer : KSerializer<Internal> by FirstOrdinalSerializer(Internal::class)
560+
561+
internal fun toPublic() =
562+
when (this) {
563+
SUCCESS -> UrlRetrievalStatus.SUCCESS
564+
ERROR -> UrlRetrievalStatus.ERROR
565+
PAYWALL -> UrlRetrievalStatus.PAYWALL
566+
UNSAFE -> UrlRetrievalStatus.UNSAFE
567+
else -> UrlRetrievalStatus.UNSPECIFIED
568+
}
569+
}
570+
571+
public companion object {
572+
/** Unspecified retrieval status. */
573+
@JvmField public val UNSPECIFIED: UrlRetrievalStatus = UrlRetrievalStatus("UNSPECIFIED", 0)
574+
575+
/** The URL retrieval was successful. */
576+
@JvmField public val SUCCESS: UrlRetrievalStatus = UrlRetrievalStatus("SUCCESS", 1)
577+
578+
/** The URL retrieval failed. */
579+
@JvmField public val ERROR: UrlRetrievalStatus = UrlRetrievalStatus("ERROR", 2)
580+
581+
/** The URL retrieval failed because the content is behind a paywall. */
582+
@JvmField public val PAYWALL: UrlRetrievalStatus = UrlRetrievalStatus("PAYWALL", 3)
583+
584+
/** The URL retrieval failed because the content is unsafe. */
585+
@JvmField public val UNSAFE: UrlRetrievalStatus = UrlRetrievalStatus("UNSAFE", 4)
586+
}
587+
}

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/type/PublicPreviewAPI.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,5 @@ package com.google.firebase.ai.type
2323
"This API is part of an experimental public preview and may change in " +
2424
"backwards-incompatible ways without notice.",
2525
)
26+
@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY)
2627
public annotation class PublicPreviewAPI()

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,27 +24,36 @@ import kotlinx.serialization.json.JsonObject
2424
* can be used to gather information or complete tasks.
2525
*/
2626
public class Tool
27+
@OptIn(PublicPreviewAPI::class)
2728
internal constructor(
2829
internal val functionDeclarations: List<FunctionDeclaration>?,
2930
internal val googleSearch: GoogleSearch?,
3031
internal val codeExecution: JsonObject?,
32+
@property:PublicPreviewAPI internal val urlContext: UrlContext?,
3133
) {
34+
35+
@OptIn(PublicPreviewAPI::class)
3236
internal fun toInternal() =
3337
Internal(
3438
functionDeclarations?.map { it.toInternal() } ?: emptyList(),
3539
googleSearch = this.googleSearch?.toInternal(),
36-
codeExecution = this.codeExecution
40+
codeExecution = this.codeExecution,
41+
urlContext = this.urlContext?.toInternal()
3742
)
43+
44+
@OptIn(PublicPreviewAPI::class)
3845
@Serializable
3946
internal data class Internal(
4047
val functionDeclarations: List<FunctionDeclaration.Internal>? = null,
4148
val googleSearch: GoogleSearch.Internal? = null,
4249
// This is a json object because it is not possible to make a data class with no parameters.
4350
val codeExecution: JsonObject? = null,
51+
val urlContext: UrlContext.Internal? = null,
4452
)
4553
public companion object {
4654

47-
private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap())) }
55+
@OptIn(PublicPreviewAPI::class)
56+
private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap()), null) }
4857

4958
/**
5059
* Creates a [Tool] instance that provides the model with access to the [functionDeclarations].
@@ -53,7 +62,7 @@ internal constructor(
5362
*/
5463
@JvmStatic
5564
public fun functionDeclarations(functionDeclarations: List<FunctionDeclaration>): Tool {
56-
return Tool(functionDeclarations, null, null)
65+
@OptIn(PublicPreviewAPI::class) return Tool(functionDeclarations, null, null, null)
5766
}
5867

5968
/** Creates a [Tool] instance that allows the model to use code execution. */
@@ -62,6 +71,20 @@ internal constructor(
6271
return codeExecutionInstance
6372
}
6473

74+
/**
75+
* Creates a [Tool] instance that allows you to provide additional context to the models in the
76+
* form of public web URLs. By including URLs in your request, the Gemini model will access the
77+
* content from those pages to inform and enhance its response.
78+
*
79+
* @param urlContext Specifies the URL context configuration.
80+
* @return A [Tool] configured for URL context.
81+
*/
82+
@PublicPreviewAPI
83+
@JvmStatic
84+
public fun urlContext(urlContext: UrlContext = UrlContext()): Tool {
85+
return Tool(null, null, null, urlContext)
86+
}
87+
6588
/**
6689
* Creates a [Tool] instance that allows the model to use grounding with Google Search.
6790
*
@@ -80,7 +103,7 @@ internal constructor(
80103
*/
81104
@JvmStatic
82105
public fun googleSearch(googleSearch: GoogleSearch = GoogleSearch()): Tool {
83-
return Tool(null, googleSearch, null)
106+
@OptIn(PublicPreviewAPI::class) return Tool(null, googleSearch, null, null)
84107
}
85108
}
86109
}

‎firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UrlContext.kt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.firebase.ai.type
18+
19+
import kotlinx.serialization.Serializable
20+
21+
/** Specifies the URL context configuration. */
22+
@PublicPreviewAPI
23+
public class UrlContext {
24+
@Serializable internal class Internal()
25+
26+
internal fun toInternal() = Internal()
27+
}

0 commit comments

Comments
 (0)