diff --git a/firebase-ai/CHANGELOG.md b/firebase-ai/CHANGELOG.md index d694b73cce8..a31cc0efe5a 100644 --- a/firebase-ai/CHANGELOG.md +++ b/firebase-ai/CHANGELOG.md @@ -1,7 +1,8 @@ # Unreleased - [changed] **Breaking Change**: Removed the `candidateCount` option from `LiveGenerationConfig` - (#7382) +- [changed] Added support for the URL context tool, which allows the model to access content from + provided public web URLs to inform and enhance its responses. (#7382) - [changed] Added better error messages to `ServiceConnectionHandshakeFailedException` (#7412) - [changed] Marked the public constructor for `UsageMetadata` as deprecated (#7420) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index a390a14147e..6def4f53bb7 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -191,11 +191,13 @@ package com.google.firebase.ai.type { method public com.google.firebase.ai.type.FinishReason? getFinishReason(); method public com.google.firebase.ai.type.GroundingMetadata? getGroundingMetadata(); method public java.util.List getSafetyRatings(); + method public com.google.firebase.ai.type.UrlContextMetadata? getUrlContextMetadata(); property public final com.google.firebase.ai.type.CitationMetadata? citationMetadata; property public final com.google.firebase.ai.type.Content content; property public final com.google.firebase.ai.type.FinishReason? finishReason; property public final com.google.firebase.ai.type.GroundingMetadata? groundingMetadata; property public final java.util.List safetyRatings; + property public final com.google.firebase.ai.type.UrlContextMetadata? urlContextMetadata; } public final class Citation { @@ -942,7 +944,7 @@ package com.google.firebase.ai.type { property public final java.util.List safetyRatings; } - @kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) public @interface PublicPreviewAPI { + @kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) @kotlin.annotation.Target(allowedTargets={kotlin.annotation.AnnotationTarget.CLASS, kotlin.annotation.AnnotationTarget.FUNCTION, kotlin.annotation.AnnotationTarget.PROPERTY}) public @interface PublicPreviewAPI { } public final class QuotaExceededException extends com.google.firebase.ai.type.FirebaseAIException { @@ -1201,6 +1203,7 @@ package com.google.firebase.ai.type { method public static com.google.firebase.ai.type.Tool codeExecution(); method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List functionDeclarations); method public static com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch()); + method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext()); field public static final com.google.firebase.ai.type.Tool.Companion Companion; } @@ -1208,6 +1211,7 @@ package com.google.firebase.ai.type { method public com.google.firebase.ai.type.Tool codeExecution(); method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List functionDeclarations); method public com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch()); + method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.type.Tool urlContext(com.google.firebase.ai.type.UrlContext urlContext = com.google.firebase.ai.type.UrlContext()); } public final class ToolConfig { @@ -1220,19 +1224,55 @@ package com.google.firebase.ai.type { public final class UnsupportedUserLocationException extends com.google.firebase.ai.type.FirebaseAIException { } + @com.google.firebase.ai.type.PublicPreviewAPI public final class UrlContext { + ctor public UrlContext(); + } + + @com.google.firebase.ai.type.PublicPreviewAPI public final class UrlContextMetadata { + method public java.util.List getUrlMetadata(); + property public final java.util.List urlMetadata; + } + + @com.google.firebase.ai.type.PublicPreviewAPI public final class UrlMetadata { + method public String? getRetrievedUrl(); + method public com.google.firebase.ai.type.UrlRetrievalStatus getUrlRetrievalStatus(); + property public final String? retrievedUrl; + property public final com.google.firebase.ai.type.UrlRetrievalStatus urlRetrievalStatus; + } + + @com.google.firebase.ai.type.PublicPreviewAPI public final class UrlRetrievalStatus { + method public String getName(); + method public int getOrdinal(); + property public final String name; + property public final int ordinal; + field public static final com.google.firebase.ai.type.UrlRetrievalStatus.Companion Companion; + field public static final com.google.firebase.ai.type.UrlRetrievalStatus ERROR; + field public static final com.google.firebase.ai.type.UrlRetrievalStatus PAYWALL; + field public static final com.google.firebase.ai.type.UrlRetrievalStatus SUCCESS; + field public static final com.google.firebase.ai.type.UrlRetrievalStatus UNSAFE; + field public static final com.google.firebase.ai.type.UrlRetrievalStatus UNSPECIFIED; + } + + public static final class UrlRetrievalStatus.Companion { + } + public final class UsageMetadata { - ctor public UsageMetadata(int promptTokenCount, Integer? candidatesTokenCount, int totalTokenCount, java.util.List promptTokensDetails, java.util.List candidatesTokensDetails, int thoughtsTokenCount); + ctor @Deprecated public UsageMetadata(int promptTokenCount, Integer? candidatesTokenCount, int totalTokenCount, java.util.List promptTokensDetails, java.util.List candidatesTokensDetails, int thoughtsTokenCount); method public Integer? getCandidatesTokenCount(); method public java.util.List getCandidatesTokensDetails(); method public int getPromptTokenCount(); method public java.util.List getPromptTokensDetails(); method public int getThoughtsTokenCount(); + method public int getToolUsePromptTokenCount(); + method public java.util.List getToolUsePromptTokensDetails(); method public int getTotalTokenCount(); property public final Integer? candidatesTokenCount; property public final java.util.List candidatesTokensDetails; property public final int promptTokenCount; property public final java.util.List promptTokensDetails; property public final int thoughtsTokenCount; + property public final int toolUsePromptTokenCount; + property public final java.util.List toolUsePromptTokensDetails; property public final int totalTokenCount; } diff --git a/firebase-ai/gradle.properties b/firebase-ai/gradle.properties index 794b7a23197..a61baee5a19 100644 --- a/firebase-ai/gradle.properties +++ b/firebase-ai/gradle.properties @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -version=17.3.1 +version=17.4.0 latestReleasedVersion=17.3.0 diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt index 04cb9cc974a..5b0c57ce61a 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt @@ -33,37 +33,47 @@ import kotlinx.serialization.json.JsonNames * @property safetyRatings A list of [SafetyRating]s describing the generated content. * @property citationMetadata Metadata about the sources used to generate this content. * @property finishReason The reason the model stopped generating content, if it exist. - * @property groundingMetadata Metadata returned to the client when grounding is enabled. + * @property groundingMetadata Metadata returned to the client when the model grounds its response. + * @property urlContextMetadata Metadata returned to the client when the [UrlContext] tool is + * enabled. */ public class Candidate +@OptIn(PublicPreviewAPI::class) internal constructor( public val content: Content, public val safetyRatings: List, public val citationMetadata: CitationMetadata?, public val finishReason: FinishReason?, - public val groundingMetadata: GroundingMetadata? + public val groundingMetadata: GroundingMetadata?, + @property:PublicPreviewAPI public val urlContextMetadata: UrlContextMetadata? ) { + @OptIn(PublicPreviewAPI::class) @Serializable internal data class Internal( val content: Content.Internal? = null, val finishReason: FinishReason.Internal? = null, val safetyRatings: List? = null, val citationMetadata: CitationMetadata.Internal? = null, - val groundingMetadata: GroundingMetadata.Internal? = null + val groundingMetadata: GroundingMetadata.Internal? = null, + val urlContextMetadata: UrlContextMetadata.Internal? = null ) { + + @OptIn(PublicPreviewAPI::class) internal fun toPublic(): Candidate { val safetyRatings = safetyRatings?.mapNotNull { it.toPublic() }.orEmpty() val citations = citationMetadata?.toPublic() val finishReason = finishReason?.toPublic() val groundingMetadata = groundingMetadata?.toPublic() + val urlContextMetadata = urlContextMetadata?.toPublic() return Candidate( this.content?.toPublic() ?: content("model") {}, safetyRatings, citations, finishReason, - groundingMetadata + groundingMetadata, + urlContextMetadata ) } } @@ -372,8 +382,7 @@ public class SearchEntryPoint( } /** - * Represents a chunk of retrieved data that supports a claim in the model's response. This is part - * of the grounding information provided when grounding is enabled. + * Represents a chunk of retrieved data that supports a claim in the model's response. * * @property web Contains details if the grounding chunk is from a web source. */ @@ -492,3 +501,87 @@ public class Segment( ) } } + +/** + * Metadata related to the [UrlContext] tool. + * + * @property urlMetadata List of [UrlMetadata] used to provide context to the Gemini model. + */ +@PublicPreviewAPI +public class UrlContextMetadata internal constructor(public val urlMetadata: List) { + + @Serializable + @PublicPreviewAPI + internal data class Internal(val urlMetadata: List?) { + internal fun toPublic() = UrlContextMetadata(urlMetadata?.map { it.toPublic() } ?: emptyList()) + } +} + +/** + * Metadata for a single URL retrieved by the [UrlContext] tool. + * + * @property retrievedUrl The retrieved URL. + * @property urlRetrievalStatus The status of the URL retrieval. + */ +@PublicPreviewAPI +public class UrlMetadata +internal constructor( + public val retrievedUrl: String?, + public val urlRetrievalStatus: UrlRetrievalStatus +) { + @Serializable + internal data class Internal( + val retrievedUrl: String?, + val urlRetrievalStatus: UrlRetrievalStatus.Internal + ) { + internal fun toPublic() = UrlMetadata(retrievedUrl, urlRetrievalStatus.toPublic()) + } +} + +/** + * The status of a URL retrieval. + * + * @property name The name of the retrieval status. + * @property ordinal The ordinal value of the retrieval status. + */ +@PublicPreviewAPI +public class UrlRetrievalStatus +private constructor(public val name: String, public val ordinal: Int) { + + @Serializable(Internal.Serializer::class) + internal enum class Internal { + @SerialName("URL_RETRIEVAL_STATUS_UNSPECIFIED") UNSPECIFIED, + @SerialName("URL_RETRIEVAL_STATUS_SUCCESS") SUCCESS, + @SerialName("URL_RETRIEVAL_STATUS_ERROR") ERROR, + @SerialName("URL_RETRIEVAL_STATUS_PAYWALL") PAYWALL, + @SerialName("URL_RETRIEVAL_STATUS_UNSAFE") UNSAFE; + + internal object Serializer : KSerializer by FirstOrdinalSerializer(Internal::class) + + internal fun toPublic() = + when (this) { + SUCCESS -> UrlRetrievalStatus.SUCCESS + ERROR -> UrlRetrievalStatus.ERROR + PAYWALL -> UrlRetrievalStatus.PAYWALL + UNSAFE -> UrlRetrievalStatus.UNSAFE + else -> UrlRetrievalStatus.UNSPECIFIED + } + } + + public companion object { + /** Unspecified retrieval status. */ + @JvmField public val UNSPECIFIED: UrlRetrievalStatus = UrlRetrievalStatus("UNSPECIFIED", 0) + + /** The URL retrieval was successful. */ + @JvmField public val SUCCESS: UrlRetrievalStatus = UrlRetrievalStatus("SUCCESS", 1) + + /** The URL retrieval failed. */ + @JvmField public val ERROR: UrlRetrievalStatus = UrlRetrievalStatus("ERROR", 2) + + /** The URL retrieval failed because the content is behind a paywall. */ + @JvmField public val PAYWALL: UrlRetrievalStatus = UrlRetrievalStatus("PAYWALL", 3) + + /** The URL retrieval failed because the content is unsafe. */ + @JvmField public val UNSAFE: UrlRetrievalStatus = UrlRetrievalStatus("UNSAFE", 4) + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/PublicPreviewAPI.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/PublicPreviewAPI.kt index bc4a53cc8eb..3979fbc0a20 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/PublicPreviewAPI.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/PublicPreviewAPI.kt @@ -23,4 +23,5 @@ package com.google.firebase.ai.type "This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.", ) +@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY) public annotation class PublicPreviewAPI() diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt index 471e58f50b5..43a66a10d62 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Tool.kt @@ -24,27 +24,36 @@ import kotlinx.serialization.json.JsonObject * can be used to gather information or complete tasks. */ public class Tool +@OptIn(PublicPreviewAPI::class) internal constructor( internal val functionDeclarations: List?, internal val googleSearch: GoogleSearch?, internal val codeExecution: JsonObject?, + @property:PublicPreviewAPI internal val urlContext: UrlContext?, ) { + + @OptIn(PublicPreviewAPI::class) internal fun toInternal() = Internal( functionDeclarations?.map { it.toInternal() } ?: emptyList(), googleSearch = this.googleSearch?.toInternal(), - codeExecution = this.codeExecution + codeExecution = this.codeExecution, + urlContext = this.urlContext?.toInternal() ) + + @OptIn(PublicPreviewAPI::class) @Serializable internal data class Internal( val functionDeclarations: List? = null, val googleSearch: GoogleSearch.Internal? = null, // This is a json object because it is not possible to make a data class with no parameters. val codeExecution: JsonObject? = null, + val urlContext: UrlContext.Internal? = null, ) public companion object { - private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap())) } + @OptIn(PublicPreviewAPI::class) + private val codeExecutionInstance by lazy { Tool(null, null, JsonObject(emptyMap()), null) } /** * Creates a [Tool] instance that provides the model with access to the [functionDeclarations]. @@ -53,7 +62,7 @@ internal constructor( */ @JvmStatic public fun functionDeclarations(functionDeclarations: List): Tool { - return Tool(functionDeclarations, null, null) + @OptIn(PublicPreviewAPI::class) return Tool(functionDeclarations, null, null, null) } /** Creates a [Tool] instance that allows the model to use code execution. */ @@ -62,6 +71,20 @@ internal constructor( return codeExecutionInstance } + /** + * Creates a [Tool] instance that allows you to provide additional context to the models in the + * form of public web URLs. By including URLs in your request, the Gemini model will access the + * content from those pages to inform and enhance its response. + * + * @param urlContext Specifies the URL context configuration. + * @return A [Tool] configured for URL context. + */ + @PublicPreviewAPI + @JvmStatic + public fun urlContext(urlContext: UrlContext = UrlContext()): Tool { + return Tool(null, null, null, urlContext) + } + /** * Creates a [Tool] instance that allows the model to use grounding with Google Search. * @@ -80,7 +103,7 @@ internal constructor( */ @JvmStatic public fun googleSearch(googleSearch: GoogleSearch = GoogleSearch()): Tool { - return Tool(null, googleSearch, null) + @OptIn(PublicPreviewAPI::class) return Tool(null, googleSearch, null, null) } } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UrlContext.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UrlContext.kt new file mode 100644 index 00000000000..a2f15e25043 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UrlContext.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.type + +import kotlinx.serialization.Serializable + +/** Specifies the URL context configuration. */ +@PublicPreviewAPI +public class UrlContext { + @Serializable internal class Internal() + + internal fun toInternal() = Internal() +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UsageMetadata.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UsageMetadata.kt index e733445e916..60e0ef72e72 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UsageMetadata.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/UsageMetadata.kt @@ -28,7 +28,10 @@ import kotlinx.serialization.Serializable * prompt. * @param candidatesTokensDetails The breakdown, by modality, of how many tokens are consumed by the * candidates. + * @param toolUsePromptTokensDetails The breakdown, by modality, of how many tokens are consumed by + * tools. * @param thoughtsTokenCount The number of tokens used by the model's internal "thinking" process. + * @param toolUsePromptTokenCount The number of tokens used by tools. */ public class UsageMetadata internal constructor( @@ -38,7 +41,8 @@ internal constructor( public val promptTokensDetails: List, public val candidatesTokensDetails: List, public val thoughtsTokenCount: Int, - placeholder: Unit + public val toolUsePromptTokenCount: Int, + public val toolUsePromptTokensDetails: List ) { @Deprecated("Not intended for public use") @@ -56,7 +60,8 @@ internal constructor( promptTokensDetails, candidatesTokensDetails, thoughtsTokenCount, - Unit + 0, + emptyList() ) @Serializable @@ -67,6 +72,8 @@ internal constructor( val promptTokensDetails: List? = null, val candidatesTokensDetails: List? = null, val thoughtsTokenCount: Int? = null, + val toolUsePromptTokenCount: Int? = null, + val toolUsePromptTokensDetails: List? = null, ) { internal fun toPublic(): UsageMetadata = @@ -76,7 +83,10 @@ internal constructor( totalTokenCount ?: 0, promptTokensDetails = promptTokensDetails?.map { it.toPublic() } ?: emptyList(), candidatesTokensDetails = candidatesTokensDetails?.map { it.toPublic() } ?: emptyList(), - thoughtsTokenCount ?: 0 + thoughtsTokenCount ?: 0, + toolUsePromptTokenCount ?: 0, + toolUsePromptTokensDetails = toolUsePromptTokensDetails?.map { it.toPublic() } + ?: emptyList(), ) } } diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/DevAPIUnarySnapshotTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/DevAPIUnarySnapshotTests.kt index 10c6f9ec53f..e0ba386091b 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/DevAPIUnarySnapshotTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/DevAPIUnarySnapshotTests.kt @@ -18,12 +18,16 @@ package com.google.firebase.ai import com.google.firebase.ai.type.FinishReason import com.google.firebase.ai.type.InvalidAPIKeyException +import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.ResponseStoppedException import com.google.firebase.ai.type.ServerException +import com.google.firebase.ai.type.UrlRetrievalStatus import com.google.firebase.ai.util.goldenDevAPIUnaryFile import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.collections.shouldNotBeEmpty +import io.kotest.matchers.ints.shouldBeGreaterThan import io.kotest.matchers.nulls.shouldBeNull import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.should @@ -132,6 +136,76 @@ internal class DevAPIUnarySnapshotTests { } } + @OptIn(PublicPreviewAPI::class) + @Test + fun `url context`() = + goldenDevAPIUnaryFile("unary-success-url-context.json") { + withTimeout(testTimeout) { + val response = model.generateContent("prompt") + + response.candidates.shouldNotBeEmpty() + val candidate = response.candidates.first() + + val urlContextMetadata = candidate.urlContextMetadata + urlContextMetadata.shouldNotBeNull() + + urlContextMetadata.urlMetadata.shouldNotBeEmpty() + urlContextMetadata.urlMetadata.shouldHaveSize(1) + urlContextMetadata.urlMetadata[0].retrievedUrl.shouldBe("https://berkshirehathaway.com") + urlContextMetadata.urlMetadata[0].urlRetrievalStatus.shouldBe(UrlRetrievalStatus.SUCCESS) + + val groundingMetadata = candidate.groundingMetadata + groundingMetadata.shouldNotBeNull() + + groundingMetadata.groundingChunks.shouldNotBeEmpty() + groundingMetadata.groundingChunks.forEach { it.web.shouldNotBeNull() } + groundingMetadata.groundingSupports.shouldHaveSize(4) + + val usageMetadata = response.usageMetadata + + usageMetadata.shouldNotBeNull() + usageMetadata.toolUsePromptTokenCount.shouldBeGreaterThan(0) + usageMetadata.toolUsePromptTokensDetails.shouldHaveSize(1) + } + } + + @OptIn(PublicPreviewAPI::class) + @Test + fun `url context mixed validity`() = + goldenDevAPIUnaryFile("unary-success-url-context-mixed-validity.json") { + withTimeout(testTimeout) { + val response = model.generateContent("prompt") + + response.candidates.shouldNotBeEmpty() + val candidate = response.candidates.first() + + val urlContextMetadata = candidate.urlContextMetadata + urlContextMetadata.shouldNotBeNull() + + urlContextMetadata.urlMetadata.shouldNotBeEmpty() + urlContextMetadata.urlMetadata.shouldHaveSize(3) + urlContextMetadata.urlMetadata[0] + .retrievedUrl + .shouldBe("https://a-completely-non-existent-url-for-testing.org") + urlContextMetadata.urlMetadata[0].urlRetrievalStatus.shouldBe(UrlRetrievalStatus.ERROR) + urlContextMetadata.urlMetadata[1].retrievedUrl.shouldBe("https://ai.google.dev") + urlContextMetadata.urlMetadata[1].urlRetrievalStatus.shouldBe(UrlRetrievalStatus.SUCCESS) + + val groundingMetadata = candidate.groundingMetadata + groundingMetadata.shouldNotBeNull() + + groundingMetadata.groundingChunks.shouldNotBeEmpty() + groundingMetadata.groundingChunks.forEach { it.web.shouldNotBeNull() } + groundingMetadata.groundingSupports.shouldHaveSize(3) + + val usageMetadata = response.usageMetadata + + usageMetadata.shouldNotBeNull() + usageMetadata.toolUsePromptTokenCount.shouldBeGreaterThan(0) + usageMetadata.toolUsePromptTokensDetails.shouldHaveSize(1) + } + } + @Test fun `thinking function call and thought signature`() = goldenDevAPIUnaryFile("unary-success-thinking-function-call-thought-summary-signature.json") { diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/GenerativeModelTesting.kt b/firebase-ai/src/test/java/com/google/firebase/ai/GenerativeModelTesting.kt index 3a8b6473612..e12e133efc8 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/GenerativeModelTesting.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/GenerativeModelTesting.kt @@ -23,6 +23,7 @@ import com.google.firebase.ai.common.util.doBlocking import com.google.firebase.ai.type.Candidate import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.GenerateContentResponse +import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.RequestOptions import com.google.firebase.ai.type.ServerException import com.google.firebase.ai.type.TextPart @@ -41,7 +42,6 @@ import io.ktor.http.content.TextContent import io.ktor.http.headersOf import kotlin.time.Duration.Companion.seconds import kotlinx.coroutines.withTimeout -import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.encodeToString import org.junit.Before import org.junit.Test @@ -146,7 +146,7 @@ internal class GenerativeModelTesting { exception.message shouldContain "location" } - @OptIn(ExperimentalSerializationApi::class) + @OptIn(PublicPreviewAPI::class) private fun generateContentResponseAsJsonString(text: String): String { return JSON.encodeToString( GenerateContentResponse.Internal( diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt index 177fde19a8b..e6bea7a9e6f 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt @@ -32,6 +32,9 @@ import com.google.firebase.ai.type.Schema import com.google.firebase.ai.type.SearchEntryPoint import com.google.firebase.ai.type.Segment import com.google.firebase.ai.type.Tool +import com.google.firebase.ai.type.UrlContext +import com.google.firebase.ai.type.UrlContextMetadata +import com.google.firebase.ai.type.UrlMetadata import com.google.firebase.ai.type.WebGroundingChunk import io.kotest.assertions.json.shouldEqualJson import org.junit.Test @@ -162,7 +165,10 @@ internal class SerializationTests { }, "groundingMetadata": { "${'$'}ref": "GroundingMetadata" - } + }, + "urlContextMetadata": { + "${'$'}ref": "UrlContextMetadata" + } } } """ @@ -291,6 +297,53 @@ internal class SerializationTests { expectedJsonAsString shouldEqualJson actualJson.toString() } + @Test + fun `test UrlContextMetadata serialization as Json`() { + val expectedJsonAsString = + """ + { + "id": "UrlContextMetadata", + "type": "object", + "properties": { + "urlMetadata": { "type": "array", "items": { "${'$'}ref": "UrlMetadata" } } + } + } + """ + .trimIndent() + val actualJson = descriptorToJson(UrlContextMetadata.Internal.serializer().descriptor) + println(actualJson) + expectedJsonAsString shouldEqualJson actualJson.toString() + } + + @Test + fun `test UrlMetadata serialization as Json`() { + val expectedJsonAsString = + """ + { + "id": "UrlMetadata", + "type": "object", + "properties": { + "retrievedUrl": { + "type": "string" + }, + "urlRetrievalStatus": { + "type": "string", + "enum": [ + "UNSPECIFIED", + "SUCCESS", + "ERROR", + "PAYWALL", + "UNSAFE" + ] + } + } + } + """ + .trimIndent() + val actualJson = descriptorToJson(UrlMetadata.Internal.serializer().descriptor) + expectedJsonAsString shouldEqualJson actualJson.toString() + } + @Test fun `test GroundingAttribution serialization as Json`() { val expectedJsonAsString = @@ -447,6 +500,9 @@ internal class SerializationTests { "additionalProperties": { "${'$'}ref": "JsonElement" } + }, + "urlContext": { + "${'$'}ref": "UrlContext" } } } @@ -470,4 +526,19 @@ internal class SerializationTests { val actualJson = descriptorToJson(GoogleSearch.Internal.serializer().descriptor) expectedJsonAsString shouldEqualJson actualJson.toString() } + + @Test + fun `test UrlContext serialization as Json`() { + val expectedJsonAsString = + """ + { + "id": "UrlContext", + "type": "object", + "properties": {} + } + """ + .trimIndent() + val actualJson = descriptorToJson(UrlContext.Internal.serializer().descriptor) + expectedJsonAsString shouldEqualJson actualJson.toString() + } } diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt index 55877725296..83e4bc2b2bc 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt @@ -34,12 +34,15 @@ import com.google.firebase.ai.type.ServerException import com.google.firebase.ai.type.ServiceDisabledException import com.google.firebase.ai.type.TextPart import com.google.firebase.ai.type.UnsupportedUserLocationException +import com.google.firebase.ai.type.UrlRetrievalStatus import com.google.firebase.ai.util.goldenVertexUnaryFile import com.google.firebase.ai.util.shouldNotBeNullOrEmpty import io.kotest.assertions.throwables.shouldThrow import io.kotest.inspectors.forAtLeastOne import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.collections.shouldNotBeEmpty +import io.kotest.matchers.ints.shouldBeGreaterThan import io.kotest.matchers.nulls.shouldBeNull import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.should @@ -663,4 +666,95 @@ internal class VertexAIUnarySnapshotTests { secondGroundingSupport.groundingChunkIndices.first() shouldBe 1 } } + + @Test + fun `url context`() = + goldenVertexUnaryFile("unary-success-url-context.json") { + withTimeout(testTimeout) { + val response = model.generateContent("prompt") + + response.candidates.shouldNotBeEmpty() + val candidate = response.candidates.first() + + val urlContextMetadata = candidate.urlContextMetadata + urlContextMetadata.shouldNotBeNull() + + urlContextMetadata.urlMetadata.shouldNotBeEmpty() + urlContextMetadata.urlMetadata.shouldHaveSize(1) + urlContextMetadata.urlMetadata[0].retrievedUrl.shouldBe("https://berkshirehathaway.com") + urlContextMetadata.urlMetadata[0].urlRetrievalStatus.shouldBe(UrlRetrievalStatus.SUCCESS) + + val groundingMetadata = candidate.groundingMetadata + groundingMetadata.shouldNotBeNull() + + groundingMetadata.groundingChunks.shouldNotBeEmpty() + groundingMetadata.groundingChunks.forEach { it.web.shouldNotBeNull() } + groundingMetadata.groundingSupports.shouldHaveSize(2) + + val usageMetadata = response.usageMetadata + + usageMetadata.shouldNotBeNull() + usageMetadata.toolUsePromptTokenCount.shouldBeGreaterThan(0) + usageMetadata.toolUsePromptTokensDetails + .shouldBeEmpty() // This isn't yet supported in Vertex AI + } + } + + @Test + fun `url context mixed validity`() = + goldenVertexUnaryFile("unary-success-url-context-mixed-validity.json") { + withTimeout(testTimeout) { + val response = model.generateContent("prompt") + + response.candidates.shouldNotBeEmpty() + val candidate = response.candidates.first() + + val urlContextMetadata = candidate.urlContextMetadata + urlContextMetadata.shouldNotBeNull() + + urlContextMetadata.urlMetadata.shouldNotBeEmpty() + urlContextMetadata.urlMetadata.shouldHaveSize(3) + urlContextMetadata.urlMetadata[2] + .retrievedUrl + .shouldBe("https://a-completely-non-existent-url-for-testing.org") + urlContextMetadata.urlMetadata[2].urlRetrievalStatus.shouldBe(UrlRetrievalStatus.ERROR) + urlContextMetadata.urlMetadata[1].retrievedUrl.shouldBe("https://ai.google.dev") + urlContextMetadata.urlMetadata[1].urlRetrievalStatus.shouldBe(UrlRetrievalStatus.SUCCESS) + + val groundingMetadata = candidate.groundingMetadata + groundingMetadata.shouldNotBeNull() + + groundingMetadata.groundingChunks.shouldNotBeEmpty() + groundingMetadata.groundingChunks.forEach { it.web.shouldNotBeNull() } + groundingMetadata.groundingSupports.shouldHaveSize(6) + + val usageMetadata = response.usageMetadata + + usageMetadata.shouldNotBeNull() + usageMetadata.toolUsePromptTokenCount.shouldBeGreaterThan(0) + usageMetadata.toolUsePromptTokensDetails + .shouldBeEmpty() // This isn't yet supported in Vertex AI + } + } + + // This test only applies to Vertex AI, since this is a bug in the backend. + @Test + fun `url context missing retrievedUrl`() = + goldenVertexUnaryFile("unary-success-url-context-missing-retrievedurl.json") { + withTimeout(testTimeout) { + val response = model.generateContent("prompt") + + response.candidates.shouldNotBeEmpty() + val candidate = response.candidates.first() + + val urlContextMetadata = candidate.urlContextMetadata + urlContextMetadata.shouldNotBeNull() + + urlContextMetadata.urlMetadata.shouldNotBeEmpty() + urlContextMetadata.urlMetadata.shouldHaveSize(20) + // Not all the retrievedUrls are null. Only the last 10. We only need to check one. + urlContextMetadata.urlMetadata.last().retrievedUrl.shouldBeNull() + urlContextMetadata.urlMetadata.last().urlRetrievalStatus.shouldNotBeNull() + } + } } diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/common/APIControllerTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/common/APIControllerTests.kt index c2e5dd591c7..f87298487ea 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/common/APIControllerTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/common/APIControllerTests.kt @@ -26,10 +26,12 @@ import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.CountTokensResponse import com.google.firebase.ai.type.FunctionCallingConfig import com.google.firebase.ai.type.GoogleSearch +import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.RequestOptions import com.google.firebase.ai.type.TextPart import com.google.firebase.ai.type.Tool import com.google.firebase.ai.type.ToolConfig +import com.google.firebase.ai.type.UrlContext import io.kotest.assertions.json.shouldContainJsonKey import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.shouldBe @@ -284,6 +286,7 @@ internal class RequestFormatTests { ) withTimeout(5.seconds) { + @OptIn(PublicPreviewAPI::class) controller .generateContentStream( GenerateContentRequest( @@ -300,6 +303,45 @@ internal class RequestFormatTests { requestBodyAsText shouldContainJsonKey "tools[0].googleSearch" } + @Test + fun `url context tool serialization contains correct keys`() = doBlocking { + val channel = ByteChannel(autoFlush = true) + val mockEngine = MockEngine { + respond(channel, HttpStatusCode.OK, headersOf(HttpHeaders.ContentType, "application/json")) + } + prepareStreamingResponse(createResponses("Random")).forEach { channel.writeFully(it) } + + val controller = + APIController( + "super_cool_test_key", + "gemini-pro-2.5", + RequestOptions(), + mockEngine, + TEST_CLIENT_ID, + mockFirebaseApp, + TEST_VERSION, + TEST_APP_ID, + null, + ) + + withTimeout(5.seconds) { + @OptIn(PublicPreviewAPI::class) + controller + .generateContentStream( + GenerateContentRequest( + model = "unused", + contents = listOf(Content.Internal(parts = listOf(TextPart.Internal("Arbitrary")))), + tools = listOf(Tool.Internal(urlContext = UrlContext.Internal())), + ) + ) + .collect { channel.close() } + } + + val requestBodyAsText = (mockEngine.requestHistory.first().body as TextContent).text + + requestBodyAsText shouldContainJsonKey "tools[0].urlContext" + } + @Test fun `headers from HeaderProvider are added to the request`() = doBlocking { val response = JSON.encodeToString(CountTokensResponse.Internal(totalTokens = 10)) @@ -393,6 +435,7 @@ internal class RequestFormatTests { ) withTimeout(5.seconds) { + @OptIn(PublicPreviewAPI::class) controller .generateContentStream( GenerateContentRequest( diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/common/util/tests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/common/util/tests.kt index 6cc501cedd5..d9081cae371 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/common/util/tests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/common/util/tests.kt @@ -24,6 +24,7 @@ import com.google.firebase.ai.common.JSON import com.google.firebase.ai.type.Candidate import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.GenerateContentResponse +import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.RequestOptions import com.google.firebase.ai.type.TextPart import io.ktor.client.engine.mock.MockEngine @@ -32,7 +33,6 @@ import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.headersOf import io.ktor.utils.io.ByteChannel -import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.encodeToString import org.mockito.Mockito @@ -44,7 +44,7 @@ internal fun prepareStreamingResponse( response: List ): List = response.map { "data: ${JSON.encodeToString(it)}$SSE_SEPARATOR".toByteArray() } -@OptIn(ExperimentalSerializationApi::class) +@OptIn(PublicPreviewAPI::class) internal fun createResponses(vararg text: String): List { val candidates = text.map { Candidate.Internal(Content.Internal(parts = listOf(TextPart.Internal(it)))) } diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/type/ToolTest.kt b/firebase-ai/src/test/java/com/google/firebase/ai/type/ToolTest.kt index a9f11dd2595..41c764b624a 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/type/ToolTest.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/type/ToolTest.kt @@ -28,6 +28,7 @@ internal class ToolTest { tool.googleSearch.shouldNotBeNull() tool.functionDeclarations.shouldBeNull() tool.codeExecution.shouldBeNull() + @OptIn(PublicPreviewAPI::class) tool.urlContext.shouldBeNull() } @Test @@ -38,6 +39,7 @@ internal class ToolTest { tool.functionDeclarations?.first() shouldBe functionDeclaration tool.googleSearch.shouldBeNull() tool.codeExecution.shouldBeNull() + @OptIn(PublicPreviewAPI::class) tool.urlContext.shouldBeNull() } @Test @@ -46,5 +48,17 @@ internal class ToolTest { tool.codeExecution.shouldNotBeNull() tool.functionDeclarations.shouldBeNull() tool.googleSearch.shouldBeNull() + @OptIn(PublicPreviewAPI::class) tool.urlContext.shouldBeNull() + } + + @OptIn(PublicPreviewAPI::class) + @Test + fun `urlContext() creates a tool with a urlContext property`() { + val tool = Tool.urlContext() + + tool.googleSearch.shouldBeNull() + tool.functionDeclarations.shouldBeNull() + tool.codeExecution.shouldBeNull() + @OptIn(PublicPreviewAPI::class) tool.urlContext.shouldNotBeNull() } }