From ec8a8c2dc6a2f6d4f1b6f153d688616bd3d22331 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 21 Oct 2025 18:40:25 -0700 Subject: [PATCH 01/13] Server Templates --- firebase-ai/api.txt | 13 ++ .../com/google/firebase/ai/FirebaseAI.kt | 54 ++++++++ .../com/google/firebase/ai/ImagenModel.kt | 2 +- .../firebase/ai/TemplateGenerativeModel.kt | 129 ++++++++++++++++++ .../google/firebase/ai/TemplateImagenModel.kt | 101 ++++++++++++++ .../firebase/ai/common/APIController.kt | 52 +++++++ .../com/google/firebase/ai/common/Request.kt | 9 ++ 7 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index f73c51d7112..6a8bb6cba5d 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -36,6 +36,10 @@ package com.google.firebase.ai { method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List? tools = null); method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List? tools = null, com.google.firebase.ai.type.Content? systemInstruction = null); method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List? tools = null, com.google.firebase.ai.type.Content? systemInstruction = null, com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions()); + method public com.google.firebase.ai.TemplateGenerativeModel templateGenerativeModel(); + method public com.google.firebase.ai.TemplateGenerativeModel templateGenerativeModel(com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions()); + method public com.google.firebase.ai.TemplateImagenModel templateImagenModel(); + method public com.google.firebase.ai.TemplateImagenModel templateImagenModel(com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions()); property public static final com.google.firebase.ai.FirebaseAI instance; field public static final com.google.firebase.ai.FirebaseAI.Companion Companion; } @@ -83,6 +87,15 @@ package com.google.firebase.ai { method public suspend Object? connect(kotlin.coroutines.Continuation); } + public final class TemplateGenerativeModel { + method public suspend Object? generateContent(String templateId, java.util.Map inputs, kotlin.coroutines.Continuation); + method public kotlinx.coroutines.flow.Flow generateContentStream(String templateId, java.util.Map inputs); + } + + public final class TemplateImagenModel { + method public suspend Object? generateImages(String templateId, java.util.Map inputs, kotlin.coroutines.Continuation>); + } + } package com.google.firebase.ai.java { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt index dd2309c984a..5864f02b093 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt @@ -106,6 +106,33 @@ internal constructor( ) } + /** + * Instantiates a new [TemplateGenerativeModel] given the provided parameters. + * + * @param requestOptions Configuration options for sending requests to the backend. + * @return The initialized [TemplateGenerativeModel] instance. + */ + @JvmOverloads + public fun templateGenerativeModel( + requestOptions: RequestOptions = RequestOptions(), + ): TemplateGenerativeModel { + val templateUri = + when (backend.backend) { + GenerativeBackendEnum.VERTEX_AI -> + "projects/${firebaseApp.options.projectId}/locations/${backend.location}/templates/" + GenerativeBackendEnum.GOOGLE_AI -> "projects/${firebaseApp.options.projectId}/templates/" + } + return TemplateGenerativeModel( + templateUri, + firebaseApp.options.apiKey, + firebaseApp, + useLimitedUseAppCheckTokens, + requestOptions, + appCheckProvider.get(), + internalAuthProvider.get(), + ) + } + /** * Instantiates a new [LiveGenerationConfig] given the provided parameters. * @@ -205,6 +232,33 @@ internal constructor( ) } + /** + * Instantiates a new [ImagenModel] given the provided parameters. + * + * @param requestOptions Configuration options for sending requests to the backend. + * @return The initialized [TemplateImagenModel] instance. + */ + @JvmOverloads + public fun templateImagenModel( + requestOptions: RequestOptions = RequestOptions(), + ): TemplateImagenModel { + val templateUri = + when (backend.backend) { + GenerativeBackendEnum.VERTEX_AI -> + "projects/${firebaseApp.options.projectId}/locations/${backend.location}/templates/" + GenerativeBackendEnum.GOOGLE_AI -> "projects/${firebaseApp.options.projectId}/templates/" + } + return TemplateImagenModel( + templateUri, + firebaseApp.options.apiKey, + firebaseApp, + useLimitedUseAppCheckTokens, + requestOptions, + appCheckProvider.get(), + internalAuthProvider.get(), + ) + } + public companion object { /** The [FirebaseAI] instance for the default [FirebaseApp] using the Google AI Backend. */ @JvmStatic diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index 62f11319f68..c562ea0233b 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -233,7 +233,7 @@ internal constructor( } @OptIn(PublicPreviewAPI::class) -private fun ImagenGenerationResponse.Internal.validate(): ImagenGenerationResponse.Internal { +internal fun ImagenGenerationResponse.Internal.validate(): ImagenGenerationResponse.Internal { if (predictions.none { it.mimeType != null }) { throw ContentBlockedException( message = predictions.first { it.raiFilteredReason != null }.raiFilteredReason diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt new file mode 100644 index 00000000000..09dd2a69aaf --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt @@ -0,0 +1,129 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai + +import com.google.firebase.FirebaseApp +import com.google.firebase.ai.common.APIController +import com.google.firebase.ai.common.AppCheckHeaderProvider +import com.google.firebase.ai.common.TemplateGenerateContentRequest +import com.google.firebase.ai.type.FinishReason +import com.google.firebase.ai.type.FirebaseAIException +import com.google.firebase.ai.type.GenerateContentResponse +import com.google.firebase.ai.type.PromptBlockedException +import com.google.firebase.ai.type.RequestOptions +import com.google.firebase.ai.type.ResponseStoppedException +import com.google.firebase.ai.type.SerializationException +import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider +import com.google.firebase.auth.internal.InternalAuthProvider +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.catch +import kotlinx.coroutines.flow.map + +/** + * Represents a multimodal model (like Gemini), capable of generating content based on various + * templated input types. + */ +public class TemplateGenerativeModel +internal constructor( + private val templateUri: String, + private val controller: APIController, +) { + + internal constructor( + templateUri: String, + apiKey: String, + firebaseApp: FirebaseApp, + useLimitedUseAppCheckTokens: Boolean, + requestOptions: RequestOptions = RequestOptions(), + appCheckTokenProvider: InteropAppCheckTokenProvider? = null, + internalAuthProvider: InternalAuthProvider? = null + ) : this( + templateUri, + APIController( + apiKey, + "", + requestOptions, + "gl-kotlin/${KotlinVersion.CURRENT}-ai fire/${BuildConfig.VERSION_NAME}", + firebaseApp, + AppCheckHeaderProvider( + TAG, + useLimitedUseAppCheckTokens, + appCheckTokenProvider, + internalAuthProvider + ), + ), + ) + + /** + * Generates new content using the given templateId with the given inputs. + * + * @param templateId The ID of server prompt template. + * @param inputs the inputs needed to fill in the prompt + * @return The content generated by the model. + * @throws [FirebaseAIException] if the request failed. + * @see [FirebaseAIException] for types of errors. + */ + public suspend fun generateContent( + templateId: String, + inputs: Map + ): GenerateContentResponse = + try { + controller + .templateGenerateContent("$templateUri$templateId", constructRequest(inputs)) + .toPublic() + .validate() + } catch (e: Throwable) { + throw FirebaseAIException.from(e) + } + + /** + * Generates new content as a stream using the given templateId with the given inputs. + * + * @param templateId The ID of server prompt template. + * @param inputs the inputs needed to fill in the prompt + * @return A [Flow] which will emit responses as they are returned by the model. + * @throws [FirebaseAIException] if the request failed. + * @see [FirebaseAIException] for types of errors. + */ + public fun generateContentStream( + templateId: String, + inputs: Map + ): Flow = + controller + .templateGenerateContentStream("$templateUri$templateId", constructRequest(inputs)) + .catch { throw FirebaseAIException.from(it) } + .map { it.toPublic().validate() } + + internal fun constructRequest(inputs: Map): TemplateGenerateContentRequest { + return TemplateGenerateContentRequest(inputs) + } + + private fun GenerateContentResponse.validate() = apply { + if (candidates.isEmpty() && promptFeedback == null) { + throw SerializationException("Error deserializing response, found no valid fields") + } + promptFeedback?.blockReason?.let { throw PromptBlockedException(this) } + candidates + .mapNotNull { it.finishReason } + .firstOrNull { it != FinishReason.STOP } + ?.let { throw ResponseStoppedException(this) } + } + + private companion object { + private val TAG = TemplateGenerativeModel::class.java.simpleName + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt new file mode 100644 index 00000000000..4934d6a3ef4 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt @@ -0,0 +1,101 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai + +import com.google.firebase.FirebaseApp +import com.google.firebase.ai.common.APIController +import com.google.firebase.ai.common.AppCheckHeaderProvider +import com.google.firebase.ai.common.TemplateGenerateImageRequest +import com.google.firebase.ai.type.FirebaseAIException +import com.google.firebase.ai.type.ImagenGenerationResponse +import com.google.firebase.ai.type.ImagenInlineImage +import com.google.firebase.ai.type.RequestOptions +import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider +import com.google.firebase.auth.internal.InternalAuthProvider + +/** + * Represents a generative model (like Imagen), capable of generating images based a template. + * + * See the documentation for a list of + * [supported models](https://firebase.google.com/docs/ai-logic/models). + */ +public class TemplateImagenModel +internal constructor( + private val templateUri: String, + private val controller: APIController, +) { + + @JvmOverloads + internal constructor( + templateUri: String, + apiKey: String, + firebaseApp: FirebaseApp, + useLimitedUseAppCheckTokens: Boolean, + requestOptions: RequestOptions = RequestOptions(), + appCheckTokenProvider: InteropAppCheckTokenProvider? = null, + internalAuthProvider: InternalAuthProvider? = null, + ) : this( + templateUri, + APIController( + apiKey, + "", + requestOptions, + "gl-kotlin/${KotlinVersion.CURRENT}-ai fire/${BuildConfig.VERSION_NAME}", + firebaseApp, + AppCheckHeaderProvider( + TAG, + useLimitedUseAppCheckTokens, + appCheckTokenProvider, + internalAuthProvider + ), + ), + ) + + /** + * Generates an image, returning the result directly to the caller. + * + * @param templateId The ID of server prompt template. + * @param inputs the inputs needed to fill in the prompt + */ + public suspend fun generateImages( + templateId: String, + inputs: Map + ): ImagenGenerationResponse = + try { + controller + .templateGenerateImage( + "$templateUri$templateId", + constructTemplateGenerateImageRequest(inputs) + ) + .validate() + .toPublicInline() + } catch (e: Throwable) { + throw FirebaseAIException.from(e) + } + + private fun constructTemplateGenerateImageRequest( + inputs: Map + ): TemplateGenerateImageRequest { + return TemplateGenerateImageRequest(inputs) + } + + internal companion object { + private val TAG = TemplateImagenModel::class.java.simpleName + internal const val DEFAULT_FILTERED_ERROR = + "Unable to show generated images. All images were filtered out because they violated Vertex AI's usage guidelines. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback." + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt index 220e5efedac..e992f92e674 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/APIController.kt @@ -164,6 +164,38 @@ internal constructor( throw FirebaseAIException.from(e) } + suspend fun templateGenerateContent( + templateId: String, + request: TemplateGenerateContentRequest + ): GenerateContentResponse.Internal = + try { + client + .post( + "${requestOptions.endpoint}/${requestOptions.apiVersion}/$templateId:templateGenerateContent" + ) { + applyCommonConfiguration(request) + applyHeaderProvider() + } + .also { validateResponse(it) } + .body() + .validate() + } catch (e: Throwable) { + throw FirebaseAIException.from(e) + } + + fun templateGenerateContentStream( + templateId: String, + request: TemplateGenerateContentRequest + ): Flow = + client + .postStream( + "${requestOptions.endpoint}/${requestOptions.apiVersion}/$templateId:templateStreamGenerateContent?alt=sse" + ) { + applyCommonConfiguration(request) + } + .map { it.validate() } + .catch { throw FirebaseAIException.from(it) } + suspend fun generateImage(request: GenerateImageRequest): ImagenGenerationResponse.Internal = try { client @@ -177,6 +209,24 @@ internal constructor( throw FirebaseAIException.from(e) } + suspend fun templateGenerateImage( + templateId: String, + request: TemplateGenerateImageRequest + ): ImagenGenerationResponse.Internal = + try { + client + .post( + "${requestOptions.endpoint}/${requestOptions.apiVersion}/$templateId:templatePredict" + ) { + applyCommonConfiguration(request) + applyHeaderProvider() + } + .also { validateResponse(it) } + .body() + } catch (e: Throwable) { + throw FirebaseAIException.from(e) + } + private fun getBidiEndpoint(location: String): String = when (backend?.backend) { GenerativeBackendEnum.VERTEX_AI, @@ -228,6 +278,8 @@ internal constructor( is GenerateContentRequest -> setBody(request) is CountTokensRequest -> setBody(request) is GenerateImageRequest -> setBody(request) + is TemplateGenerateContentRequest -> setBody(request) + is TemplateGenerateImageRequest -> setBody(request) } applyCommonHeaders() } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt index bb6bf242bb0..ca5f1165563 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt @@ -28,6 +28,7 @@ import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.SafetySetting import com.google.firebase.ai.type.Tool import com.google.firebase.ai.type.ToolConfig +import kotlinx.serialization.Contextual import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable @@ -45,6 +46,14 @@ internal data class GenerateContentRequest( @SerialName("system_instruction") val systemInstruction: Content.Internal? = null, ) : Request +@Serializable +internal data class TemplateGenerateContentRequest(val inputs: Map) : + Request + +@Serializable +internal data class TemplateGenerateImageRequest(val inputs: Map) : + Request + @Serializable internal data class CountTokensRequest( val generateContentRequest: GenerateContentRequest? = null, From 8a9e2e99cd59e0503f89921eeb1b1946c368d9ab Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 22 Oct 2025 10:06:12 -0700 Subject: [PATCH 02/13] add publicpreviewapi annotations and clean up code --- firebase-ai/api.txt | 12 ++++----- .../com/google/firebase/ai/FirebaseAI.kt | 25 +++++++++---------- .../firebase/ai/TemplateGenerativeModel.kt | 2 ++ .../google/firebase/ai/TemplateImagenModel.kt | 2 ++ 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 6a8bb6cba5d..3b119042655 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -36,10 +36,10 @@ package com.google.firebase.ai { method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List? tools = null); method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List? tools = null, com.google.firebase.ai.type.Content? systemInstruction = null); method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List? tools = null, com.google.firebase.ai.type.Content? systemInstruction = null, com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions()); - method public com.google.firebase.ai.TemplateGenerativeModel templateGenerativeModel(); - method public com.google.firebase.ai.TemplateGenerativeModel templateGenerativeModel(com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions()); - method public com.google.firebase.ai.TemplateImagenModel templateImagenModel(); - method public com.google.firebase.ai.TemplateImagenModel templateImagenModel(com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions()); + method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.TemplateGenerativeModel templateGenerativeModel(); + method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.TemplateGenerativeModel templateGenerativeModel(com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions()); + method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.TemplateImagenModel templateImagenModel(); + method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.TemplateImagenModel templateImagenModel(com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions()); property public static final com.google.firebase.ai.FirebaseAI instance; field public static final com.google.firebase.ai.FirebaseAI.Companion Companion; } @@ -87,12 +87,12 @@ package com.google.firebase.ai { method public suspend Object? connect(kotlin.coroutines.Continuation); } - public final class TemplateGenerativeModel { + @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateGenerativeModel { method public suspend Object? generateContent(String templateId, java.util.Map inputs, kotlin.coroutines.Continuation); method public kotlinx.coroutines.flow.Flow generateContentStream(String templateId, java.util.Map inputs); } - public final class TemplateImagenModel { + @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateImagenModel { method public suspend Object? generateImages(String templateId, java.util.Map inputs, kotlin.coroutines.Continuation>); } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt index 5864f02b093..52eeed8959b 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt @@ -113,15 +113,11 @@ internal constructor( * @return The initialized [TemplateGenerativeModel] instance. */ @JvmOverloads + @PublicPreviewAPI public fun templateGenerativeModel( requestOptions: RequestOptions = RequestOptions(), ): TemplateGenerativeModel { - val templateUri = - when (backend.backend) { - GenerativeBackendEnum.VERTEX_AI -> - "projects/${firebaseApp.options.projectId}/locations/${backend.location}/templates/" - GenerativeBackendEnum.GOOGLE_AI -> "projects/${firebaseApp.options.projectId}/templates/" - } + val templateUri = getTemplateUri(backend) return TemplateGenerativeModel( templateUri, firebaseApp.options.apiKey, @@ -233,21 +229,17 @@ internal constructor( } /** - * Instantiates a new [ImagenModel] given the provided parameters. + * Instantiates a new [TemplateImagenModel] given the provided parameters. * * @param requestOptions Configuration options for sending requests to the backend. * @return The initialized [TemplateImagenModel] instance. */ @JvmOverloads + @PublicPreviewAPI public fun templateImagenModel( requestOptions: RequestOptions = RequestOptions(), ): TemplateImagenModel { - val templateUri = - when (backend.backend) { - GenerativeBackendEnum.VERTEX_AI -> - "projects/${firebaseApp.options.projectId}/locations/${backend.location}/templates/" - GenerativeBackendEnum.GOOGLE_AI -> "projects/${firebaseApp.options.projectId}/templates/" - } + val templateUri = getTemplateUri(backend) return TemplateImagenModel( templateUri, firebaseApp.options.apiKey, @@ -312,6 +304,13 @@ internal constructor( private val TAG = FirebaseAI::class.java.simpleName } + + private fun getTemplateUri(backend: GenerativeBackend): String = + when (backend.backend) { + GenerativeBackendEnum.VERTEX_AI -> + "projects/${firebaseApp.options.projectId}/locations/${backend.location}/templates/" + GenerativeBackendEnum.GOOGLE_AI -> "projects/${firebaseApp.options.projectId}/templates/" + } } /** The [FirebaseAI] instance for the default [FirebaseApp] using the Google AI Backend. */ diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt index 09dd2a69aaf..b104654e74a 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt @@ -24,6 +24,7 @@ import com.google.firebase.ai.type.FinishReason import com.google.firebase.ai.type.FirebaseAIException import com.google.firebase.ai.type.GenerateContentResponse import com.google.firebase.ai.type.PromptBlockedException +import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.RequestOptions import com.google.firebase.ai.type.ResponseStoppedException import com.google.firebase.ai.type.SerializationException @@ -37,6 +38,7 @@ import kotlinx.coroutines.flow.map * Represents a multimodal model (like Gemini), capable of generating content based on various * templated input types. */ +@PublicPreviewAPI public class TemplateGenerativeModel internal constructor( private val templateUri: String, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt index 4934d6a3ef4..80da3b6aaef 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt @@ -23,6 +23,7 @@ import com.google.firebase.ai.common.TemplateGenerateImageRequest import com.google.firebase.ai.type.FirebaseAIException import com.google.firebase.ai.type.ImagenGenerationResponse import com.google.firebase.ai.type.ImagenInlineImage +import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.RequestOptions import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider import com.google.firebase.auth.internal.InternalAuthProvider @@ -33,6 +34,7 @@ import com.google.firebase.auth.internal.InternalAuthProvider * See the documentation for a list of * [supported models](https://firebase.google.com/docs/ai-logic/models). */ +@PublicPreviewAPI public class TemplateImagenModel internal constructor( private val templateUri: String, From a36782655d9541edd9cfd25fda4483890e6ec3da Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 22 Oct 2025 10:53:30 -0700 Subject: [PATCH 03/13] add java types --- firebase-ai/api.txt | 17 ++++ .../ai/java/TemplateGenerativeModelFutures.kt | 99 +++++++++++++++++++ .../ai/java/TemplateImagenModelFutures.kt | 60 +++++++++++ 3 files changed, 176 insertions(+) create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateGenerativeModelFutures.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 3b119042655..17838d5f6ae 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -179,6 +179,23 @@ package com.google.firebase.ai.java { method public com.google.firebase.ai.java.LiveSessionFutures from(com.google.firebase.ai.type.LiveSession session); } + public abstract class TemplateGenerativeModelFutures { + method public static final com.google.firebase.ai.java.TemplateGenerativeModelFutures from(com.google.firebase.ai.TemplateGenerativeModel model); + method public abstract com.google.common.util.concurrent.ListenableFuture generateContent(String templateId, java.util.Map inputs); + method public abstract org.reactivestreams.Publisher generateContentStream(String templateId, java.util.Map inputs); + method public abstract com.google.firebase.ai.TemplateGenerativeModel getGenerativeModel(); + field public static final com.google.firebase.ai.java.TemplateGenerativeModelFutures.Companion Companion; + } + + public static final class TemplateGenerativeModelFutures.Companion { + method public com.google.firebase.ai.java.TemplateGenerativeModelFutures from(com.google.firebase.ai.TemplateGenerativeModel model); + } + + public abstract class TemplateImagenModelFutures { + method public abstract com.google.common.util.concurrent.ListenableFuture> generateImages(String templateId, java.util.Map inputs); + method public abstract com.google.firebase.ai.TemplateImagenModel getImageModel(); + } + } package com.google.firebase.ai.type { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateGenerativeModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateGenerativeModelFutures.kt new file mode 100644 index 00000000000..08ed69723c0 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateGenerativeModelFutures.kt @@ -0,0 +1,99 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.java + +import androidx.concurrent.futures.SuspendToFutureAdapter +import com.google.common.util.concurrent.ListenableFuture +import com.google.firebase.ai.TemplateGenerativeModel +import com.google.firebase.ai.java.ImagenModelFutures.FuturesImpl +import com.google.firebase.ai.type.FirebaseAIException +import com.google.firebase.ai.type.GenerateContentResponse +import com.google.firebase.ai.type.PublicPreviewAPI +import kotlinx.coroutines.reactive.asPublisher +import org.reactivestreams.Publisher + +/** + * Wrapper class providing Java compatible methods for [TemplateGenerativeModel]. + * + * @see [TemplateGenerativeModel] + */ +@OptIn(PublicPreviewAPI::class) +public abstract class TemplateGenerativeModelFutures internal constructor() { + + /** + * Generates new content using the given templateId with the given inputs. + * + * @param templateId The ID of server prompt template. + * @param inputs the inputs needed to fill in the prompt + * @return The content generated by the model. + * @throws [FirebaseAIException] if the request failed. + * @see [FirebaseAIException] for types of errors. + */ + public abstract fun generateContent( + templateId: String, + inputs: Map + ): ListenableFuture + + /** + * Generates new content as a stream using the given templateId with the given inputs. + * + * @param templateId The ID of server prompt template. + * @param inputs the inputs needed to fill in the prompt + * @return A [Publisher] which will emit responses as they are returned by the model. + * @throws [FirebaseAIException] if the request failed. + * @see [FirebaseAIException] for types of errors. + */ + public abstract fun generateContentStream( + templateId: String, + inputs: Map + ): Publisher + + /** Returns the [TemplateGenerativeModel] object wrapped by this object. */ + public abstract fun getGenerativeModel(): TemplateGenerativeModel + + private class FuturesImpl(private val model: TemplateGenerativeModel) : + TemplateGenerativeModelFutures() { + override fun generateContent( + templateId: String, + inputs: Map + ): ListenableFuture { + return SuspendToFutureAdapter.launchFuture { model.generateContent(templateId, inputs) } + } + + override fun generateContentStream( + templateId: String, + inputs: Map + ): Publisher { + return model.generateContentStream(templateId, inputs).asPublisher() + } + + override fun getGenerativeModel(): TemplateGenerativeModel { + return model + } + } + + public companion object { + + /** + * @return a [TemplateGenerativeModelFutures] created around the provided + * [TemplateGenerativeModel] + */ + @JvmStatic + public fun from(model: TemplateGenerativeModel): TemplateGenerativeModelFutures = + FuturesImpl(model) + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt new file mode 100644 index 00000000000..0b7b6aa6808 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.java + +import androidx.concurrent.futures.SuspendToFutureAdapter +import com.google.common.util.concurrent.ListenableFuture +import com.google.firebase.ai.TemplateImagenModel +import com.google.firebase.ai.type.ImagenGenerationResponse +import com.google.firebase.ai.type.ImagenInlineImage +import com.google.firebase.ai.type.PublicPreviewAPI + +/** + * Wrapper class providing Java compatible methods for [TemplateImagenModel]. + * + * @see [TemplateImagenModel] + */ +@OptIn(PublicPreviewAPI::class) +public abstract class TemplateImagenModelFutures internal constructor() { + + /** + * Generates an image, returning the result directly to the caller. + * + * @param templateId The ID of server prompt template. + * @param inputs the inputs needed to fill in the prompt + */ + public abstract fun generateImages( + templateId: String, + inputs: Map + ): ListenableFuture> + + /** Returns the [TemplateImagenModel] object wrapped by this object. */ + public abstract fun getImageModel(): TemplateImagenModel + + private class FuturesImpl(private val model: TemplateImagenModel) : TemplateImagenModelFutures() { + override fun generateImages( + templateId: String, + inputs: Map + ): ListenableFuture> { + return SuspendToFutureAdapter.launchFuture { model.generateImages(templateId, inputs) } + } + + override fun getImageModel(): TemplateImagenModel { + return model + } + } +} From 60f580b3bc31887e069ed6b0725b56dc6c4ea658 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 22 Oct 2025 10:54:58 -0700 Subject: [PATCH 04/13] add missing template imagen model futures entry point --- firebase-ai/api.txt | 6 ++++++ .../google/firebase/ai/java/TemplateImagenModelFutures.kt | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 17838d5f6ae..b5932bf9b0e 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -192,8 +192,14 @@ package com.google.firebase.ai.java { } public abstract class TemplateImagenModelFutures { + method public static final com.google.firebase.ai.java.TemplateImagenModelFutures from(com.google.firebase.ai.TemplateImagenModel model); method public abstract com.google.common.util.concurrent.ListenableFuture> generateImages(String templateId, java.util.Map inputs); method public abstract com.google.firebase.ai.TemplateImagenModel getImageModel(); + field public static final com.google.firebase.ai.java.TemplateImagenModelFutures.Companion Companion; + } + + public static final class TemplateImagenModelFutures.Companion { + method public com.google.firebase.ai.java.TemplateImagenModelFutures from(com.google.firebase.ai.TemplateImagenModel model); } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt index 0b7b6aa6808..cc177dc41af 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt @@ -19,6 +19,7 @@ package com.google.firebase.ai.java import androidx.concurrent.futures.SuspendToFutureAdapter import com.google.common.util.concurrent.ListenableFuture import com.google.firebase.ai.TemplateImagenModel +import com.google.firebase.ai.java.ImagenModelFutures.FuturesImpl import com.google.firebase.ai.type.ImagenGenerationResponse import com.google.firebase.ai.type.ImagenInlineImage import com.google.firebase.ai.type.PublicPreviewAPI @@ -57,4 +58,10 @@ public abstract class TemplateImagenModelFutures internal constructor() { return model } } + public companion object { + + /** @return a [TemplateImagenModelFutures] created around the provided [TemplateImagenModel] */ + @JvmStatic + public fun from(model: TemplateImagenModel): TemplateImagenModelFutures = FuturesImpl(model) + } } From 1c898197f924c02a854f35d1cad2c38af3501073 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 22 Oct 2025 11:10:32 -0700 Subject: [PATCH 05/13] fixes for comments --- .../main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt | 2 -- .../google/firebase/ai/java/TemplateGenerativeModelFutures.kt | 1 - .../com/google/firebase/ai/java/TemplateImagenModelFutures.kt | 1 - 3 files changed, 4 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt index 80da3b6aaef..522dc7adb51 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt @@ -97,7 +97,5 @@ internal constructor( internal companion object { private val TAG = TemplateImagenModel::class.java.simpleName - internal const val DEFAULT_FILTERED_ERROR = - "Unable to show generated images. All images were filtered out because they violated Vertex AI's usage guidelines. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback." } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateGenerativeModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateGenerativeModelFutures.kt index 08ed69723c0..f1045e3db1d 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateGenerativeModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateGenerativeModelFutures.kt @@ -19,7 +19,6 @@ package com.google.firebase.ai.java import androidx.concurrent.futures.SuspendToFutureAdapter import com.google.common.util.concurrent.ListenableFuture import com.google.firebase.ai.TemplateGenerativeModel -import com.google.firebase.ai.java.ImagenModelFutures.FuturesImpl import com.google.firebase.ai.type.FirebaseAIException import com.google.firebase.ai.type.GenerateContentResponse import com.google.firebase.ai.type.PublicPreviewAPI diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt index cc177dc41af..5a38982fbda 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateImagenModelFutures.kt @@ -19,7 +19,6 @@ package com.google.firebase.ai.java import androidx.concurrent.futures.SuspendToFutureAdapter import com.google.common.util.concurrent.ListenableFuture import com.google.firebase.ai.TemplateImagenModel -import com.google.firebase.ai.java.ImagenModelFutures.FuturesImpl import com.google.firebase.ai.type.ImagenGenerationResponse import com.google.firebase.ai.type.ImagenInlineImage import com.google.firebase.ai.type.PublicPreviewAPI From 7645acf2111d10bc6fb610ff73ae644fa3a308a4 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 22 Oct 2025 12:02:54 -0700 Subject: [PATCH 06/13] fix serialization --- .../com/google/firebase/ai/TemplateGenerativeModel.kt | 7 ++++++- .../com/google/firebase/ai/TemplateImagenModel.kt | 7 ++++++- .../kotlin/com/google/firebase/ai/common/Request.kt | 10 +++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt index b104654e74a..a3045b92396 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt @@ -33,6 +33,9 @@ import com.google.firebase.auth.internal.InternalAuthProvider import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.catch import kotlinx.coroutines.flow.map +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import org.json.JSONObject /** * Represents a multimodal model (like Gemini), capable of generating content based on various @@ -111,7 +114,9 @@ internal constructor( .map { it.toPublic().validate() } internal fun constructRequest(inputs: Map): TemplateGenerateContentRequest { - return TemplateGenerateContentRequest(inputs) + return TemplateGenerateContentRequest( + Json.parseToJsonElement(JSONObject(inputs).toString()).jsonObject + ) } private fun GenerateContentResponse.validate() = apply { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt index 522dc7adb51..e6281f49519 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateImagenModel.kt @@ -27,6 +27,9 @@ import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.RequestOptions import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider import com.google.firebase.auth.internal.InternalAuthProvider +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import org.json.JSONObject /** * Represents a generative model (like Imagen), capable of generating images based a template. @@ -92,7 +95,9 @@ internal constructor( private fun constructTemplateGenerateImageRequest( inputs: Map ): TemplateGenerateImageRequest { - return TemplateGenerateImageRequest(inputs) + return TemplateGenerateImageRequest( + Json.parseToJsonElement(JSONObject(inputs).toString()).jsonObject + ) } internal companion object { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt index ca5f1165563..8e5f2150689 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt @@ -28,10 +28,10 @@ import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.SafetySetting import com.google.firebase.ai.type.Tool import com.google.firebase.ai.type.ToolConfig -import kotlinx.serialization.Contextual import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonObject internal interface Request @@ -46,13 +46,9 @@ internal data class GenerateContentRequest( @SerialName("system_instruction") val systemInstruction: Content.Internal? = null, ) : Request -@Serializable -internal data class TemplateGenerateContentRequest(val inputs: Map) : - Request +@Serializable internal data class TemplateGenerateContentRequest(val inputs: JsonObject) : Request -@Serializable -internal data class TemplateGenerateImageRequest(val inputs: Map) : - Request +@Serializable internal data class TemplateGenerateImageRequest(val inputs: JsonObject) : Request @Serializable internal data class CountTokensRequest( From f761374664779977daff3fba8c88657787aba562 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Thu, 23 Oct 2025 11:07:00 -0700 Subject: [PATCH 07/13] add TemplateChat and TemplateChatFutures --- firebase-ai/api.txt | 25 ++- .../com/google/firebase/ai/TemplateChat.kt | 166 ++++++++++++++++++ .../firebase/ai/TemplateGenerativeModel.kt | 25 ++- .../com/google/firebase/ai/common/Request.kt | 6 +- .../firebase/ai/java/TemplateChatFutures.kt | 98 +++++++++++ 5 files changed, 311 insertions(+), 9 deletions(-) create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateChat.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateChatFutures.kt diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index b5932bf9b0e..3a3a13f4c3d 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -87,9 +87,18 @@ package com.google.firebase.ai { method public suspend Object? connect(kotlin.coroutines.Continuation); } + @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateChat { + ctor public TemplateChat(com.google.firebase.ai.TemplateGenerativeModel model, String templateID, java.util.List history = java.util.ArrayList()); + method public java.util.List getHistory(); + method public suspend Object? sendMessage(com.google.firebase.ai.type.Content prompt, java.util.Map inputs, kotlin.coroutines.Continuation); + method public kotlinx.coroutines.flow.Flow sendMessageStream(com.google.firebase.ai.type.Content prompt, java.util.Map inputs); + property public final java.util.List history; + } + @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateGenerativeModel { - method public suspend Object? generateContent(String templateId, java.util.Map inputs, kotlin.coroutines.Continuation); - method public kotlinx.coroutines.flow.Flow generateContentStream(String templateId, java.util.Map inputs); + method public suspend Object? generateContent(String templateId, java.util.Map inputs, java.util.List? history = null, kotlin.coroutines.Continuation); + method public kotlinx.coroutines.flow.Flow generateContentStream(String templateId, java.util.Map inputs, java.util.List? history = null); + method public com.google.firebase.ai.TemplateChat startChat(String templateId, java.util.List history = emptyList()); } @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateImagenModel { @@ -179,6 +188,18 @@ package com.google.firebase.ai.java { method public com.google.firebase.ai.java.LiveSessionFutures from(com.google.firebase.ai.type.LiveSession session); } + @com.google.firebase.ai.type.PublicPreviewAPI public abstract class TemplateChatFutures { + method public static final com.google.firebase.ai.java.TemplateChatFutures from(com.google.firebase.ai.TemplateChat chat); + method public abstract com.google.firebase.ai.TemplateChat getChat(); + method public abstract com.google.common.util.concurrent.ListenableFuture sendMessage(com.google.firebase.ai.type.Content prompt, java.util.Map inputs); + method public abstract org.reactivestreams.Publisher sendMessageStream(com.google.firebase.ai.type.Content prompt, java.util.Map inputs); + field public static final com.google.firebase.ai.java.TemplateChatFutures.Companion Companion; + } + + public static final class TemplateChatFutures.Companion { + method public com.google.firebase.ai.java.TemplateChatFutures from(com.google.firebase.ai.TemplateChat chat); + } + public abstract class TemplateGenerativeModelFutures { method public static final com.google.firebase.ai.java.TemplateGenerativeModelFutures from(com.google.firebase.ai.TemplateGenerativeModel model); method public abstract com.google.common.util.concurrent.ListenableFuture generateContent(String templateId, java.util.Map inputs); diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateChat.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateChat.kt new file mode 100644 index 00000000000..0f9cd192209 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateChat.kt @@ -0,0 +1,166 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai + +import android.graphics.Bitmap +import com.google.firebase.ai.type.Content +import com.google.firebase.ai.type.GenerateContentResponse +import com.google.firebase.ai.type.ImagePart +import com.google.firebase.ai.type.InlineDataPart +import com.google.firebase.ai.type.InvalidStateException +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.TextPart +import com.google.firebase.ai.type.content +import java.util.LinkedList +import java.util.concurrent.Semaphore +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.onCompletion +import kotlinx.coroutines.flow.onEach + +/** + * Representation of a multi-turn interaction with a model. + * + * Captures and stores the history of communication in memory, and provides it as context with each + * new message. + * + * **Note:** This object is not thread-safe, and calling [sendMessage] multiple times without + * waiting for a response will throw an [InvalidStateException]. + * + * @param model The model to use for the interaction. + * @param templateID The template ID for this chat session + * @property history The previous content from the chat that has been successfully sent and received + * from the model. This will be provided to the model for each message sent (as context for the + * discussion). + */ +@PublicPreviewAPI +public class TemplateChat( + private val model: TemplateGenerativeModel, + private val templateID: String, + public val history: MutableList = ArrayList() +) { + private var lock = Semaphore(1) + + /** + * Sends a message using the provided [prompt]; automatically providing the existing [history] as + * context. + * + * If successful, the message and response will be added to the [history]. If unsuccessful, + * [history] will remain unchanged. + * + * @param prompt The input that, together with the history, will be given to the model as the + * prompt. + * @param inputs the inputs needed to fill in the template ID + * @throws InvalidStateException if [prompt] is not coming from the 'user' role. + * @throws InvalidStateException if the [Chat] instance has an active request. + */ + public suspend fun sendMessage( + prompt: Content, + inputs: Map + ): GenerateContentResponse { + prompt.assertComesFromUser() + attemptLock() + try { + val fullPrompt = history + prompt + val response = model.generateContent(templateID, inputs, fullPrompt) + history.add(prompt) + history.add(response.candidates.first().content) + return response + } finally { + lock.release() + } + } + + /** + * Sends a message using the existing history of this chat as context and the provided [Content] + * prompt. + * + * The response from the model is returned as a stream. + * + * If successful, the message and response will be added to the history. If unsuccessful, history + * will remain unchanged. + * + * @param prompt The input that, together with the history, will be given to the model as the + * prompt. + * @param inputs the inputs needed to fill in the template ID + * @throws InvalidStateException if [prompt] is not coming from the 'user' role. + * @throws InvalidStateException if the [Chat] instance has an active request. + */ + public fun sendMessageStream( + prompt: Content, + inputs: Map + ): Flow { + prompt.assertComesFromUser() + attemptLock() + + val fullPrompt = history + prompt + val flow = model.generateContentStream(templateID, inputs, fullPrompt) + val bitmaps = LinkedList() + val inlineDataParts = LinkedList() + val text = StringBuilder() + + /** + * TODO: revisit when images and inline data are returned. This will cause issues with how + * things are structured in the response. eg; a text/image/text response will be (incorrectly) + * represented as image/text + */ + return flow + .onEach { + for (part in it.candidates.first().content.parts) { + when (part) { + is TextPart -> text.append(part.text) + is ImagePart -> bitmaps.add(part.image) + is InlineDataPart -> inlineDataParts.add(part) + } + } + } + .onCompletion { + lock.release() + if (it == null) { + val content = + content("model") { + for (bitmap in bitmaps) { + image(bitmap) + } + for (inlineDataPart in inlineDataParts) { + inlineData(inlineDataPart.inlineData, inlineDataPart.mimeType) + } + if (text.isNotBlank()) { + text(text.toString()) + } + } + + history.add(prompt) + history.add(content) + } + } + } + + private fun Content.assertComesFromUser() { + if (role !in listOf("user", "function")) { + throw InvalidStateException("Chat prompts should come from the 'user' or 'function' role.") + } + } + + private fun attemptLock() { + if (!lock.tryAcquire()) { + throw InvalidStateException( + "This chat instance currently has an ongoing request, please wait for it to complete " + + "before sending more messages" + ) + } + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt index a3045b92396..aaeea267e0a 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt @@ -20,6 +20,7 @@ import com.google.firebase.FirebaseApp import com.google.firebase.ai.common.APIController import com.google.firebase.ai.common.AppCheckHeaderProvider import com.google.firebase.ai.common.TemplateGenerateContentRequest +import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.FinishReason import com.google.firebase.ai.type.FirebaseAIException import com.google.firebase.ai.type.GenerateContentResponse @@ -78,17 +79,19 @@ internal constructor( * * @param templateId The ID of server prompt template. * @param inputs the inputs needed to fill in the prompt + * @param history history to be passed to the model (for multi-turn generation) * @return The content generated by the model. * @throws [FirebaseAIException] if the request failed. * @see [FirebaseAIException] for types of errors. */ public suspend fun generateContent( templateId: String, - inputs: Map + inputs: Map, + history: List? = null ): GenerateContentResponse = try { controller - .templateGenerateContent("$templateUri$templateId", constructRequest(inputs)) + .templateGenerateContent("$templateUri$templateId", constructRequest(inputs, history)) .toPublic() .validate() } catch (e: Throwable) { @@ -100,22 +103,32 @@ internal constructor( * * @param templateId The ID of server prompt template. * @param inputs the inputs needed to fill in the prompt + * @param history history to be passed to the model (for multi-turn generation) * @return A [Flow] which will emit responses as they are returned by the model. * @throws [FirebaseAIException] if the request failed. * @see [FirebaseAIException] for types of errors. */ public fun generateContentStream( templateId: String, - inputs: Map + inputs: Map, + history: List? = null ): Flow = controller - .templateGenerateContentStream("$templateUri$templateId", constructRequest(inputs)) + .templateGenerateContentStream("$templateUri$templateId", constructRequest(inputs, history)) .catch { throw FirebaseAIException.from(it) } .map { it.toPublic().validate() } - internal fun constructRequest(inputs: Map): TemplateGenerateContentRequest { + /** Creates a [TemplateChat] instance using this model with the optionally provided history. */ + public fun startChat(templateId: String, history: List = emptyList()): TemplateChat = + TemplateChat(this, templateId, history.toMutableList()) + + internal fun constructRequest( + inputs: Map, + history: List? = null + ): TemplateGenerateContentRequest { return TemplateGenerateContentRequest( - Json.parseToJsonElement(JSONObject(inputs).toString()).jsonObject + Json.parseToJsonElement(JSONObject(inputs).toString()).jsonObject, + history?.let { it.map { it.toInternal() } } ) } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt index 8e5f2150689..e3a4afcb56c 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt @@ -46,7 +46,11 @@ internal data class GenerateContentRequest( @SerialName("system_instruction") val systemInstruction: Content.Internal? = null, ) : Request -@Serializable internal data class TemplateGenerateContentRequest(val inputs: JsonObject) : Request +@Serializable +internal data class TemplateGenerateContentRequest( + val inputs: JsonObject, + val history: List? +) : Request @Serializable internal data class TemplateGenerateImageRequest(val inputs: JsonObject) : Request diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateChatFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateChatFutures.kt new file mode 100644 index 00000000000..527e934d641 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateChatFutures.kt @@ -0,0 +1,98 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.ai.java + +import androidx.concurrent.futures.SuspendToFutureAdapter +import com.google.common.util.concurrent.ListenableFuture +import com.google.firebase.ai.TemplateChat +import com.google.firebase.ai.type.Content +import com.google.firebase.ai.type.GenerateContentResponse +import com.google.firebase.ai.type.InvalidStateException +import com.google.firebase.ai.type.PublicPreviewAPI +import kotlinx.coroutines.reactive.asPublisher +import org.reactivestreams.Publisher + +/** + * Wrapper class providing Java compatible methods for [TemplateChat]. + * + * @see [TemplateChat] + */ +@PublicPreviewAPI +public abstract class TemplateChatFutures internal constructor() { + + /** + * Sends a message using the existing history of this chat as context and the provided [Content] + * prompt. + * + * If successful, the message and response will be added to the history. If unsuccessful, history + * will remain unchanged. + * + * @param prompt The input that, together with the history, will be given to the model as the + * prompt. + * @param inputs the inputs needed to fill in the template ID + * @throws InvalidStateException if [prompt] is not coming from the 'user' role + * @throws InvalidStateException if the [TemplateChat] instance has an active request + */ + public abstract fun sendMessage( + prompt: Content, + inputs: Map + ): ListenableFuture + + /** + * Sends a message using the existing history of this chat as context and the provided [Content] + * prompt. + * + * The response from the model is returned as a stream. + * + * If successful, the message and response will be added to the history. If unsuccessful, history + * will remain unchanged. + * + * @param prompt The input that, together with the history, will be given to the model as the + * prompt. + * @param inputs the inputs needed to fill in the template ID + * @throws InvalidStateException if [prompt] is not coming from the 'user' role + * @throws InvalidStateException if the [TemplateChat] instance has an active request + */ + public abstract fun sendMessageStream( + prompt: Content, + inputs: Map + ): Publisher + + /** Returns the [TemplateChat] object wrapped by this object. */ + public abstract fun getChat(): TemplateChat + + private class FuturesImpl(private val chat: TemplateChat) : TemplateChatFutures() { + override fun sendMessage( + prompt: Content, + inputs: Map + ): ListenableFuture = + SuspendToFutureAdapter.launchFuture { chat.sendMessage(prompt, inputs) } + + override fun sendMessageStream( + prompt: Content, + inputs: Map + ): Publisher = chat.sendMessageStream(prompt, inputs).asPublisher() + + override fun getChat(): TemplateChat = chat + } + + public companion object { + + /** @return a [TemplateChatFutures] created around the provided [TemplateChat] */ + @JvmStatic public fun from(chat: TemplateChat): TemplateChatFutures = FuturesImpl(chat) + } +} From a105587da85d8da0719b4383e25cc4f79b64373f Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Fri, 24 Oct 2025 14:56:48 -0700 Subject: [PATCH 08/13] fix for issue with thought in parts sent as history for template chat --- .../firebase/ai/TemplateGenerativeModel.kt | 2 +- .../com/google/firebase/ai/type/Content.kt | 3 ++- .../com/google/firebase/ai/type/Part.kt | 19 ++++++++++--------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt index aaeea267e0a..fb2aedb3359 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt @@ -128,7 +128,7 @@ internal constructor( ): TemplateGenerateContentRequest { return TemplateGenerateContentRequest( Json.parseToJsonElement(JSONObject(inputs).toString()).jsonObject, - history?.let { it.map { it.toInternal() } } + history?.let { it.map { it.toInternal(nullPartThought = true) } } ) } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt index 350d46e9063..8d8e801bebe 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt @@ -85,7 +85,8 @@ constructor(public val role: String? = "user", public val parts: List) { } @OptIn(ExperimentalSerializationApi::class) - internal fun toInternal() = Internal(this.role ?: "user", this.parts.map { it.toInternal() }) + internal fun toInternal(nullPartThought: Boolean = false) = + Internal(this.role ?: "user", this.parts.map { it.toInternal(nullPartThought) }) @ExperimentalSerializationApi @Serializable diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt index 6b578b8e46e..78c0b515fd6 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt @@ -337,13 +337,14 @@ internal object PartSerializer : } } -internal fun Part.toInternal(): InternalPart { +internal fun Part.toInternal(nullThought: Boolean): InternalPart { + val thought = if (nullThought) null else isThought return when (this) { - is TextPart -> TextPart.Internal(text, isThought, thoughtSignature) + is TextPart -> TextPart.Internal(text, thought, thoughtSignature) is ImagePart -> InlineDataPart.Internal( InlineData.Internal("image/jpeg", encodeBitmapToBase64Jpeg(image)), - isThought, + thought, thoughtSignature ) is InlineDataPart -> @@ -352,37 +353,37 @@ internal fun Part.toInternal(): InternalPart { mimeType, android.util.Base64.encodeToString(inlineData, BASE_64_FLAGS) ), - isThought, + thought, thoughtSignature ) is FunctionCallPart -> FunctionCallPart.Internal( FunctionCallPart.Internal.FunctionCall(name, args, id), - isThought, + thought, thoughtSignature ) is FunctionResponsePart -> FunctionResponsePart.Internal( FunctionResponsePart.Internal.FunctionResponse(name, response, id), - isThought, + thought, thoughtSignature ) is FileDataPart -> FileDataPart.Internal( FileDataPart.Internal.FileData(mimeType = mimeType, fileUri = uri), - isThought, + thought, thoughtSignature ) is ExecutableCodePart -> ExecutableCodePart.Internal( ExecutableCodePart.Internal.ExecutableCode(language, code), - isThought, + thought, thoughtSignature ) is CodeExecutionResultPart -> CodeExecutionResultPart.Internal( CodeExecutionResultPart.Internal.CodeExecutionResult(outcome, output), - isThought, + thought, thoughtSignature ) else -> From b9e3c883e226f71bf67609143636a1a9e9147dd9 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 29 Oct 2025 08:23:13 -0700 Subject: [PATCH 09/13] remove template chat --- firebase-ai/api.txt | 21 --- .../com/google/firebase/ai/TemplateChat.kt | 166 ------------------ .../firebase/ai/TemplateGenerativeModel.kt | 4 - .../firebase/ai/java/TemplateChatFutures.kt | 98 ----------- 4 files changed, 289 deletions(-) delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateChat.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateChatFutures.kt diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 3a3a13f4c3d..167528a40c5 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -87,18 +87,9 @@ package com.google.firebase.ai { method public suspend Object? connect(kotlin.coroutines.Continuation); } - @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateChat { - ctor public TemplateChat(com.google.firebase.ai.TemplateGenerativeModel model, String templateID, java.util.List history = java.util.ArrayList()); - method public java.util.List getHistory(); - method public suspend Object? sendMessage(com.google.firebase.ai.type.Content prompt, java.util.Map inputs, kotlin.coroutines.Continuation); - method public kotlinx.coroutines.flow.Flow sendMessageStream(com.google.firebase.ai.type.Content prompt, java.util.Map inputs); - property public final java.util.List history; - } - @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateGenerativeModel { method public suspend Object? generateContent(String templateId, java.util.Map inputs, java.util.List? history = null, kotlin.coroutines.Continuation); method public kotlinx.coroutines.flow.Flow generateContentStream(String templateId, java.util.Map inputs, java.util.List? history = null); - method public com.google.firebase.ai.TemplateChat startChat(String templateId, java.util.List history = emptyList()); } @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateImagenModel { @@ -188,18 +179,6 @@ package com.google.firebase.ai.java { method public com.google.firebase.ai.java.LiveSessionFutures from(com.google.firebase.ai.type.LiveSession session); } - @com.google.firebase.ai.type.PublicPreviewAPI public abstract class TemplateChatFutures { - method public static final com.google.firebase.ai.java.TemplateChatFutures from(com.google.firebase.ai.TemplateChat chat); - method public abstract com.google.firebase.ai.TemplateChat getChat(); - method public abstract com.google.common.util.concurrent.ListenableFuture sendMessage(com.google.firebase.ai.type.Content prompt, java.util.Map inputs); - method public abstract org.reactivestreams.Publisher sendMessageStream(com.google.firebase.ai.type.Content prompt, java.util.Map inputs); - field public static final com.google.firebase.ai.java.TemplateChatFutures.Companion Companion; - } - - public static final class TemplateChatFutures.Companion { - method public com.google.firebase.ai.java.TemplateChatFutures from(com.google.firebase.ai.TemplateChat chat); - } - public abstract class TemplateGenerativeModelFutures { method public static final com.google.firebase.ai.java.TemplateGenerativeModelFutures from(com.google.firebase.ai.TemplateGenerativeModel model); method public abstract com.google.common.util.concurrent.ListenableFuture generateContent(String templateId, java.util.Map inputs); diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateChat.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateChat.kt deleted file mode 100644 index 0f9cd192209..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateChat.kt +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.firebase.ai - -import android.graphics.Bitmap -import com.google.firebase.ai.type.Content -import com.google.firebase.ai.type.GenerateContentResponse -import com.google.firebase.ai.type.ImagePart -import com.google.firebase.ai.type.InlineDataPart -import com.google.firebase.ai.type.InvalidStateException -import com.google.firebase.ai.type.PublicPreviewAPI -import com.google.firebase.ai.type.TextPart -import com.google.firebase.ai.type.content -import java.util.LinkedList -import java.util.concurrent.Semaphore -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.onCompletion -import kotlinx.coroutines.flow.onEach - -/** - * Representation of a multi-turn interaction with a model. - * - * Captures and stores the history of communication in memory, and provides it as context with each - * new message. - * - * **Note:** This object is not thread-safe, and calling [sendMessage] multiple times without - * waiting for a response will throw an [InvalidStateException]. - * - * @param model The model to use for the interaction. - * @param templateID The template ID for this chat session - * @property history The previous content from the chat that has been successfully sent and received - * from the model. This will be provided to the model for each message sent (as context for the - * discussion). - */ -@PublicPreviewAPI -public class TemplateChat( - private val model: TemplateGenerativeModel, - private val templateID: String, - public val history: MutableList = ArrayList() -) { - private var lock = Semaphore(1) - - /** - * Sends a message using the provided [prompt]; automatically providing the existing [history] as - * context. - * - * If successful, the message and response will be added to the [history]. If unsuccessful, - * [history] will remain unchanged. - * - * @param prompt The input that, together with the history, will be given to the model as the - * prompt. - * @param inputs the inputs needed to fill in the template ID - * @throws InvalidStateException if [prompt] is not coming from the 'user' role. - * @throws InvalidStateException if the [Chat] instance has an active request. - */ - public suspend fun sendMessage( - prompt: Content, - inputs: Map - ): GenerateContentResponse { - prompt.assertComesFromUser() - attemptLock() - try { - val fullPrompt = history + prompt - val response = model.generateContent(templateID, inputs, fullPrompt) - history.add(prompt) - history.add(response.candidates.first().content) - return response - } finally { - lock.release() - } - } - - /** - * Sends a message using the existing history of this chat as context and the provided [Content] - * prompt. - * - * The response from the model is returned as a stream. - * - * If successful, the message and response will be added to the history. If unsuccessful, history - * will remain unchanged. - * - * @param prompt The input that, together with the history, will be given to the model as the - * prompt. - * @param inputs the inputs needed to fill in the template ID - * @throws InvalidStateException if [prompt] is not coming from the 'user' role. - * @throws InvalidStateException if the [Chat] instance has an active request. - */ - public fun sendMessageStream( - prompt: Content, - inputs: Map - ): Flow { - prompt.assertComesFromUser() - attemptLock() - - val fullPrompt = history + prompt - val flow = model.generateContentStream(templateID, inputs, fullPrompt) - val bitmaps = LinkedList() - val inlineDataParts = LinkedList() - val text = StringBuilder() - - /** - * TODO: revisit when images and inline data are returned. This will cause issues with how - * things are structured in the response. eg; a text/image/text response will be (incorrectly) - * represented as image/text - */ - return flow - .onEach { - for (part in it.candidates.first().content.parts) { - when (part) { - is TextPart -> text.append(part.text) - is ImagePart -> bitmaps.add(part.image) - is InlineDataPart -> inlineDataParts.add(part) - } - } - } - .onCompletion { - lock.release() - if (it == null) { - val content = - content("model") { - for (bitmap in bitmaps) { - image(bitmap) - } - for (inlineDataPart in inlineDataParts) { - inlineData(inlineDataPart.inlineData, inlineDataPart.mimeType) - } - if (text.isNotBlank()) { - text(text.toString()) - } - } - - history.add(prompt) - history.add(content) - } - } - } - - private fun Content.assertComesFromUser() { - if (role !in listOf("user", "function")) { - throw InvalidStateException("Chat prompts should come from the 'user' or 'function' role.") - } - } - - private fun attemptLock() { - if (!lock.tryAcquire()) { - throw InvalidStateException( - "This chat instance currently has an ongoing request, please wait for it to complete " + - "before sending more messages" - ) - } - } -} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt index fb2aedb3359..e28874f7363 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt @@ -118,10 +118,6 @@ internal constructor( .catch { throw FirebaseAIException.from(it) } .map { it.toPublic().validate() } - /** Creates a [TemplateChat] instance using this model with the optionally provided history. */ - public fun startChat(templateId: String, history: List = emptyList()): TemplateChat = - TemplateChat(this, templateId, history.toMutableList()) - internal fun constructRequest( inputs: Map, history: List? = null diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateChatFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateChatFutures.kt deleted file mode 100644 index 527e934d641..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/TemplateChatFutures.kt +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.firebase.ai.java - -import androidx.concurrent.futures.SuspendToFutureAdapter -import com.google.common.util.concurrent.ListenableFuture -import com.google.firebase.ai.TemplateChat -import com.google.firebase.ai.type.Content -import com.google.firebase.ai.type.GenerateContentResponse -import com.google.firebase.ai.type.InvalidStateException -import com.google.firebase.ai.type.PublicPreviewAPI -import kotlinx.coroutines.reactive.asPublisher -import org.reactivestreams.Publisher - -/** - * Wrapper class providing Java compatible methods for [TemplateChat]. - * - * @see [TemplateChat] - */ -@PublicPreviewAPI -public abstract class TemplateChatFutures internal constructor() { - - /** - * Sends a message using the existing history of this chat as context and the provided [Content] - * prompt. - * - * If successful, the message and response will be added to the history. If unsuccessful, history - * will remain unchanged. - * - * @param prompt The input that, together with the history, will be given to the model as the - * prompt. - * @param inputs the inputs needed to fill in the template ID - * @throws InvalidStateException if [prompt] is not coming from the 'user' role - * @throws InvalidStateException if the [TemplateChat] instance has an active request - */ - public abstract fun sendMessage( - prompt: Content, - inputs: Map - ): ListenableFuture - - /** - * Sends a message using the existing history of this chat as context and the provided [Content] - * prompt. - * - * The response from the model is returned as a stream. - * - * If successful, the message and response will be added to the history. If unsuccessful, history - * will remain unchanged. - * - * @param prompt The input that, together with the history, will be given to the model as the - * prompt. - * @param inputs the inputs needed to fill in the template ID - * @throws InvalidStateException if [prompt] is not coming from the 'user' role - * @throws InvalidStateException if the [TemplateChat] instance has an active request - */ - public abstract fun sendMessageStream( - prompt: Content, - inputs: Map - ): Publisher - - /** Returns the [TemplateChat] object wrapped by this object. */ - public abstract fun getChat(): TemplateChat - - private class FuturesImpl(private val chat: TemplateChat) : TemplateChatFutures() { - override fun sendMessage( - prompt: Content, - inputs: Map - ): ListenableFuture = - SuspendToFutureAdapter.launchFuture { chat.sendMessage(prompt, inputs) } - - override fun sendMessageStream( - prompt: Content, - inputs: Map - ): Publisher = chat.sendMessageStream(prompt, inputs).asPublisher() - - override fun getChat(): TemplateChat = chat - } - - public companion object { - - /** @return a [TemplateChatFutures] created around the provided [TemplateChat] */ - @JvmStatic public fun from(chat: TemplateChat): TemplateChatFutures = FuturesImpl(chat) - } -} From d8ec7cc60de718cc207b706b681520616f79ccc4 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 29 Oct 2025 08:34:01 -0700 Subject: [PATCH 10/13] fixes for comments --- .../google/firebase/ai/TemplateGenerativeModel.kt | 14 +++++++------- .../kotlin/com/google/firebase/ai/type/Content.kt | 7 +++++-- .../kotlin/com/google/firebase/ai/type/Part.kt | 5 ++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt index e28874f7363..2d5bb5c7c46 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt @@ -75,10 +75,10 @@ internal constructor( ) /** - * Generates new content using the given templateId with the given inputs. + * Generates content from a prompt template and inputs. * - * @param templateId The ID of server prompt template. - * @param inputs the inputs needed to fill in the prompt + * @param templateId The ID of the prompt template to use. + * @param inputs A map of variables to substitute into the template. * @param history history to be passed to the model (for multi-turn generation) * @return The content generated by the model. * @throws [FirebaseAIException] if the request failed. @@ -99,10 +99,10 @@ internal constructor( } /** - * Generates new content as a stream using the given templateId with the given inputs. + * Generates content as a stream from a prompt template and inputs. * - * @param templateId The ID of server prompt template. - * @param inputs the inputs needed to fill in the prompt + * @param templateId The ID of the prompt template to use. + * @param inputs A map of variables to substitute into the template. * @param history history to be passed to the model (for multi-turn generation) * @return A [Flow] which will emit responses as they are returned by the model. * @throws [FirebaseAIException] if the request failed. @@ -124,7 +124,7 @@ internal constructor( ): TemplateGenerateContentRequest { return TemplateGenerateContentRequest( Json.parseToJsonElement(JSONObject(inputs).toString()).jsonObject, - history?.let { it.map { it.toInternal(nullPartThought = true) } } + history?.let { it.map { it.toTemplateInternal() } } ) } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt index 8d8e801bebe..d7450df3f3b 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Content.kt @@ -85,8 +85,11 @@ constructor(public val role: String? = "user", public val parts: List) { } @OptIn(ExperimentalSerializationApi::class) - internal fun toInternal(nullPartThought: Boolean = false) = - Internal(this.role ?: "user", this.parts.map { it.toInternal(nullPartThought) }) + internal fun toInternal() = Internal(this.role ?: "user", this.parts.map { it.toInternal() }) + + @OptIn(ExperimentalSerializationApi::class) + internal fun toTemplateInternal() = + Internal(this.role ?: "user", this.parts.map { it.toInternal(true) }) @ExperimentalSerializationApi @Serializable diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt index 78c0b515fd6..8e1702d5cd2 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Part.kt @@ -19,7 +19,6 @@ package com.google.firebase.ai.type import android.graphics.Bitmap import android.graphics.BitmapFactory import android.util.Log -import com.google.firebase.ai.type.ImagenImageFormat.Internal import java.io.ByteArrayOutputStream import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.SerialName @@ -337,8 +336,8 @@ internal object PartSerializer : } } -internal fun Part.toInternal(nullThought: Boolean): InternalPart { - val thought = if (nullThought) null else isThought +internal fun Part.toInternal(ignoreThoughtFlag: Boolean = false): InternalPart { + val thought = if (ignoreThoughtFlag) null else isThought return when (this) { is TextPart -> TextPart.Internal(text, thought, thoughtSignature) is ImagePart -> From aa8949cd02bd57955e95e6320ff5dd6a7285e802 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 29 Oct 2025 08:38:48 -0700 Subject: [PATCH 11/13] add serialization tests --- .../google/firebase/ai/SerializationTests.kt | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt index 476d68261d2..7cdfc253a40 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt @@ -16,6 +16,8 @@ package com.google.firebase.ai +import com.google.firebase.ai.common.TemplateGenerateContentRequest +import com.google.firebase.ai.common.TemplateGenerateImageRequest import com.google.firebase.ai.common.util.descriptorToJson import com.google.firebase.ai.type.Candidate import com.google.firebase.ai.type.CountTokensResponse @@ -512,6 +514,56 @@ internal class SerializationTests { expectedJsonAsString shouldEqualJson actualJson.toString() } + @Test + fun `test template request serialization as Json`() { + val expectedJsonAsString = + """ + { + "id": "TemplateGenerateContentRequest", + "type": "object", + "properties": { + "inputs": { + "type": "object", + "additionalProperties": { + "${"$"}ref": "JsonElement" + } + }, + "history": { + "type": "array", + "items": { + "${"$"}ref": "Content" + } + } + } + } + """ + .trimIndent() + val actualJson = descriptorToJson(TemplateGenerateContentRequest.serializer().descriptor) + expectedJsonAsString shouldEqualJson actualJson.toString() + } + + @Test + fun `test template imagen request serialization as Json`() { + val expectedJsonAsString = + """ + { + "id": "TemplateGenerateImageRequest", + "type": "object", + "properties": { + "inputs": { + "type": "object", + "additionalProperties": { + "${"$"}ref": "JsonElement" + } + } + } + } + """ + .trimIndent() + val actualJson = descriptorToJson(TemplateGenerateImageRequest.serializer().descriptor) + expectedJsonAsString shouldEqualJson actualJson.toString() + } + @Test fun `test GoogleSearch serialization as Json`() { val expectedJsonAsString = From e63f756807e646b24a8ca9d477a7acbc5982e516 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 29 Oct 2025 08:40:07 -0700 Subject: [PATCH 12/13] format --- .../google/firebase/ai/SerializationTests.kt | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt index 7cdfc253a40..215b1eca9eb 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt @@ -514,10 +514,10 @@ internal class SerializationTests { expectedJsonAsString shouldEqualJson actualJson.toString() } - @Test - fun `test template request serialization as Json`() { - val expectedJsonAsString = - """ + @Test + fun `test template request serialization as Json`() { + val expectedJsonAsString = + """ { "id": "TemplateGenerateContentRequest", "type": "object", @@ -537,15 +537,15 @@ internal class SerializationTests { } } """ - .trimIndent() - val actualJson = descriptorToJson(TemplateGenerateContentRequest.serializer().descriptor) - expectedJsonAsString shouldEqualJson actualJson.toString() - } + .trimIndent() + val actualJson = descriptorToJson(TemplateGenerateContentRequest.serializer().descriptor) + expectedJsonAsString shouldEqualJson actualJson.toString() + } - @Test - fun `test template imagen request serialization as Json`() { - val expectedJsonAsString = - """ + @Test + fun `test template imagen request serialization as Json`() { + val expectedJsonAsString = + """ { "id": "TemplateGenerateImageRequest", "type": "object", @@ -559,10 +559,10 @@ internal class SerializationTests { } } """ - .trimIndent() - val actualJson = descriptorToJson(TemplateGenerateImageRequest.serializer().descriptor) - expectedJsonAsString shouldEqualJson actualJson.toString() - } + .trimIndent() + val actualJson = descriptorToJson(TemplateGenerateImageRequest.serializer().descriptor) + expectedJsonAsString shouldEqualJson actualJson.toString() + } @Test fun `test GoogleSearch serialization as Json`() { From ef90dce1c7327306f8cdf6ad4674d65a0404c842 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 29 Oct 2025 13:28:17 -0700 Subject: [PATCH 13/13] remove history param from TemplateGenerativeModel --- firebase-ai/api.txt | 4 ++-- .../com/google/firebase/ai/TemplateGenerativeModel.kt | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 167528a40c5..b5932bf9b0e 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -88,8 +88,8 @@ package com.google.firebase.ai { } @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateGenerativeModel { - method public suspend Object? generateContent(String templateId, java.util.Map inputs, java.util.List? history = null, kotlin.coroutines.Continuation); - method public kotlinx.coroutines.flow.Flow generateContentStream(String templateId, java.util.Map inputs, java.util.List? history = null); + method public suspend Object? generateContent(String templateId, java.util.Map inputs, kotlin.coroutines.Continuation); + method public kotlinx.coroutines.flow.Flow generateContentStream(String templateId, java.util.Map inputs); } @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateImagenModel { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt index 2d5bb5c7c46..8672813fff3 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt @@ -79,7 +79,6 @@ internal constructor( * * @param templateId The ID of the prompt template to use. * @param inputs A map of variables to substitute into the template. - * @param history history to be passed to the model (for multi-turn generation) * @return The content generated by the model. * @throws [FirebaseAIException] if the request failed. * @see [FirebaseAIException] for types of errors. @@ -87,11 +86,10 @@ internal constructor( public suspend fun generateContent( templateId: String, inputs: Map, - history: List? = null ): GenerateContentResponse = try { controller - .templateGenerateContent("$templateUri$templateId", constructRequest(inputs, history)) + .templateGenerateContent("$templateUri$templateId", constructRequest(inputs)) .toPublic() .validate() } catch (e: Throwable) { @@ -103,18 +101,16 @@ internal constructor( * * @param templateId The ID of the prompt template to use. * @param inputs A map of variables to substitute into the template. - * @param history history to be passed to the model (for multi-turn generation) * @return A [Flow] which will emit responses as they are returned by the model. * @throws [FirebaseAIException] if the request failed. * @see [FirebaseAIException] for types of errors. */ public fun generateContentStream( templateId: String, - inputs: Map, - history: List? = null + inputs: Map ): Flow = controller - .templateGenerateContentStream("$templateUri$templateId", constructRequest(inputs, history)) + .templateGenerateContentStream("$templateUri$templateId", constructRequest(inputs)) .catch { throw FirebaseAIException.from(it) } .map { it.toPublic().validate() }