Skip to content

Commit 705d850

Browse files
Fry-kunFry-kun
authored
Merge branch 'main' into custom-tabs
2 parents 5b33dda + 22be120 commit 705d850

33 files changed

+1020
-120
lines changed

‎firebase-vertexai/CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Unreleased
22
* [fixed] Added support for new values sent by the server for `FinishReason` and `BlockReason`.
33
* [changed] Added support for modality-based token count. (#6658)
4+
* [feature] Added support for generating images with Imagen models.
45

56
# 16.1.0
67
* [changed] Internal improvements to correctly handle empty model responses.
@@ -65,3 +66,4 @@
6566
* [feature] Added support for `responseMimeType` in `GenerationConfig`.
6667
* [changed] Renamed `GoogleGenerativeAIException` to `FirebaseVertexAIException`.
6768
* [changed] Updated the KDocs for various classes and functions.
69+

‎firebase-vertexai/api.txt‎

Lines changed: 132 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ package com.google.firebase.vertexai {
2525
method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(com.google.firebase.FirebaseApp app);
2626
method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(com.google.firebase.FirebaseApp app = Firebase.app, String location);
2727
method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(String location);
28+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName);
29+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null);
30+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null);
31+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null, com.google.firebase.vertexai.type.RequestOptions requestOptions = com.google.firebase.vertexai.type.RequestOptions());
2832
property public static final com.google.firebase.vertexai.FirebaseVertexAI instance;
2933
field public static final com.google.firebase.vertexai.FirebaseVertexAI.Companion Companion;
3034
}
@@ -55,6 +59,10 @@ package com.google.firebase.vertexai {
5559
method public com.google.firebase.vertexai.Chat startChat(java.util.List<com.google.firebase.vertexai.type.Content> history = emptyList());
5660
}
5761

62+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenModel {
63+
method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation<? super com.google.firebase.vertexai.type.ImagenGenerationResponse<com.google.firebase.vertexai.type.ImagenInlineImage>>);
64+
}
65+
5866
}
5967

6068
package com.google.firebase.vertexai.java {
@@ -86,6 +94,17 @@ package com.google.firebase.vertexai.java {
8694
method public com.google.firebase.vertexai.java.GenerativeModelFutures from(com.google.firebase.vertexai.GenerativeModel model);
8795
}
8896

97+
@com.google.firebase.vertexai.type.PublicPreviewAPI public abstract class ImagenModelFutures {
98+
method public static final com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model);
99+
method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.vertexai.type.ImagenGenerationResponse<com.google.firebase.vertexai.type.ImagenInlineImage>> generateImages(String prompt);
100+
method public abstract com.google.firebase.vertexai.ImagenModel getImageModel();
101+
field public static final com.google.firebase.vertexai.java.ImagenModelFutures.Companion Companion;
102+
}
103+
104+
public static final class ImagenModelFutures.Companion {
105+
method public com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model);
106+
}
107+
89108
}
90109

91110
package com.google.firebase.vertexai.type {
@@ -163,6 +182,9 @@ package com.google.firebase.vertexai.type {
163182
property public final String? role;
164183
}
165184

185+
public final class ContentBlockedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
186+
}
187+
166188
public final class ContentKt {
167189
method public static com.google.firebase.vertexai.type.Content content(String? role = "user", kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.Content.Builder,kotlin.Unit> init);
168190
}
@@ -376,6 +398,111 @@ package com.google.firebase.vertexai.type {
376398
property public final android.graphics.Bitmap image;
377399
}
378400

401+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenAspectRatio {
402+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio.Companion Companion;
403+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio LANDSCAPE_16x9;
404+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio LANDSCAPE_4x3;
405+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio PORTRAIT_3x4;
406+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio PORTRAIT_9x16;
407+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio SQUARE_1x1;
408+
}
409+
410+
public static final class ImagenAspectRatio.Companion {
411+
}
412+
413+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenGenerationConfig {
414+
ctor public ImagenGenerationConfig(String? negativePrompt = null, Integer? numberOfImages = 1, com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio = null, com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat = null, Boolean? addWatermark = null);
415+
method public Boolean? getAddWatermark();
416+
method public com.google.firebase.vertexai.type.ImagenAspectRatio? getAspectRatio();
417+
method public com.google.firebase.vertexai.type.ImagenImageFormat? getImageFormat();
418+
method public String? getNegativePrompt();
419+
method public Integer? getNumberOfImages();
420+
property public final Boolean? addWatermark;
421+
property public final com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio;
422+
property public final com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat;
423+
property public final String? negativePrompt;
424+
property public final Integer? numberOfImages;
425+
field public static final com.google.firebase.vertexai.type.ImagenGenerationConfig.Companion Companion;
426+
}
427+
428+
public static final class ImagenGenerationConfig.Builder {
429+
ctor public ImagenGenerationConfig.Builder();
430+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig build();
431+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder setAddWatermark(boolean addWatermark);
432+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder setAspectRatio(com.google.firebase.vertexai.type.ImagenAspectRatio aspectRatio);
433+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder setImageFormat(com.google.firebase.vertexai.type.ImagenImageFormat imageFormat);
434+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder setNegativePrompt(String negativePrompt);
435+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder setNumberOfImages(int numberOfImages);
436+
field public Boolean? addWatermark;
437+
field public com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio;
438+
field public com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat;
439+
field public String? negativePrompt;
440+
field public Integer? numberOfImages;
441+
}
442+
443+
public static final class ImagenGenerationConfig.Companion {
444+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder builder();
445+
}
446+
447+
public final class ImagenGenerationConfigKt {
448+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public static com.google.firebase.vertexai.type.ImagenGenerationConfig imagenGenerationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder,kotlin.Unit> init);
449+
}
450+
451+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenGenerationResponse<T> {
452+
method public String? getFilteredReason();
453+
method public java.util.List<T> getImages();
454+
property public final String? filteredReason;
455+
property public final java.util.List<T> images;
456+
}
457+
458+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenImageFormat {
459+
method public Integer? getCompressionQuality();
460+
method public String getMimeType();
461+
method public static com.google.firebase.vertexai.type.ImagenImageFormat jpeg(Integer? compressionQuality = null);
462+
method public static com.google.firebase.vertexai.type.ImagenImageFormat png();
463+
property public final Integer? compressionQuality;
464+
property public final String mimeType;
465+
field public static final com.google.firebase.vertexai.type.ImagenImageFormat.Companion Companion;
466+
}
467+
468+
public static final class ImagenImageFormat.Companion {
469+
method public com.google.firebase.vertexai.type.ImagenImageFormat jpeg(Integer? compressionQuality = null);
470+
method public com.google.firebase.vertexai.type.ImagenImageFormat png();
471+
}
472+
473+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenInlineImage {
474+
method public android.graphics.Bitmap asBitmap();
475+
method public byte[] getData();
476+
method public String getMimeType();
477+
property public final byte[] data;
478+
property public final String mimeType;
479+
}
480+
481+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {
482+
field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel ALLOW_ADULT;
483+
field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel ALLOW_ALL;
484+
field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel BLOCK_ALL;
485+
field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel.Companion Companion;
486+
}
487+
488+
public static final class ImagenPersonFilterLevel.Companion {
489+
}
490+
491+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenSafetyFilterLevel {
492+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_LOW_AND_ABOVE;
493+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_MEDIUM_AND_ABOVE;
494+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_NONE;
495+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_ONLY_HIGH;
496+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel.Companion Companion;
497+
}
498+
499+
public static final class ImagenSafetyFilterLevel.Companion {
500+
}
501+
502+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenSafetySettings {
503+
ctor public ImagenSafetySettings(com.google.firebase.vertexai.type.ImagenSafetyFilterLevel safetyFilterLevel, com.google.firebase.vertexai.type.ImagenPersonFilterLevel personFilterLevel);
504+
}
505+
379506
public final class InlineDataPart implements com.google.firebase.vertexai.type.Part {
380507
ctor public InlineDataPart(byte[] inlineData, String mimeType);
381508
method public byte[] getInlineData();
@@ -413,8 +540,8 @@ package com.google.firebase.vertexai.type {
413540
}
414541

415542
public final class PromptBlockedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
416-
method public com.google.firebase.vertexai.type.GenerateContentResponse getResponse();
417-
property public final com.google.firebase.vertexai.type.GenerateContentResponse response;
543+
method public com.google.firebase.vertexai.type.GenerateContentResponse? getResponse();
544+
property public final com.google.firebase.vertexai.type.GenerateContentResponse? response;
418545
}
419546

420547
public final class PromptFeedback {
@@ -427,6 +554,9 @@ package com.google.firebase.vertexai.type {
427554
property public final java.util.List<com.google.firebase.vertexai.type.SafetyRating> safetyRatings;
428555
}
429556

557+
@kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) public @interface PublicPreviewAPI {
558+
}
559+
430560
public final class RequestOptions {
431561
ctor public RequestOptions();
432562
ctor public RequestOptions(long timeoutInMillis = 180.seconds.inWholeMilliseconds);

‎firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/FirebaseVertexAI.kt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ import com.google.firebase.auth.internal.InternalAuthProvider
2424
import com.google.firebase.inject.Provider
2525
import com.google.firebase.vertexai.type.Content
2626
import com.google.firebase.vertexai.type.GenerationConfig
27+
import com.google.firebase.vertexai.type.ImagenGenerationConfig
28+
import com.google.firebase.vertexai.type.ImagenSafetySettings
2729
import com.google.firebase.vertexai.type.InvalidLocationException
30+
import com.google.firebase.vertexai.type.PublicPreviewAPI
2831
import com.google.firebase.vertexai.type.RequestOptions
2932
import com.google.firebase.vertexai.type.SafetySetting
3033
import com.google.firebase.vertexai.type.Tool
@@ -79,6 +82,37 @@ internal constructor(
7982
)
8083
}
8184

85+
/**
86+
* Instantiates a new [ImagenModel] given the provided parameters.
87+
*
88+
* @param modelName The name of the model to use, for example `"imagen-3.0-generate-001"`.
89+
* @param generationConfig The configuration parameters to use for image generation.
90+
* @param safetySettings The safety bounds the model will abide by during image generation.
91+
* @param requestOptions Configuration options for sending requests to the backend.
92+
* @return The initialized [ImagenModel] instance.
93+
*/
94+
@JvmOverloads
95+
@PublicPreviewAPI
96+
public fun imagenModel(
97+
modelName: String,
98+
generationConfig: ImagenGenerationConfig? = null,
99+
safetySettings: ImagenSafetySettings? = null,
100+
requestOptions: RequestOptions = RequestOptions(),
101+
): ImagenModel {
102+
if (location.trim().isEmpty() || location.contains("/")) {
103+
throw InvalidLocationException(location)
104+
}
105+
return ImagenModel(
106+
"projects/${firebaseApp.options.projectId}/locations/${location}/publishers/google/models/${modelName}",
107+
firebaseApp.options.apiKey,
108+
generationConfig,
109+
safetySettings,
110+
requestOptions,
111+
appCheckProvider.get(),
112+
internalAuthProvider.get(),
113+
)
114+
}
115+
82116
public companion object {
83117
/** The [FirebaseVertexAI] instance for the default [FirebaseApp] */
84118
@JvmStatic

‎firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/GenerativeModel.kt‎

Lines changed: 7 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,12 @@
1717
package com.google.firebase.vertexai
1818

1919
import android.graphics.Bitmap
20-
import android.util.Log
2120
import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider
2221
import com.google.firebase.auth.internal.InternalAuthProvider
2322
import com.google.firebase.vertexai.common.APIController
23+
import com.google.firebase.vertexai.common.AppCheckHeaderProvider
2424
import com.google.firebase.vertexai.common.CountTokensRequest
2525
import com.google.firebase.vertexai.common.GenerateContentRequest
26-
import com.google.firebase.vertexai.common.HeaderProvider
2726
import com.google.firebase.vertexai.type.Content
2827
import com.google.firebase.vertexai.type.CountTokensResponse
2928
import com.google.firebase.vertexai.type.FinishReason
@@ -38,12 +37,10 @@ import com.google.firebase.vertexai.type.SerializationException
3837
import com.google.firebase.vertexai.type.Tool
3938
import com.google.firebase.vertexai.type.ToolConfig
4039
import com.google.firebase.vertexai.type.content
41-
import kotlin.time.Duration
42-
import kotlin.time.Duration.Companion.seconds
4340
import kotlinx.coroutines.flow.Flow
4441
import kotlinx.coroutines.flow.catch
4542
import kotlinx.coroutines.flow.map
46-
import kotlinx.coroutines.tasks.await
43+
import kotlinx.serialization.ExperimentalSerializationApi
4744

4845
/**
4946
* Represents a multimodal model (like Gemini), capable of generating content based on various input
@@ -57,10 +54,8 @@ internal constructor(
5754
private val tools: List<Tool>? = null,
5855
private val toolConfig: ToolConfig? = null,
5956
private val systemInstruction: Content? = null,
60-
private val controller: APIController
57+
private val controller: APIController,
6158
) {
62-
63-
@JvmOverloads
6459
internal constructor(
6560
modelName: String,
6661
apiKey: String,
@@ -84,42 +79,8 @@ internal constructor(
8479
modelName,
8580
requestOptions,
8681
"gl-kotlin/${KotlinVersion.CURRENT} fire/${BuildConfig.VERSION_NAME}",
87-
object : HeaderProvider {
88-
override val timeout: Duration
89-
get() = 10.seconds
90-
91-
override suspend fun generateHeaders(): Map<String, String> {
92-
val headers = mutableMapOf<String, String>()
93-
if (appCheckTokenProvider == null) {
94-
Log.w(TAG, "AppCheck not registered, skipping")
95-
} else {
96-
val token = appCheckTokenProvider.getToken(false).await()
97-
98-
if (token.error != null) {
99-
Log.w(TAG, "Error obtaining AppCheck token", token.error)
100-
}
101-
// The Firebase App Check backend can differentiate between apps without App Check, and
102-
// wrongly configured apps by verifying the value of the token, so it always needs to be
103-
// included.
104-
headers["X-Firebase-AppCheck"] = token.token
105-
}
106-
107-
if (internalAuthProvider == null) {
108-
Log.w(TAG, "Auth not registered, skipping")
109-
} else {
110-
try {
111-
val token = internalAuthProvider.getAccessToken(false).await()
112-
113-
headers["Authorization"] = "Firebase ${token.token!!}"
114-
} catch (e: Exception) {
115-
Log.w(TAG, "Error getting Auth token ", e)
116-
}
117-
}
118-
119-
return headers
120-
}
121-
}
122-
)
82+
AppCheckHeaderProvider(TAG, appCheckTokenProvider, internalAuthProvider),
83+
),
12384
)
12485

12586
/**
@@ -239,6 +200,7 @@ internal constructor(
239200
return countTokens(content { image(prompt) })
240201
}
241202

203+
@OptIn(ExperimentalSerializationApi::class)
242204
private fun constructRequest(vararg prompt: Content) =
243205
GenerateContentRequest(
244206
modelName,
@@ -247,7 +209,7 @@ internal constructor(
247209
generationConfig?.toInternal(),
248210
tools?.map { it.toInternal() },
249211
toolConfig?.toInternal(),
250-
systemInstruction?.copy(role = "system")?.toInternal()
212+
systemInstruction?.copy(role = "system")?.toInternal(),
251213
)
252214

253215
private fun constructCountTokensRequest(vararg prompt: Content) =

0 commit comments

Comments
 (0)