From 8319ae9cce7f159157ed7fbf26d46a981fd0443e Mon Sep 17 00:00:00 2001 From: gsiddh <92327772+gsiddh@users.noreply.github.com> Date: Tue, 15 Apr 2025 15:35:13 -0700 Subject: [PATCH 1/3] Vaihi add langmodel types. (#8927) * Adding LanguageModel types. These are based off https://github.com/webmachinelearning/prompt-api?tab=readme-ov-file#full-api-surface-in-web-idl * Adding LanguageModel types. * Remove bunch of exports * yarn formatted * after lint --- packages/vertexai/src/types/language-model.ts | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 packages/vertexai/src/types/language-model.ts diff --git a/packages/vertexai/src/types/language-model.ts b/packages/vertexai/src/types/language-model.ts new file mode 100644 index 00000000000..5bfb38beea4 --- /dev/null +++ b/packages/vertexai/src/types/language-model.ts @@ -0,0 +1,83 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface LanguageModel extends EventTarget { + create(options?: LanguageModelCreateOptions): Promise; + availability(options?: LanguageModelCreateCoreOptions): Promise; + prompt( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): Promise; + promptStreaming( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): ReadableStream; + measureInputUsage( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): Promise; + destroy(): undefined; +} +enum Availability { + 'unavailable', + 'downloadable', + 'downloading', + 'available' +} +interface LanguageModelCreateCoreOptions { + topK?: number; + temperature?: number; + expectedInputs?: LanguageModelExpectedInput[]; +} +interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions { + signal?: AbortSignal; + systemPrompt?: string; + initialPrompts?: LanguageModelInitialPrompts; +} +interface LanguageModelPromptOptions { + signal?: AbortSignal; +} +interface LanguageModelExpectedInput { + type: LanguageModelMessageType; + languages?: string[]; +} +type LanguageModelPrompt = + | LanguageModelMessage[] + | LanguageModelMessageShorthand[] + | string; +type LanguageModelInitialPrompts = + | LanguageModelMessage[] + | LanguageModelMessageShorthand[]; +interface LanguageModelMessage { + role: LanguageModelMessageRole; + content: LanguageModelMessageContent[]; +} +interface LanguageModelMessageShorthand { + role: LanguageModelMessageRole; + content: string; +} +interface LanguageModelMessageContent { + type: LanguageModelMessageType; + content: LanguageModelMessageContentValue; +} +type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; +type LanguageModelMessageType = 'text' | 'image' | 'audio'; +type LanguageModelMessageContentValue = + | ImageBitmapSource + | AudioBuffer + | BufferSource + | string; From 40f9882cda241143174ae8ea92c0c13984daf1bb Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Mon, 24 Mar 2025 14:12:35 -0700 Subject: [PATCH 2/3] Define HybridParams --- common/api-review/vertexai.api.md | 24 ++++++++- docs-devsite/_toc.yaml | 2 + docs-devsite/vertexai.hybridparams.md | 51 +++++++++++++++++++ docs-devsite/vertexai.md | 32 +++++++++--- docs-devsite/vertexai.modelparams.md | 2 +- docs-devsite/vertexai.requestoptions.md | 2 +- packages/vertexai/src/api.ts | 18 +++++-- packages/vertexai/src/types/enums.ts | 10 ++++ packages/vertexai/src/types/language-model.ts | 5 +- packages/vertexai/src/types/requests.ts | 14 ++++- 10 files changed, 145 insertions(+), 15 deletions(-) create mode 100644 docs-devsite/vertexai.hybridparams.md diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index e7f00c2f4e0..fc7d9182586 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -344,7 +344,7 @@ export class GenerativeModel extends VertexAIModel { } // @public -export function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +export function getGenerativeModel(vertexAI: VertexAI, onCloudOrHybridParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel; // @beta export function getImagenModel(vertexAI: VertexAI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; @@ -416,6 +416,18 @@ export enum HarmSeverity { HARM_SEVERITY_NEGLIGIBLE = "HARM_SEVERITY_NEGLIGIBLE" } +// @public +export interface HybridParams { + // (undocumented) + mode?: InferenceMode; + // (undocumented) + onCloudParams?: ModelParams; + // Warning: (ae-forgotten-export) The symbol "LanguageModelCreateOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + onDeviceParams?: LanguageModelCreateOptions; +} + // @beta export enum ImagenAspectRatio { LANDSCAPE_16x9 = "16:9", @@ -500,6 +512,16 @@ export interface ImagenSafetySettings { safetyFilterLevel?: ImagenSafetyFilterLevel; } +// @public +export enum InferenceMode { + // (undocumented) + ONLY_ON_CLOUD = "ONLY_ON_CLOUD", + // (undocumented) + ONLY_ON_DEVICE = "ONLY_ON_DEVICE", + // (undocumented) + PREFER_ON_DEVICE = "PREFER_ON_DEVICE" +} + // @public export interface InlineDataPart { // (undocumented) diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 665222edb9d..64e24534590 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -536,6 +536,8 @@ toc: path: /docs/reference/js/vertexai.groundingattribution.md - title: GroundingMetadata path: /docs/reference/js/vertexai.groundingmetadata.md + - title: HybridParams + path: /docs/reference/js/vertexai.hybridparams.md - title: ImagenGCSImage path: /docs/reference/js/vertexai.imagengcsimage.md - title: ImagenGenerationConfig diff --git a/docs-devsite/vertexai.hybridparams.md b/docs-devsite/vertexai.hybridparams.md new file mode 100644 index 00000000000..ea175a7234b --- /dev/null +++ b/docs-devsite/vertexai.hybridparams.md @@ -0,0 +1,51 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# HybridParams interface +Configures on-device and on-cloud inference. + +Signature: + +```typescript +export interface HybridParams +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [mode](./vertexai.hybridparams.md#hybridparamsmode) | [InferenceMode](./vertexai.md#inferencemode) | | +| [onCloudParams](./vertexai.hybridparams.md#hybridparamsoncloudparams) | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | | +| [onDeviceParams](./vertexai.hybridparams.md#hybridparamsondeviceparams) | LanguageModelCreateOptions | | + +## HybridParams.mode + +Signature: + +```typescript +mode?: InferenceMode; +``` + +## HybridParams.onCloudParams + +Signature: + +```typescript +onCloudParams?: ModelParams; +``` + +## HybridParams.onDeviceParams + +Signature: + +```typescript +onDeviceParams?: LanguageModelCreateOptions; +``` diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index f67254eef20..a3ba28ad609 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -19,7 +19,7 @@ The Vertex AI in Firebase Web SDK. | function(app, ...) | | [getVertexAI(app, options)](./vertexai.md#getvertexai_04094cf) | Returns a [VertexAI](./vertexai.vertexai.md#vertexai_interface) instance for the given app. | | function(vertexAI, ...) | -| [getGenerativeModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getgenerativemodel_e3037c9) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | +| [getGenerativeModel(vertexAI, onCloudOrHybridParams, requestOptions)](./vertexai.md#getgenerativemodel_202434f) | Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | | [getImagenModel(vertexAI, modelParams, requestOptions)](./vertexai.md#getimagenmodel_812c375) | (Public Preview) Returns an [ImagenModel](./vertexai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen.Only Imagen 3 models (named imagen-3.0-*) are supported. | ## Classes @@ -55,6 +55,7 @@ The Vertex AI in Firebase Web SDK. | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface).See the the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | +| [InferenceMode](./vertexai.md#inferencemode) | Determines whether inference happens on-device or on-cloud. | | [Modality](./vertexai.md#modality) | Content part modality. | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -91,6 +92,7 @@ The Vertex AI in Firebase Web SDK. | [GenerativeContentBlob](./vertexai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./vertexai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./vertexai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [HybridParams](./vertexai.hybridparams.md#hybridparams_interface) | Configures on-device and on-cloud inference. | | [ImagenGCSImage](./vertexai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./vertexai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | @@ -99,10 +101,10 @@ The Vertex AI in Firebase Web SDK. | [ImagenSafetySettings](./vertexai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | | [InlineDataPart](./vertexai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModalityTokenCount](./vertexai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | -| [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | +| [ModelParams](./vertexai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_202434f). | | [ObjectSchemaInterface](./vertexai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./vertexai.objectschema.md#objectschema_class) class. | | [PromptFeedback](./vertexai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | -| [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). | +| [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_202434f). | | [RetrievedContextAttribution](./vertexai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | | [SafetyRating](./vertexai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./vertexai.generatecontentcandidate.md#generatecontentcandidate_interface) | | [SafetySetting](./vertexai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | @@ -160,14 +162,14 @@ export declare function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions ## function(vertexAI, ...) -### getGenerativeModel(vertexAI, modelParams, requestOptions) {:#getgenerativemodel_e3037c9} +### getGenerativeModel(vertexAI, onCloudOrHybridParams, requestOptions) {:#getgenerativemodel_202434f} Returns a [GenerativeModel](./vertexai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. Signature: ```typescript -export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +export declare function getGenerativeModel(vertexAI: VertexAI, onCloudOrHybridParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel; ``` #### Parameters @@ -175,7 +177,7 @@ export declare function getGenerativeModel(vertexAI: VertexAI, modelParams: Mode | Parameter | Type | Description | | --- | --- | --- | | vertexAI | [VertexAI](./vertexai.vertexai.md#vertexai_interface) | | -| modelParams | [ModelParams](./vertexai.modelparams.md#modelparams_interface) | | +| onCloudOrHybridParams | [ModelParams](./vertexai.modelparams.md#modelparams_interface) \| [HybridParams](./vertexai.hybridparams.md#hybridparams_interface) | | | requestOptions | [RequestOptions](./vertexai.requestoptions.md#requestoptions_interface) | | Returns: @@ -489,6 +491,24 @@ export declare enum ImagenSafetyFilterLevel | BLOCK\_NONE | "block_none" | (Public Preview) The least aggressive filtering level; blocks very few sensitive prompts and responses.Access to this feature is restricted and may require your case to be reviewed and approved by Cloud support. | | BLOCK\_ONLY\_HIGH | "block_only_high" | (Public Preview) Blocks few sensitive prompts and responses. | +## InferenceMode + +Determines whether inference happens on-device or on-cloud. + +Signature: + +```typescript +export declare enum InferenceMode +``` + +## Enumeration Members + +| Member | Value | Description | +| --- | --- | --- | +| ONLY\_ON\_CLOUD | "ONLY_ON_CLOUD" | | +| ONLY\_ON\_DEVICE | "ONLY_ON_DEVICE" | | +| PREFER\_ON\_DEVICE | "PREFER_ON_DEVICE" | | + ## Modality Content part modality. diff --git a/docs-devsite/vertexai.modelparams.md b/docs-devsite/vertexai.modelparams.md index d3963d240eb..6645d498d8e 100644 --- a/docs-devsite/vertexai.modelparams.md +++ b/docs-devsite/vertexai.modelparams.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ModelParams interface -Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). +Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_202434f). Signature: diff --git a/docs-devsite/vertexai.requestoptions.md b/docs-devsite/vertexai.requestoptions.md index dcd0c552ecb..334ce7956d6 100644 --- a/docs-devsite/vertexai.requestoptions.md +++ b/docs-devsite/vertexai.requestoptions.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # RequestOptions interface -Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_e3037c9). +Params passed to [getGenerativeModel()](./vertexai.md#getgenerativemodel_202434f). Signature: diff --git a/packages/vertexai/src/api.ts b/packages/vertexai/src/api.ts index 7843a5bdeee..323cfd10e80 100644 --- a/packages/vertexai/src/api.ts +++ b/packages/vertexai/src/api.ts @@ -23,6 +23,7 @@ import { VertexAIService } from './service'; import { VertexAI, VertexAIOptions } from './public-types'; import { ImagenModelParams, + HybridParams, ModelParams, RequestOptions, VertexAIErrorCode @@ -70,16 +71,27 @@ export function getVertexAI( */ export function getGenerativeModel( vertexAI: VertexAI, - modelParams: ModelParams, + onCloudOrHybridParams: ModelParams | HybridParams, requestOptions?: RequestOptions ): GenerativeModel { - if (!modelParams.model) { + // Disambiguates onCloudOrHybridParams input. + const hybridParams = onCloudOrHybridParams as HybridParams; + let onCloudParams: ModelParams; + if (hybridParams.mode) { + onCloudParams = hybridParams.onCloudParams || { + model: 'gemini-2.0-flash-lite' + }; + } else { + onCloudParams = onCloudOrHybridParams as ModelParams; + } + + if (!onCloudParams.model) { throw new VertexAIError( VertexAIErrorCode.NO_MODEL, `Must provide a model name. Example: getGenerativeModel({ model: 'my-model-name' })` ); } - return new GenerativeModel(vertexAI, modelParams, requestOptions); + return new GenerativeModel(vertexAI, onCloudParams, requestOptions); } /** diff --git a/packages/vertexai/src/types/enums.ts b/packages/vertexai/src/types/enums.ts index a9481d40f5f..1f81ed79a8f 100644 --- a/packages/vertexai/src/types/enums.ts +++ b/packages/vertexai/src/types/enums.ts @@ -240,3 +240,13 @@ export enum Modality { */ DOCUMENT = 'DOCUMENT' } + +/** + * Determines whether inference happens on-device or on-cloud. + * @public + */ +export enum InferenceMode { + PREFER_ON_DEVICE = 'PREFER_ON_DEVICE', + ONLY_ON_DEVICE = 'ONLY_ON_DEVICE', + ONLY_ON_CLOUD = 'ONLY_ON_CLOUD' +} diff --git a/packages/vertexai/src/types/language-model.ts b/packages/vertexai/src/types/language-model.ts index 5bfb38beea4..e564ca467b4 100644 --- a/packages/vertexai/src/types/language-model.ts +++ b/packages/vertexai/src/types/language-model.ts @@ -38,12 +38,13 @@ enum Availability { 'downloading', 'available' } -interface LanguageModelCreateCoreOptions { +export interface LanguageModelCreateCoreOptions { topK?: number; temperature?: number; expectedInputs?: LanguageModelExpectedInput[]; } -interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions { +export interface LanguageModelCreateOptions + extends LanguageModelCreateCoreOptions { signal?: AbortSignal; systemPrompt?: string; initialPrompts?: LanguageModelInitialPrompts; diff --git a/packages/vertexai/src/types/requests.ts b/packages/vertexai/src/types/requests.ts index c15258b06d0..c643fcd6084 100644 --- a/packages/vertexai/src/types/requests.ts +++ b/packages/vertexai/src/types/requests.ts @@ -17,11 +17,13 @@ import { TypedSchema } from '../requests/schema-builder'; import { Content, Part } from './content'; +import { LanguageModelCreateOptions } from './language-model'; import { FunctionCallingMode, HarmBlockMethod, HarmBlockThreshold, - HarmCategory + HarmCategory, + InferenceMode } from './enums'; import { ObjectSchemaInterface, SchemaRequest } from './schema'; @@ -213,3 +215,13 @@ export interface FunctionCallingConfig { mode?: FunctionCallingMode; allowedFunctionNames?: string[]; } + +/** + * Configures on-device and on-cloud inference. + * @public + */ +export interface HybridParams { + mode?: InferenceMode; + onDeviceParams?: LanguageModelCreateOptions; + onCloudParams?: ModelParams; +} From 7b14419095187a294b9fa5f24ada43277e95f37e Mon Sep 17 00:00:00 2001 From: Siddharth Gupta Date: Wed, 16 Apr 2025 14:24:52 -0700 Subject: [PATCH 3/3] Moving to a string type for inference mode --- common/api-review/vertexai.api.md | 9 +------- docs-devsite/vertexai.md | 30 +++++++++---------------- packages/vertexai/src/types/enums.ts | 10 --------- packages/vertexai/src/types/requests.ts | 11 +++++++-- 4 files changed, 21 insertions(+), 39 deletions(-) diff --git a/common/api-review/vertexai.api.md b/common/api-review/vertexai.api.md index fc7d9182586..8ac48b8d3d2 100644 --- a/common/api-review/vertexai.api.md +++ b/common/api-review/vertexai.api.md @@ -513,14 +513,7 @@ export interface ImagenSafetySettings { } // @public -export enum InferenceMode { - // (undocumented) - ONLY_ON_CLOUD = "ONLY_ON_CLOUD", - // (undocumented) - ONLY_ON_DEVICE = "ONLY_ON_DEVICE", - // (undocumented) - PREFER_ON_DEVICE = "PREFER_ON_DEVICE" -} +export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; // @public export interface InlineDataPart { diff --git a/docs-devsite/vertexai.md b/docs-devsite/vertexai.md index a3ba28ad609..3f77f3cd163 100644 --- a/docs-devsite/vertexai.md +++ b/docs-devsite/vertexai.md @@ -55,7 +55,6 @@ The Vertex AI in Firebase Web SDK. | [ImagenAspectRatio](./vertexai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./vertexai.imagengenerationconfig.md#imagengenerationconfig_interface).See the the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./vertexai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./vertexai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | -| [InferenceMode](./vertexai.md#inferencemode) | Determines whether inference happens on-device or on-cloud. | | [Modality](./vertexai.md#modality) | Content part modality. | | [SchemaType](./vertexai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [VertexAIErrorCode](./vertexai.md#vertexaierrorcode) | Standardized error codes that [VertexAIError](./vertexai.vertexaierror.md#vertexaierror_class) can have. | @@ -132,6 +131,7 @@ The Vertex AI in Firebase Web SDK. | Type Alias | Description | | --- | --- | +| [InferenceMode](./vertexai.md#inferencemode) | Determines whether inference happens on-device or in-cloud. | | [Part](./vertexai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [Role](./vertexai.md#role) | Role is the producer of the content. | | [Tool](./vertexai.md#tool) | Defines a tool that model can call to access external knowledge. | @@ -225,6 +225,16 @@ Possible roles. POSSIBLE_ROLES: readonly ["user", "model", "function", "system"] ``` +## InferenceMode + +Determines whether inference happens on-device or in-cloud. + +Signature: + +```typescript +export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; +``` + ## Part Content part - includes text, image/video, or function call/response part types. @@ -491,24 +501,6 @@ export declare enum ImagenSafetyFilterLevel | BLOCK\_NONE | "block_none" | (Public Preview) The least aggressive filtering level; blocks very few sensitive prompts and responses.Access to this feature is restricted and may require your case to be reviewed and approved by Cloud support. | | BLOCK\_ONLY\_HIGH | "block_only_high" | (Public Preview) Blocks few sensitive prompts and responses. | -## InferenceMode - -Determines whether inference happens on-device or on-cloud. - -Signature: - -```typescript -export declare enum InferenceMode -``` - -## Enumeration Members - -| Member | Value | Description | -| --- | --- | --- | -| ONLY\_ON\_CLOUD | "ONLY_ON_CLOUD" | | -| ONLY\_ON\_DEVICE | "ONLY_ON_DEVICE" | | -| PREFER\_ON\_DEVICE | "PREFER_ON_DEVICE" | | - ## Modality Content part modality. diff --git a/packages/vertexai/src/types/enums.ts b/packages/vertexai/src/types/enums.ts index 1f81ed79a8f..a9481d40f5f 100644 --- a/packages/vertexai/src/types/enums.ts +++ b/packages/vertexai/src/types/enums.ts @@ -240,13 +240,3 @@ export enum Modality { */ DOCUMENT = 'DOCUMENT' } - -/** - * Determines whether inference happens on-device or on-cloud. - * @public - */ -export enum InferenceMode { - PREFER_ON_DEVICE = 'PREFER_ON_DEVICE', - ONLY_ON_DEVICE = 'ONLY_ON_DEVICE', - ONLY_ON_CLOUD = 'ONLY_ON_CLOUD' -} diff --git a/packages/vertexai/src/types/requests.ts b/packages/vertexai/src/types/requests.ts index c643fcd6084..e62f84c948e 100644 --- a/packages/vertexai/src/types/requests.ts +++ b/packages/vertexai/src/types/requests.ts @@ -22,8 +22,7 @@ import { FunctionCallingMode, HarmBlockMethod, HarmBlockThreshold, - HarmCategory, - InferenceMode + HarmCategory } from './enums'; import { ObjectSchemaInterface, SchemaRequest } from './schema'; @@ -225,3 +224,11 @@ export interface HybridParams { onDeviceParams?: LanguageModelCreateOptions; onCloudParams?: ModelParams; } + +/** + * Determines whether inference happens on-device or in-cloud. + */ +export type InferenceMode = + | 'prefer_on_device' + | 'only_on_device' + | 'only_in_cloud'; \ No newline at end of file