Skip to content

Commit 3bb3f36

Browse files
dlarocquedlarocque
authored
Live API v1 (#9205)
* Live API v1 * update docs * Docs and naming changes * Skip problematic Google AI tests * Fix mediaChunks errors * Log warnings for 'close' event reasons * Add note about support to getLiveGenerativeModel docs * Review comments * add license header * docs review * remove candidateCount * Live Refactor WebSocket (#9212) * refactor websocket * remove test ws server
1 parent 2e03df7 commit 3bb3f36

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2721
-592
lines changed

‎common/api-review/ai.api.md‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ export const AIErrorCode: {
3232
readonly REQUEST_ERROR: "request-error";
3333
readonly RESPONSE_ERROR: "response-error";
3434
readonly FETCH_ERROR: "fetch-error";
35+
readonly SESSION_CLOSED: "session-closed";
3536
readonly INVALID_CONTENT: "invalid-content";
3637
readonly API_NOT_ENABLED: "api-not-enabled";
3738
readonly INVALID_SCHEMA: "invalid-schema";
@@ -262,6 +263,7 @@ export type FinishReason = (typeof FinishReason)[keyof typeof FinishReason];
262263
export interface FunctionCall {
263264
// (undocumented)
264265
args: object;
266+
id?: string;
265267
// (undocumented)
266268
name: string;
267269
}
@@ -310,6 +312,7 @@ export interface FunctionDeclarationsTool {
310312

311313
// @public
312314
export interface FunctionResponse {
315+
id?: string;
313316
// (undocumented)
314317
name: string;
315318
// (undocumented)
@@ -444,6 +447,9 @@ export function getGenerativeModel(ai: AI, modelParams: ModelParams, requestOpti
444447
// @beta
445448
export function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
446449

450+
// @beta
451+
export function getLiveGenerativeModel(ai: AI, modelParams: LiveModelParams): LiveGenerativeModel;
452+
447453
// @public
448454
export class GoogleAIBackend extends Backend {
449455
constructor();
@@ -699,6 +705,95 @@ export class IntegerSchema extends Schema {
699705
constructor(schemaParams?: SchemaParams);
700706
}
701707

708+
// @beta
709+
export interface LiveGenerationConfig {
710+
frequencyPenalty?: number;
711+
maxOutputTokens?: number;
712+
presencePenalty?: number;
713+
responseModalities?: [ResponseModality];
714+
speechConfig?: SpeechConfig;
715+
temperature?: number;
716+
topK?: number;
717+
topP?: number;
718+
}
719+
720+
// @beta
721+
export class LiveGenerativeModel extends AIModel {
722+
// Warning: (ae-forgotten-export) The symbol "WebSocketHandler" needs to be exported by the entry point index.d.ts
723+
//
724+
// @internal
725+
constructor(ai: AI, modelParams: LiveModelParams,
726+
_webSocketHandler: WebSocketHandler);
727+
connect(): Promise<LiveSession>;
728+
// (undocumented)
729+
generationConfig: LiveGenerationConfig;
730+
// (undocumented)
731+
systemInstruction?: Content;
732+
// (undocumented)
733+
toolConfig?: ToolConfig;
734+
// (undocumented)
735+
tools?: Tool[];
736+
}
737+
738+
// @beta
739+
export interface LiveModelParams {
740+
// (undocumented)
741+
generationConfig?: LiveGenerationConfig;
742+
// (undocumented)
743+
model: string;
744+
// (undocumented)
745+
systemInstruction?: string | Part | Content;
746+
// (undocumented)
747+
toolConfig?: ToolConfig;
748+
// (undocumented)
749+
tools?: Tool[];
750+
}
751+
752+
// @beta
753+
export const LiveResponseType: {
754+
SERVER_CONTENT: string;
755+
TOOL_CALL: string;
756+
TOOL_CALL_CANCELLATION: string;
757+
};
758+
759+
// @beta
760+
export type LiveResponseType = (typeof LiveResponseType)[keyof typeof LiveResponseType];
761+
762+
// @beta
763+
export interface LiveServerContent {
764+
interrupted?: boolean;
765+
modelTurn?: Content;
766+
turnComplete?: boolean;
767+
// (undocumented)
768+
type: 'serverContent';
769+
}
770+
771+
// @beta
772+
export interface LiveServerToolCall {
773+
functionCalls: FunctionCall[];
774+
// (undocumented)
775+
type: 'toolCall';
776+
}
777+
778+
// @beta
779+
export interface LiveServerToolCallCancellation {
780+
functionIds: string[];
781+
// (undocumented)
782+
type: 'toolCallCancellation';
783+
}
784+
785+
// @beta
786+
export class LiveSession {
787+
// @internal
788+
constructor(webSocketHandler: WebSocketHandler, serverMessages: AsyncGenerator<unknown>);
789+
close(): Promise<void>;
790+
isClosed: boolean;
791+
receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation>;
792+
send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
793+
sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
794+
sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
795+
}
796+
702797
// @public
703798
export const Modality: {
704799
readonly MODALITY_UNSPECIFIED: "MODALITY_UNSPECIFIED";
@@ -763,6 +858,11 @@ export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionRespon
763858
// @public
764859
export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"];
765860

861+
// @beta
862+
export interface PrebuiltVoiceConfig {
863+
voiceConfig?: string;
864+
}
865+
766866
// @public
767867
export interface PromptFeedback {
768868
// (undocumented)
@@ -926,6 +1026,11 @@ export interface Segment {
9261026
text: string;
9271027
}
9281028

1029+
// @beta
1030+
export interface SpeechConfig {
1031+
voiceConfig?: VoiceConfig;
1032+
}
1033+
9291034
// @public
9301035
export interface StartChatParams extends BaseParams {
9311036
// (undocumented)
@@ -1003,6 +1108,11 @@ export interface VideoMetadata {
10031108
startOffset: string;
10041109
}
10051110

1111+
// @beta
1112+
export interface VoiceConfig {
1113+
prebuiltVoiceConfig?: PrebuiltVoiceConfig;
1114+
}
1115+
10061116
// @public (undocumented)
10071117
export interface WebAttribution {
10081118
// (undocumented)

‎docs-devsite/_toc.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,20 @@ toc:
108108
path: /docs/reference/js/ai.inlinedatapart.md
109109
- title: IntegerSchema
110110
path: /docs/reference/js/ai.integerschema.md
111+
- title: LiveGenerationConfig
112+
path: /docs/reference/js/ai.livegenerationconfig.md
113+
- title: LiveGenerativeModel
114+
path: /docs/reference/js/ai.livegenerativemodel.md
115+
- title: LiveModelParams
116+
path: /docs/reference/js/ai.livemodelparams.md
117+
- title: LiveServerContent
118+
path: /docs/reference/js/ai.liveservercontent.md
119+
- title: LiveServerToolCall
120+
path: /docs/reference/js/ai.liveservertoolcall.md
121+
- title: LiveServerToolCallCancellation
122+
path: /docs/reference/js/ai.liveservertoolcallcancellation.md
123+
- title: LiveSession
124+
path: /docs/reference/js/ai.livesession.md
111125
- title: ModalityTokenCount
112126
path: /docs/reference/js/ai.modalitytokencount.md
113127
- title: ModelParams
@@ -118,6 +132,8 @@ toc:
118132
path: /docs/reference/js/ai.objectschema.md
119133
- title: ObjectSchemaRequest
120134
path: /docs/reference/js/ai.objectschemarequest.md
135+
- title: PrebuiltVoiceConfig
136+
path: /docs/reference/js/ai.prebuiltvoiceconfig.md
121137
- title: PromptFeedback
122138
path: /docs/reference/js/ai.promptfeedback.md
123139
- title: RequestOptions
@@ -142,6 +158,8 @@ toc:
142158
path: /docs/reference/js/ai.searchentrypoint.md
143159
- title: Segment
144160
path: /docs/reference/js/ai.segment.md
161+
- title: SpeechConfig
162+
path: /docs/reference/js/ai.speechconfig.md
145163
- title: StartChatParams
146164
path: /docs/reference/js/ai.startchatparams.md
147165
- title: StringSchema
@@ -158,6 +176,8 @@ toc:
158176
path: /docs/reference/js/ai.vertexaibackend.md
159177
- title: VideoMetadata
160178
path: /docs/reference/js/ai.videometadata.md
179+
- title: VoiceConfig
180+
path: /docs/reference/js/ai.voiceconfig.md
161181
- title: WebAttribution
162182
path: /docs/reference/js/ai.webattribution.md
163183
- title: WebGroundingChunk

‎docs-devsite/ai.functioncall.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export interface FunctionCall
2323
| Property | Type | Description |
2424
| --- | --- | --- |
2525
| [args](./ai.functioncall.md#functioncallargs) | object | |
26+
| [id](./ai.functioncall.md#functioncallid) | string | The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->. |
2627
| [name](./ai.functioncall.md#functioncallname) | string | |
2728

2829
## FunctionCall.args
@@ -33,6 +34,18 @@ export interface FunctionCall
3334
args: object;
3435
```
3536

37+
## FunctionCall.id
38+
39+
The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->.
40+
41+
This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be `undefined`<!-- -->.
42+
43+
<b>Signature:</b>
44+
45+
```typescript
46+
id?: string;
47+
```
48+
3649
## FunctionCall.name
3750

3851
<b>Signature:</b>

‎docs-devsite/ai.functionresponse.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,22 @@ export interface FunctionResponse
2222

2323
| Property | Type | Description |
2424
| --- | --- | --- |
25+
| [id](./ai.functionresponse.md#functionresponseid) | string | The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->. |
2526
| [name](./ai.functionresponse.md#functionresponsename) | string | |
2627
| [response](./ai.functionresponse.md#functionresponseresponse) | object | |
2728

29+
## FunctionResponse.id
30+
31+
The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->.
32+
33+
This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be `undefined`<!-- -->.
34+
35+
<b>Signature:</b>
36+
37+
```typescript
38+
id?: string;
39+
```
40+
2841
## FunctionResponse.name
2942

3043
<b>Signature:</b>

0 commit comments

Comments
 (0)