Skip to content

Commit 5235782

Browse files
dlarocquedlarocque
committed
Live API v1
1 parent 2e03df7 commit 5235782

36 files changed

+2645
-14
lines changed

‎common/api-review/ai.api.md‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ export const AIErrorCode: {
3232
readonly REQUEST_ERROR: "request-error";
3333
readonly RESPONSE_ERROR: "response-error";
3434
readonly FETCH_ERROR: "fetch-error";
35+
readonly SESSION_CLOSED: "session-closed";
3536
readonly INVALID_CONTENT: "invalid-content";
3637
readonly API_NOT_ENABLED: "api-not-enabled";
3738
readonly INVALID_SCHEMA: "invalid-schema";
@@ -262,6 +263,7 @@ export type FinishReason = (typeof FinishReason)[keyof typeof FinishReason];
262263
export interface FunctionCall {
263264
// (undocumented)
264265
args: object;
266+
id?: string;
265267
// (undocumented)
266268
name: string;
267269
}
@@ -310,6 +312,7 @@ export interface FunctionDeclarationsTool {
310312

311313
// @public
312314
export interface FunctionResponse {
315+
id?: string;
313316
// (undocumented)
314317
name: string;
315318
// (undocumented)
@@ -444,6 +447,9 @@ export function getGenerativeModel(ai: AI, modelParams: ModelParams, requestOpti
444447
// @beta
445448
export function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
446449

450+
// @beta
451+
export function getLiveGenerativeModel(ai: AI, modelParams: LiveModelParams): LiveGenerativeModel;
452+
447453
// @public
448454
export class GoogleAIBackend extends Backend {
449455
constructor();
@@ -699,6 +705,127 @@ export class IntegerSchema extends Schema {
699705
constructor(schemaParams?: SchemaParams);
700706
}
701707

708+
// Warning: (ae-internal-missing-underscore) The name "LiveClientContent" should be prefixed with an underscore because the declaration is marked as @internal
709+
//
710+
// @internal
711+
export interface LiveClientContent {
712+
// (undocumented)
713+
clientContent: {
714+
turns: [Content];
715+
turnComplete: boolean;
716+
};
717+
}
718+
719+
// Warning: (ae-internal-missing-underscore) The name "LiveClientRealtimeInput" should be prefixed with an underscore because the declaration is marked as @internal
720+
//
721+
// @internal
722+
export interface LiveClientRealtimeInput {
723+
// (undocumented)
724+
realtimeInput: {
725+
mediaChunks: GenerativeContentBlob[];
726+
};
727+
}
728+
729+
// Warning: (ae-internal-missing-underscore) The name "LiveClientSetup" should be prefixed with an underscore because the declaration is marked as @internal
730+
//
731+
// @internal
732+
export interface LiveClientSetup {
733+
// (undocumented)
734+
setup: {
735+
model: string;
736+
generationConfig?: LiveGenerationConfig;
737+
};
738+
}
739+
740+
// @beta
741+
export interface LiveGenerationConfig {
742+
candidateCount?: number;
743+
frequencyPenalty?: number;
744+
maxOutputTokens?: number;
745+
presencePenalty?: number;
746+
responseModalities?: [ResponseModality];
747+
speechConfig?: SpeechConfig;
748+
temperature?: number;
749+
topK?: number;
750+
topP?: number;
751+
}
752+
753+
// @beta
754+
export class LiveGenerativeModel extends AIModel {
755+
// @internal
756+
constructor(ai: AI, modelParams: LiveModelParams, _webSocketHandler: WebSocketHandler);
757+
connect(): Promise<LiveSession>;
758+
// (undocumented)
759+
generationConfig: LiveGenerationConfig;
760+
// (undocumented)
761+
systemInstruction?: Content;
762+
// (undocumented)
763+
toolConfig?: ToolConfig;
764+
// (undocumented)
765+
tools?: Tool[];
766+
// Warning: (ae-forgotten-export) The symbol "WebSocketHandler" needs to be exported by the entry point index.d.ts
767+
//
768+
// (undocumented)
769+
_webSocketHandler: WebSocketHandler;
770+
}
771+
772+
// @beta
773+
export interface LiveModelParams {
774+
generationConfig?: LiveGenerationConfig;
775+
model: string;
776+
// (undocumented)
777+
systemInstruction?: string | Part | Content;
778+
// (undocumented)
779+
toolConfig?: ToolConfig;
780+
// (undocumented)
781+
tools?: Tool[];
782+
}
783+
784+
// @beta
785+
export const LiveResponseType: {
786+
SERVER_CONTENT: string;
787+
TOOL_CALL: string;
788+
TOOL_CALL_CANCELLATION: string;
789+
};
790+
791+
// @beta
792+
export type LiveResponseType = (typeof LiveResponseType)[keyof typeof LiveResponseType];
793+
794+
// @beta
795+
export interface LiveServerContent {
796+
interrupted?: boolean;
797+
modelTurn?: Content;
798+
turnComplete?: boolean;
799+
// (undocumented)
800+
type: 'serverContent';
801+
}
802+
803+
// @beta
804+
export interface LiveServerToolCall {
805+
functionCalls: FunctionCall[];
806+
// (undocumented)
807+
type: 'toolCall';
808+
}
809+
810+
// @beta
811+
export interface LiveServerToolCallCancellation {
812+
functionIds: string[];
813+
// (undocumented)
814+
type: 'toolCallCancellation';
815+
}
816+
817+
// @beta
818+
export class LiveSession {
819+
// @internal
820+
constructor(webSocketHandler: WebSocketHandler, serverMessages: AsyncGenerator<unknown>);
821+
close(): Promise<void>;
822+
isClosed: boolean;
823+
receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation>;
824+
send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
825+
sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
826+
sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
827+
}
828+
702829
// @public
703830
export const Modality: {
704831
readonly MODALITY_UNSPECIFIED: "MODALITY_UNSPECIFIED";
@@ -763,6 +890,11 @@ export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionRespon
763890
// @public
764891
export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"];
765892

893+
// @beta
894+
export interface PrebuiltVoiceConfig {
895+
voiceConfig?: string;
896+
}
897+
766898
// @public
767899
export interface PromptFeedback {
768900
// (undocumented)
@@ -926,6 +1058,11 @@ export interface Segment {
9261058
text: string;
9271059
}
9281060

1061+
// @beta
1062+
export interface SpeechConfig {
1063+
voiceConfig?: VoiceConfig;
1064+
}
1065+
9291066
// @public
9301067
export interface StartChatParams extends BaseParams {
9311068
// (undocumented)
@@ -1003,6 +1140,11 @@ export interface VideoMetadata {
10031140
startOffset: string;
10041141
}
10051142

1143+
// @beta
1144+
export interface VoiceConfig {
1145+
prebuiltVoiceConfig?: PrebuiltVoiceConfig;
1146+
}
1147+
10061148
// @public (undocumented)
10071149
export interface WebAttribution {
10081150
// (undocumented)

‎docs-devsite/_toc.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,20 @@ toc:
108108
path: /docs/reference/js/ai.inlinedatapart.md
109109
- title: IntegerSchema
110110
path: /docs/reference/js/ai.integerschema.md
111+
- title: LiveGenerationConfig
112+
path: /docs/reference/js/ai.livegenerationconfig.md
113+
- title: LiveGenerativeModel
114+
path: /docs/reference/js/ai.livegenerativemodel.md
115+
- title: LiveModelParams
116+
path: /docs/reference/js/ai.livemodelparams.md
117+
- title: LiveServerContent
118+
path: /docs/reference/js/ai.liveservercontent.md
119+
- title: LiveServerToolCall
120+
path: /docs/reference/js/ai.liveservertoolcall.md
121+
- title: LiveServerToolCallCancellation
122+
path: /docs/reference/js/ai.liveservertoolcallcancellation.md
123+
- title: LiveSession
124+
path: /docs/reference/js/ai.livesession.md
111125
- title: ModalityTokenCount
112126
path: /docs/reference/js/ai.modalitytokencount.md
113127
- title: ModelParams
@@ -118,6 +132,8 @@ toc:
118132
path: /docs/reference/js/ai.objectschema.md
119133
- title: ObjectSchemaRequest
120134
path: /docs/reference/js/ai.objectschemarequest.md
135+
- title: PrebuiltVoiceConfig
136+
path: /docs/reference/js/ai.prebuiltvoiceconfig.md
121137
- title: PromptFeedback
122138
path: /docs/reference/js/ai.promptfeedback.md
123139
- title: RequestOptions
@@ -142,6 +158,8 @@ toc:
142158
path: /docs/reference/js/ai.searchentrypoint.md
143159
- title: Segment
144160
path: /docs/reference/js/ai.segment.md
161+
- title: SpeechConfig
162+
path: /docs/reference/js/ai.speechconfig.md
145163
- title: StartChatParams
146164
path: /docs/reference/js/ai.startchatparams.md
147165
- title: StringSchema
@@ -158,6 +176,8 @@ toc:
158176
path: /docs/reference/js/ai.vertexaibackend.md
159177
- title: VideoMetadata
160178
path: /docs/reference/js/ai.videometadata.md
179+
- title: VoiceConfig
180+
path: /docs/reference/js/ai.voiceconfig.md
161181
- title: WebAttribution
162182
path: /docs/reference/js/ai.webattribution.md
163183
- title: WebGroundingChunk

‎docs-devsite/ai.functioncall.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export interface FunctionCall
2323
| Property | Type | Description |
2424
| --- | --- | --- |
2525
| [args](./ai.functioncall.md#functioncallargs) | object | |
26+
| [id](./ai.functioncall.md#functioncallid) | string | The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->.<!-- -->This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be <code>undefined</code>. |
2627
| [name](./ai.functioncall.md#functioncallname) | string | |
2728

2829
## FunctionCall.args
@@ -33,6 +34,18 @@ export interface FunctionCall
3334
args: object;
3435
```
3536

37+
## FunctionCall.id
38+
39+
The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->.
40+
41+
This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be `undefined`<!-- -->.
42+
43+
<b>Signature:</b>
44+
45+
```typescript
46+
id?: string;
47+
```
48+
3649
## FunctionCall.name
3750

3851
<b>Signature:</b>

‎docs-devsite/ai.functionresponse.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,22 @@ export interface FunctionResponse
2222

2323
| Property | Type | Description |
2424
| --- | --- | --- |
25+
| [id](./ai.functionresponse.md#functionresponseid) | string | The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->.<!-- -->This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be <code>undefined</code>. |
2526
| [name](./ai.functionresponse.md#functionresponsename) | string | |
2627
| [response](./ai.functionresponse.md#functionresponseresponse) | object | |
2728

29+
## FunctionResponse.id
30+
31+
The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->.
32+
33+
This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be `undefined`<!-- -->.
34+
35+
<b>Signature:</b>
36+
37+
```typescript
38+
id?: string;
39+
```
40+
2841
## FunctionResponse.name
2942

3043
<b>Signature:</b>

0 commit comments

Comments
 (0)