Review comments

Author: dlarocque

common/api-review/ai.api.md

@@ -705,35 +705,6 @@ export class IntegerSchema extends Schema {
     constructor(schemaParams?: SchemaParams);
 }
 
-// @internal
-export interface _LiveClientContent {
-    // (undocumented)
-    clientContent: {
-        turns: [Content];
-        turnComplete: boolean;
-    };
-}
-
-// @internal
-export interface _LiveClientRealtimeInput {
-    // (undocumented)
-    realtimeInput: {
-        mediaChunks: GenerativeContentBlob[];
-    };
-}
-
-// @internal
-export interface _LiveClientSetup {
-    // (undocumented)
-    setup: {
-        model: string;
-        generationConfig?: LiveGenerationConfig;
-        tools?: Tool[];
-        toolConfig?: ToolConfig;
-        systemInstruction?: string | Part | Content;
-    };
-}
-
 // @beta
 export interface LiveGenerationConfig {
     candidateCount?: number;

docs-devsite/ai.functioncall.md

@@ -23,7 +23,7 @@ export interface FunctionCall
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [args](./ai.functioncall.md#functioncallargs) | object |  |
-|  [id](./ai.functioncall.md#functioncallid) | string | The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->.<!-- -->This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be <code>undefined</code>. |
+|  [id](./ai.functioncall.md#functioncallid) | string | The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->. |
 |  [name](./ai.functioncall.md#functioncallname) | string |  |
 
 ## FunctionCall.args

docs-devsite/ai.functionresponse.md

@@ -22,7 +22,7 @@ export interface FunctionResponse
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [id](./ai.functionresponse.md#functionresponseid) | string | The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->.<!-- -->This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be <code>undefined</code>. |
+|  [id](./ai.functionresponse.md#functionresponseid) | string | The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->. |
 |  [name](./ai.functionresponse.md#functionresponsename) | string |  |
 |  [response](./ai.functionresponse.md#functionresponseresponse) | object |  |
 

docs-devsite/ai.livegenerativemodel.md

@@ -15,7 +15,7 @@ https://github.com/firebase/firebase-js-sdk
 
 Class for Live generative model APIs. The Live API enables low-latency, two-way multimodal interactions with Gemini.
 
-Do not call this class' constructor directly. Instead, call [getLiveGenerativeModel()](./ai.md#getlivegenerativemodel_f2099ac)<!-- -->.
+This class should only be instantiated with [getLiveGenerativeModel()](./ai.md#getlivegenerativemodel_f2099ac)<!-- -->.
 
 The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `LiveGenerativeModel` class.
 

docs-devsite/ai.livesession.md

@@ -15,7 +15,7 @@ https://github.com/firebase/firebase-js-sdk
 
 Represents an active, real-time, bidirectional conversation with the model.
 
-Do not call this constructor directly. Instead, call [LiveGenerativeModel.connect()](./ai.livegenerativemodel.md#livegenerativemodelconnect)<!-- -->.
+This class should only be instantiated by calling [LiveGenerativeModel.connect()](./ai.livegenerativemodel.md#livegenerativemodelconnect)<!-- -->.
 
 The constructor for this class is marked as internal. Third-party code should not call the constructor directly or create subclasses that extend the `LiveSession` class.
 

docs-devsite/ai.md

@@ -39,8 +39,8 @@ The Firebase AI Web SDK.
 |  [ImagenImageFormat](./ai.imagenimageformat.md#imagenimageformat_class) | <b><i>(Public Preview)</i></b> Defines the image format for images generated by Imagen.<!-- -->Use this class to specify the desired format (JPEG or PNG) and compression quality for images generated by Imagen. This is typically included as part of [ImagenModelParams](./ai.imagenmodelparams.md#imagenmodelparams_interface)<!-- -->. |
 |  [ImagenModel](./ai.imagenmodel.md#imagenmodel_class) | <b><i>(Public Preview)</i></b> Class for Imagen model APIs.<!-- -->This class provides methods for generating images using the Imagen model. |
 |  [IntegerSchema](./ai.integerschema.md#integerschema_class) | Schema class for "integer" types. |
-|  [LiveGenerativeModel](./ai.livegenerativemodel.md#livegenerativemodel_class) | <b><i>(Public Preview)</i></b> Class for Live generative model APIs. The Live API enables low-latency, two-way multimodal interactions with Gemini.<!-- -->Do not call this class' constructor directly. Instead, call [getLiveGenerativeModel()](./ai.md#getlivegenerativemodel_f2099ac)<!-- -->. |
-|  [LiveSession](./ai.livesession.md#livesession_class) | <b><i>(Public Preview)</i></b> Represents an active, real-time, bidirectional conversation with the model.<!-- -->Do not call this constructor directly. Instead, call [LiveGenerativeModel.connect()](./ai.livegenerativemodel.md#livegenerativemodelconnect)<!-- -->. |
+|  [LiveGenerativeModel](./ai.livegenerativemodel.md#livegenerativemodel_class) | <b><i>(Public Preview)</i></b> Class for Live generative model APIs. The Live API enables low-latency, two-way multimodal interactions with Gemini.<!-- -->This class should only be instantiated with [getLiveGenerativeModel()](./ai.md#getlivegenerativemodel_f2099ac)<!-- -->. |
+|  [LiveSession](./ai.livesession.md#livesession_class) | <b><i>(Public Preview)</i></b> Represents an active, real-time, bidirectional conversation with the model.<!-- -->This class should only be instantiated by calling [LiveGenerativeModel.connect()](./ai.livegenerativemodel.md#livegenerativemodelconnect)<!-- -->. |
 |  [NumberSchema](./ai.numberschema.md#numberschema_class) | Schema class for "number" types. |
 |  [ObjectSchema](./ai.objectschema.md#objectschema_class) | Schema class for "object" types. The <code>properties</code> param must be a map of <code>Schema</code> objects. |
 |  [Schema](./ai.schema.md#schema_class) | Parent class encompassing all Schema types, with static methods that allow building specific Schema types. This class can be converted with <code>JSON.stringify()</code> into a JSON string accepted by Vertex AI REST endpoints. (This string conversion is automatically done when calling SDK methods.) |

packages/ai/src/methods/live-session.ts

@@ -18,8 +18,6 @@
 import {
   AIErrorCode,
   GenerativeContentBlob,
-  _LiveClientContent,
-  _LiveClientRealtimeInput,
   LiveResponseType,
   LiveServerContent,
   LiveServerToolCall,
@@ -30,11 +28,15 @@ import { formatNewContent } from '../requests/request-helpers';
 import { AIError } from '../errors';
 import { WebSocketHandler } from '../platform/websocket';
 import { logger } from '../logger';
+import {
+  _LiveClientContent,
+  _LiveClientRealtimeInput
+} from '../types/live-responses';
 
 /**
  * Represents an active, real-time, bidirectional conversation with the model.
  *
- * Do not call this constructor directly. Instead, call {@link LiveGenerativeModel.connect}.
+ * This class should only be instantiated by calling {@link LiveGenerativeModel.connect}.
  *
  * @beta
  */
@@ -173,18 +175,26 @@ export class LiveSession {
         if (LiveResponseType.SERVER_CONTENT in message) {
           yield {
             type: 'serverContent',
-            ...(message as { serverContent: object }).serverContent
+            ...(message as { serverContent: Omit<LiveServerContent, 'type'> })
+              .serverContent
           } as LiveServerContent;
         } else if (LiveResponseType.TOOL_CALL in message) {
           yield {
             type: 'toolCall',
-            ...(message as { toolCall: object }).toolCall
+            ...(message as { toolCall: Omit<LiveServerToolCall, 'type'> })
+              .toolCall
           } as LiveServerToolCall;
         } else if (LiveResponseType.TOOL_CALL_CANCELLATION in message) {
           yield {
             type: 'toolCallCancellation',
-            ...(message as { toolCallCancellation: object })
-              .toolCallCancellation
+            ...(
+              message as {
+                toolCallCancellation: Omit<
+                  LiveServerToolCallCancellation,
+                  'type'
+                >;
+              }
+            ).toolCallCancellation
           } as LiveServerToolCallCancellation;
         } else {
           logger.warn(

packages/ai/src/models/live-generative-model.test.ts

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 import { use, expect } from 'chai';
-import sinon, { stub } from 'sinon';
+import sinon, { SinonFakeTimers, stub } from 'sinon';
 import sinonChai from 'sinon-chai';
 import chaiAsPromised from 'chai-as-promised';
 import { AI } from '../public-types';
@@ -79,13 +79,16 @@ const fakeAI: AI = {
 
 describe('LiveGenerativeModel', () => {
   let mockHandler: MockWebSocketHandler;
+  let clock: SinonFakeTimers;
 
   beforeEach(() => {
     mockHandler = new MockWebSocketHandler();
+    clock = sinon.useFakeTimers();
   });
 
   afterEach(() => {
     sinon.restore();
+    clock.restore();
   });
 
   it('connect() should call handler.connect and send setup message', async () => {
@@ -100,7 +103,8 @@ describe('LiveGenerativeModel', () => {
     expect(mockHandler.connect).to.have.been.calledOnce;
 
     // Wait for the setup message to be sent
-    await new Promise(resolve => setTimeout(resolve, 0));
+    await clock.runAllAsync();
+
     expect(mockHandler.send).to.have.been.calledOnce;
     const setupMessage = JSON.parse(mockHandler.send.getCall(0).args[0]);
     expect(setupMessage.setup.model).to.include('my-model');
@@ -121,7 +125,7 @@ describe('LiveGenerativeModel', () => {
     const connectPromise = model.connect();
 
     // Wait for setup message
-    await new Promise(resolve => setTimeout(resolve, 0));
+    await clock.runAllAsync();
 
     // Simulate a failed handshake
     mockHandler.simulateServerMessage({ error: 'handshake failed' });
@@ -152,7 +156,9 @@ describe('LiveGenerativeModel', () => {
       mockHandler
     );
     const connectPromise = model.connect();
-    await new Promise(resolve => setTimeout(resolve, 0)); // Allow async operations
+
+    // Wait for setup message
+    await clock.runAllAsync();
 
     const sentData = JSON.parse(mockHandler.send.getCall(0).args[0]);
     expect(sentData.setup.generationConfig).to.deep.equal({ temperature: 0.8 });

packages/ai/src/models/live-generative-model.ts

@@ -26,18 +26,18 @@ import {
   LiveGenerationConfig,
   LiveModelParams,
   Tool,
-  ToolConfig,
-  _LiveClientSetup
+  ToolConfig
 } from '../public-types';
 import { WebSocketHandler } from '../platform/websocket';
 import { WebSocketUrl } from '../requests/request';
 import { formatSystemInstruction } from '../requests/request-helpers';
+import { _LiveClientSetup } from '../types/live-responses';
 
 /**
  * Class for Live generative model APIs. The Live API enables low-latency, two-way multimodal
  * interactions with Gemini.
  *
- * Do not call this class' constructor directly. Instead, call {@link getLiveGenerativeModel}.
+ * This class should only be instantiated with {@link getLiveGenerativeModel}.
  *
  * @beta
  */

packages/ai/src/types/content.ts

@@ -125,7 +125,8 @@ export interface FunctionCall {
   /**
    * The id of the function call. This must be sent back in the associated {@link FunctionResponse}.
    *
-   * This property is only supported in the Gemini Developer API ({@link GoogleAIBackend}).
+   *
+   * @remarks This property is only supported in the Gemini Developer API ({@link GoogleAIBackend}).
    * When using the Gemini Developer API ({@link GoogleAIBackend}), this property will be
    * `undefined`.
    */
@@ -147,7 +148,7 @@ export interface FunctionResponse {
   /**
    * The id of the {@link FunctionCall}.
    *
-   * This property is only supported in the Gemini Developer API ({@link GoogleAIBackend}).
+   * @remarks This property is only supported in the Gemini Developer API ({@link GoogleAIBackend}).
    * When using the Gemini Developer API ({@link GoogleAIBackend}), this property will be
    * `undefined`.
    */