feat(firebaseai): update of bidi input api (#17662)

* update of bidi input api

* Update with api review comments

* fix analyzer

* make sure the existing behavior don't change for media_chunks

* revert unrelated change

* more detail about the realtime usage

* resolve review comments

Author: cynthiajoan

packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart

@@ -133,11 +133,13 @@ class _BidiPageState extends State<BidiPage> {
                 itemBuilder: (context, idx) {
                   return MessageWidget(
                     text: _messages[idx].text,
-                    image: Image.memory(
-                      _messages[idx].imageBytes!,
-                      cacheWidth: 400,
-                      cacheHeight: 400,
-                    ),
+                    image: _messages[idx].imageBytes != null
+                        ? Image.memory(
+                            _messages[idx].imageBytes!,
+                            cacheWidth: 400,
+                            cacheHeight: 400,
+                          )
+                        : null,
                     isFromUser: _messages[idx].fromUser ?? false,
                   );
                 },
@@ -275,13 +277,10 @@ class _BidiPageState extends State<BidiPage> {
     try {
       var inputStream = await _audioInput.startRecordingStream();
       await _audioOutput.playStream();
-      // Map the Uint8List stream to InlineDataPart stream
       if (inputStream != null) {
-        final inlineDataStream = inputStream.map((data) {
-          return InlineDataPart('audio/pcm', data);
-        });
-
-        await _session.sendMediaStream(inlineDataStream);
+        await for (final data in inputStream) {
+          await _session.sendAudioRealtime(InlineDataPart('audio/pcm', data));
+        }
       }
     } catch (e) {
       developer.log(e.toString());

packages/firebase_ai/firebase_ai/lib/src/live_api.dart

@@ -179,18 +179,56 @@ class LiveServerResponse {
 /// Represents realtime input from the client in a live stream.
 class LiveClientRealtimeInput {
   /// Creates a [LiveClientRealtimeInput] instance.
-  ///
-  /// [mediaChunks] (optional): The list of media chunks.
-  LiveClientRealtimeInput({this.mediaChunks});
+  LiveClientRealtimeInput({
+    @Deprecated('Use audio, video, or text instead') this.mediaChunks,
+    this.audio,
+    this.video,
+    this.text,
+  });
+
+  /// Creates a [LiveClientRealtimeInput] with audio data.
+  LiveClientRealtimeInput.audio(this.audio)
+      // ignore: deprecated_member_use_from_same_package
+      : mediaChunks = null,
+        video = null,
+        text = null;
+
+  /// Creates a [LiveClientRealtimeInput] with video data.
+  LiveClientRealtimeInput.video(this.video)
+      // ignore: deprecated_member_use_from_same_package
+      : mediaChunks = null,
+        audio = null,
+        text = null;
+
+  /// Creates a [LiveClientRealtimeInput] with text data.
+  LiveClientRealtimeInput.text(this.text)
+      // ignore: deprecated_member_use_from_same_package
+      : mediaChunks = null,
+        audio = null,
+        video = null;
 
   /// The list of media chunks.
+  @Deprecated('Use audio, video, or text instead')
   final List<InlineDataPart>? mediaChunks;
 
+  /// Audio data.
+  final InlineDataPart? audio;
+
+  /// Video data.
+  final InlineDataPart? video;
+
+  /// Text data.
+  final String? text;
+
   // ignore: public_member_api_docs
   Map<String, dynamic> toJson() => {
         'realtime_input': {
           'media_chunks':
+              // ignore: deprecated_member_use_from_same_package
               mediaChunks?.map((e) => e.toMediaChunkJson()).toList(),
+          if (audio != null) 'audio': audio!.toMediaChunkJson(),
+          if (video != null) 'video': video!.toMediaChunkJson(),
+          if (text != null) 'text': text,
         },
       };
 }

packages/firebase_ai/firebase_ai/lib/src/live_session.dart

@@ -76,9 +76,47 @@ class LiveSession {
     _ws.sink.add(clientJson);
   }
 
+  /// Sends audio data to the server in realtime.
+  ///
+  /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for
+  /// details about the realtime input usage.
+  /// [audio]: The audio data to send.
+  Future<void> sendAudioRealtime(InlineDataPart audio) async {
+    _checkWsStatus();
+    var clientMessage = LiveClientRealtimeInput.audio(audio);
+    var clientJson = jsonEncode(clientMessage.toJson());
+    _ws.sink.add(clientJson);
+  }
+
+  /// Sends video data to the server in realtime.
+  ///
+  /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for
+  /// details about the realtime input usage.
+  /// [video]: The video data to send.
+  Future<void> sendVideoRealtime(InlineDataPart video) async {
+    _checkWsStatus();
+    var clientMessage = LiveClientRealtimeInput.video(video);
+    var clientJson = jsonEncode(clientMessage.toJson());
+    _ws.sink.add(clientJson);
+  }
+
+  /// Sends text data to the server in realtime.
+  ///
+  /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for
+  /// details about the realtime input usage.
+  /// [text]: The text data to send.
+  Future<void> sendTextRealtime(String text) async {
+    _checkWsStatus();
+    var clientMessage = LiveClientRealtimeInput.text(text);
+    var clientJson = jsonEncode(clientMessage.toJson());
+    _ws.sink.add(clientJson);
+  }
+
   /// Sends realtime input (media chunks) to the server.
   ///
   /// [mediaChunks]: The list of media chunks to send.
+  @Deprecated(
+      'Use sendAudioRealtime, sendVideoRealtime, or sendTextRealtime instead')
   Future<void> sendMediaChunks({
     required List<InlineDataPart> mediaChunks,
   }) async {
@@ -95,6 +133,7 @@ class LiveSession {
   ///
   /// Parameters:
   /// - [mediaChunkStream]: The stream of [InlineDataPart] objects to send to the server.
+  @Deprecated('Use sendAudio, sendVideo, or sendText with a stream instead')
   Future<void> sendMediaStream(Stream<InlineDataPart> mediaChunkStream) async {
     _checkWsStatus();
 
@@ -111,6 +150,7 @@ class LiveSession {
 
   Future<void> _sendMediaChunk(InlineDataPart chunk) async {
     var clientMessage = LiveClientRealtimeInput(
+        // ignore: deprecated_member_use_from_same_package
         mediaChunks: [chunk]); // Create a list with the single chunk
     var clientJson = jsonEncode(clientMessage.toJson());
     _ws.sink.add(clientJson);

packages/firebase_ai/firebase_ai/test/live_test.dart

@@ -103,6 +103,7 @@ void main() {
 
     test('LiveClientRealtimeInput toJson() returns correct JSON', () {
       final part = InlineDataPart('audio/pcm', Uint8List.fromList([1, 2, 3]));
+      // ignore: deprecated_member_use_from_same_package
       final message = LiveClientRealtimeInput(mediaChunks: [part]);
       expect(message.toJson(), {
         'realtime_input': {