From 0ceb1e23976d0d80525701c6b067506cba8f91af Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Mon, 25 Aug 2025 16:45:15 -0700 Subject: [PATCH 1/7] update of bidi input api --- .../example/lib/pages/bidi_page.dart | 25 ++++++----- .../firebase_ai/lib/src/live_api.dart | 45 ++++++++++++++++--- .../firebase_ai/lib/src/live_session.dart | 32 +++++++++++++ 3 files changed, 85 insertions(+), 17 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 3d86a4c4b04c..83c6940fc8a5 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -65,7 +65,7 @@ class _BidiPageState extends State { final config = LiveGenerationConfig( speechConfig: SpeechConfig(voiceName: 'Fenrir'), responseModalities: [ - ResponseModalities.audio, + ResponseModalities.text, ], ); @@ -133,11 +133,13 @@ class _BidiPageState extends State { itemBuilder: (context, idx) { return MessageWidget( text: _messages[idx].text, - image: Image.memory( - _messages[idx].imageBytes!, - cacheWidth: 400, - cacheHeight: 400, - ), + image: _messages[idx].imageBytes != null + ? Image.memory( + _messages[idx].imageBytes!, + cacheWidth: 400, + cacheHeight: 400, + ) + : null, isFromUser: _messages[idx].fromUser ?? false, ); }, @@ -277,11 +279,9 @@ class _BidiPageState extends State { await _audioOutput.playStream(); // Map the Uint8List stream to InlineDataPart stream if (inputStream != null) { - final inlineDataStream = inputStream.map((data) { - return InlineDataPart('audio/pcm', data); - }); - - await _session.sendMediaStream(inlineDataStream); + await for (final data in inputStream) { + await _session.sendAudio(InlineDataPart('audio/pcm', data)); + } } } catch (e) { developer.log(e.toString()); @@ -307,7 +307,8 @@ class _BidiPageState extends State { }); try { final prompt = Content.text(textPrompt); - await _session.send(input: prompt, turnComplete: true); + // await _session.send(input: prompt, turnComplete: true); + await _session.sendText(textPrompt); } catch (e) { _showError(e.toString()); } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index c5358101c52d..80c937f44527 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -180,18 +180,53 @@ class LiveServerResponse { /// Represents realtime input from the client in a live stream. class LiveClientRealtimeInput { /// Creates a [LiveClientRealtimeInput] instance. - /// - /// [mediaChunks] (optional): The list of media chunks. - LiveClientRealtimeInput({this.mediaChunks}); + LiveClientRealtimeInput({ + @Deprecated('Use audio, video, or text instead') this.mediaChunks, + this.audio, + this.video, + this.text, + }); + + /// Creates a [LiveClientRealtimeInput] with audio data. + LiveClientRealtimeInput.audio(this.audio) + : mediaChunks = null, + video = null, + text = null; + + /// Creates a [LiveClientRealtimeInput] with video data. + LiveClientRealtimeInput.video(this.video) + : mediaChunks = null, + audio = null, + text = null; + + /// Creates a [LiveClientRealtimeInput] with text data. + LiveClientRealtimeInput.text(this.text) + : mediaChunks = null, + audio = null, + video = null; /// The list of media chunks. + @Deprecated('Use audio, video, or text instead') final List? mediaChunks; + /// Audio data. + final InlineDataPart? audio; + + /// Video data. + final InlineDataPart? video; + + /// Text data. + final String? text; + // ignore: public_member_api_docs Map toJson() => { 'realtime_input': { - 'media_chunks': - mediaChunks?.map((e) => e.toMediaChunkJson()).toList(), + if (mediaChunks != null) + 'media_chunks': + mediaChunks?.map((e) => e.toMediaChunkJson()).toList(), + if (audio != null) 'audio': audio!.toMediaChunkJson(), + if (video != null) 'video': video!.toMediaChunkJson(), + if (text != null) 'text': text, }, }; } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index 20e700bc82bf..a8aac040bc42 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -76,9 +76,40 @@ class LiveSession { _ws.sink.add(clientJson); } + /// Sends audio data to the server. + /// + /// [audio]: The audio data to send. + Future sendAudio(InlineDataPart audio) async { + _checkWsStatus(); + var clientMessage = LiveClientRealtimeInput.audio(audio); + var clientJson = jsonEncode(clientMessage.toJson()); + _ws.sink.add(clientJson); + } + + /// Sends video data to the server. + /// + /// [video]: The video data to send. + Future sendVideo(InlineDataPart video) async { + _checkWsStatus(); + var clientMessage = LiveClientRealtimeInput.video(video); + var clientJson = jsonEncode(clientMessage.toJson()); + _ws.sink.add(clientJson); + } + + /// Sends text data to the server. + /// + /// [text]: The text data to send. + Future sendText(String text) async { + _checkWsStatus(); + var clientMessage = LiveClientRealtimeInput.text(text); + var clientJson = jsonEncode(clientMessage.toJson()); + _ws.sink.add(clientJson); + } + /// Sends realtime input (media chunks) to the server. /// /// [mediaChunks]: The list of media chunks to send. + @Deprecated('Use sendAudio, sendVideo, or sendText instead') Future sendMediaChunks({ required List mediaChunks, }) async { @@ -95,6 +126,7 @@ class LiveSession { /// /// Parameters: /// - [mediaChunkStream]: The stream of [InlineDataPart] objects to send to the server. + @Deprecated('Use sendAudio, sendVideo, or sendText with a stream instead') Future sendMediaStream(Stream mediaChunkStream) async { _checkWsStatus(); From b867f235af87d9938b6565c407ec18346bc05003 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 30 Sep 2025 22:02:25 -0700 Subject: [PATCH 2/7] Update with api review comments --- .../firebase_ai/example/lib/pages/bidi_page.dart | 4 ++-- .../firebase_ai/lib/src/live_session.dart | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 83c6940fc8a5..fd6811b79ab2 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -280,7 +280,7 @@ class _BidiPageState extends State { // Map the Uint8List stream to InlineDataPart stream if (inputStream != null) { await for (final data in inputStream) { - await _session.sendAudio(InlineDataPart('audio/pcm', data)); + await _session.sendAudioRealtime(InlineDataPart('audio/pcm', data)); } } } catch (e) { @@ -308,7 +308,7 @@ class _BidiPageState extends State { try { final prompt = Content.text(textPrompt); // await _session.send(input: prompt, turnComplete: true); - await _session.sendText(textPrompt); + await _session.sendTextRealtime(textPrompt); } catch (e) { _showError(e.toString()); } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index a8aac040bc42..96b75dc676a1 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -76,30 +76,30 @@ class LiveSession { _ws.sink.add(clientJson); } - /// Sends audio data to the server. + /// Sends audio data to the server in realtime. /// /// [audio]: The audio data to send. - Future sendAudio(InlineDataPart audio) async { + Future sendAudioRealtime(InlineDataPart audio) async { _checkWsStatus(); var clientMessage = LiveClientRealtimeInput.audio(audio); var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); } - /// Sends video data to the server. + /// Sends video data to the server in realtime. /// /// [video]: The video data to send. - Future sendVideo(InlineDataPart video) async { + Future sendVideoRealtime(InlineDataPart video) async { _checkWsStatus(); var clientMessage = LiveClientRealtimeInput.video(video); var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); } - /// Sends text data to the server. + /// Sends text data to the server in realtime. /// /// [text]: The text data to send. - Future sendText(String text) async { + Future sendTextRealtime(String text) async { _checkWsStatus(); var clientMessage = LiveClientRealtimeInput.text(text); var clientJson = jsonEncode(clientMessage.toJson()); @@ -109,7 +109,8 @@ class LiveSession { /// Sends realtime input (media chunks) to the server. /// /// [mediaChunks]: The list of media chunks to send. - @Deprecated('Use sendAudio, sendVideo, or sendText instead') + @Deprecated( + 'Use sendAudioRealtime, sendVideoRealtime, or sendTextRealtime instead') Future sendMediaChunks({ required List mediaChunks, }) async { From 6bf76b2e8ed35ab610218265d63916174c84f5d9 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 1 Oct 2025 10:08:22 -0700 Subject: [PATCH 3/7] fix analyzer --- .../firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart | 3 +-- packages/firebase_ai/firebase_ai/lib/src/live_api.dart | 5 +++++ packages/firebase_ai/firebase_ai/lib/src/live_session.dart | 1 + packages/firebase_ai/firebase_ai/test/live_test.dart | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index fd6811b79ab2..708ff98fbd7f 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -307,8 +307,7 @@ class _BidiPageState extends State { }); try { final prompt = Content.text(textPrompt); - // await _session.send(input: prompt, turnComplete: true); - await _session.sendTextRealtime(textPrompt); + await _session.send(input: prompt, turnComplete: true); } catch (e) { _showError(e.toString()); } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index 81e61cd645ab..824ce041966a 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -188,18 +188,21 @@ class LiveClientRealtimeInput { /// Creates a [LiveClientRealtimeInput] with audio data. LiveClientRealtimeInput.audio(this.audio) + // ignore: deprecated_member_use_from_same_package : mediaChunks = null, video = null, text = null; /// Creates a [LiveClientRealtimeInput] with video data. LiveClientRealtimeInput.video(this.video) + // ignore: deprecated_member_use_from_same_package : mediaChunks = null, audio = null, text = null; /// Creates a [LiveClientRealtimeInput] with text data. LiveClientRealtimeInput.text(this.text) + // ignore: deprecated_member_use_from_same_package : mediaChunks = null, audio = null, video = null; @@ -220,8 +223,10 @@ class LiveClientRealtimeInput { // ignore: public_member_api_docs Map toJson() => { 'realtime_input': { + // ignore: deprecated_member_use_from_same_package if (mediaChunks != null) 'media_chunks': + // ignore: deprecated_member_use_from_same_package mediaChunks?.map((e) => e.toMediaChunkJson()).toList(), if (audio != null) 'audio': audio!.toMediaChunkJson(), if (video != null) 'video': video!.toMediaChunkJson(), diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index 96b75dc676a1..f88833962603 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -144,6 +144,7 @@ class LiveSession { Future _sendMediaChunk(InlineDataPart chunk) async { var clientMessage = LiveClientRealtimeInput( + // ignore: deprecated_member_use_from_same_package mediaChunks: [chunk]); // Create a list with the single chunk var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); diff --git a/packages/firebase_ai/firebase_ai/test/live_test.dart b/packages/firebase_ai/firebase_ai/test/live_test.dart index a89c53e7901a..6b4b6b9c290f 100644 --- a/packages/firebase_ai/firebase_ai/test/live_test.dart +++ b/packages/firebase_ai/firebase_ai/test/live_test.dart @@ -103,6 +103,7 @@ void main() { test('LiveClientRealtimeInput toJson() returns correct JSON', () { final part = InlineDataPart('audio/pcm', Uint8List.fromList([1, 2, 3])); + // ignore: deprecated_member_use_from_same_package final message = LiveClientRealtimeInput(mediaChunks: [part]); expect(message.toJson(), { 'realtime_input': { From e146f805491cc9aab8573ffbdea999abfd21137f Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 1 Oct 2025 20:49:04 -0700 Subject: [PATCH 4/7] make sure the existing behavior don't change for media_chunks --- packages/firebase_ai/firebase_ai/lib/src/live_api.dart | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index 824ce041966a..7ced5c92e15c 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -223,11 +223,9 @@ class LiveClientRealtimeInput { // ignore: public_member_api_docs Map toJson() => { 'realtime_input': { - // ignore: deprecated_member_use_from_same_package - if (mediaChunks != null) - 'media_chunks': - // ignore: deprecated_member_use_from_same_package - mediaChunks?.map((e) => e.toMediaChunkJson()).toList(), + 'media_chunks': + // ignore: deprecated_member_use_from_same_package + mediaChunks?.map((e) => e.toMediaChunkJson()).toList(), if (audio != null) 'audio': audio!.toMediaChunkJson(), if (video != null) 'video': video!.toMediaChunkJson(), if (text != null) 'text': text, From efe887591a98a76497af81cfbf34e46c4f9ed042 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 1 Oct 2025 20:50:27 -0700 Subject: [PATCH 5/7] revert unrelated change --- .../firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 708ff98fbd7f..3bd23fff0797 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -65,7 +65,7 @@ class _BidiPageState extends State { final config = LiveGenerationConfig( speechConfig: SpeechConfig(voiceName: 'Fenrir'), responseModalities: [ - ResponseModalities.text, + ResponseModalities.audio, ], ); From 342dbb32ca134efa5c5a81688548008935d05e25 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 1 Oct 2025 20:54:55 -0700 Subject: [PATCH 6/7] more detail about the realtime usage --- packages/firebase_ai/firebase_ai/lib/src/live_session.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index f88833962603..f136a644d03d 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -78,6 +78,8 @@ class LiveSession { /// Sends audio data to the server in realtime. /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. /// [audio]: The audio data to send. Future sendAudioRealtime(InlineDataPart audio) async { _checkWsStatus(); @@ -88,6 +90,8 @@ class LiveSession { /// Sends video data to the server in realtime. /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. /// [video]: The video data to send. Future sendVideoRealtime(InlineDataPart video) async { _checkWsStatus(); @@ -98,6 +102,8 @@ class LiveSession { /// Sends text data to the server in realtime. /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. /// [text]: The text data to send. Future sendTextRealtime(String text) async { _checkWsStatus(); From 9d8a65386ffe552febfa06960f07b78258b0e65a Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 2 Oct 2025 16:56:05 -0700 Subject: [PATCH 7/7] resolve review comments --- .../firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 3bd23fff0797..b902d6b74d2a 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -277,7 +277,6 @@ class _BidiPageState extends State { try { var inputStream = await _audioInput.startRecordingStream(); await _audioOutput.playStream(); - // Map the Uint8List stream to InlineDataPart stream if (inputStream != null) { await for (final data in inputStream) { await _session.sendAudioRealtime(InlineDataPart('audio/pcm', data));