diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 3d86a4c4b04c..b902d6b74d2a 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -133,11 +133,13 @@ class _BidiPageState extends State { itemBuilder: (context, idx) { return MessageWidget( text: _messages[idx].text, - image: Image.memory( - _messages[idx].imageBytes!, - cacheWidth: 400, - cacheHeight: 400, - ), + image: _messages[idx].imageBytes != null + ? Image.memory( + _messages[idx].imageBytes!, + cacheWidth: 400, + cacheHeight: 400, + ) + : null, isFromUser: _messages[idx].fromUser ?? false, ); }, @@ -275,13 +277,10 @@ class _BidiPageState extends State { try { var inputStream = await _audioInput.startRecordingStream(); await _audioOutput.playStream(); - // Map the Uint8List stream to InlineDataPart stream if (inputStream != null) { - final inlineDataStream = inputStream.map((data) { - return InlineDataPart('audio/pcm', data); - }); - - await _session.sendMediaStream(inlineDataStream); + await for (final data in inputStream) { + await _session.sendAudioRealtime(InlineDataPart('audio/pcm', data)); + } } } catch (e) { developer.log(e.toString()); diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index fdbe2c63d0af..7ced5c92e15c 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -179,18 +179,56 @@ class LiveServerResponse { /// Represents realtime input from the client in a live stream. class LiveClientRealtimeInput { /// Creates a [LiveClientRealtimeInput] instance. - /// - /// [mediaChunks] (optional): The list of media chunks. - LiveClientRealtimeInput({this.mediaChunks}); + LiveClientRealtimeInput({ + @Deprecated('Use audio, video, or text instead') this.mediaChunks, + this.audio, + this.video, + this.text, + }); + + /// Creates a [LiveClientRealtimeInput] with audio data. + LiveClientRealtimeInput.audio(this.audio) + // ignore: deprecated_member_use_from_same_package + : mediaChunks = null, + video = null, + text = null; + + /// Creates a [LiveClientRealtimeInput] with video data. + LiveClientRealtimeInput.video(this.video) + // ignore: deprecated_member_use_from_same_package + : mediaChunks = null, + audio = null, + text = null; + + /// Creates a [LiveClientRealtimeInput] with text data. + LiveClientRealtimeInput.text(this.text) + // ignore: deprecated_member_use_from_same_package + : mediaChunks = null, + audio = null, + video = null; /// The list of media chunks. + @Deprecated('Use audio, video, or text instead') final List? mediaChunks; + /// Audio data. + final InlineDataPart? audio; + + /// Video data. + final InlineDataPart? video; + + /// Text data. + final String? text; + // ignore: public_member_api_docs Map toJson() => { 'realtime_input': { 'media_chunks': + // ignore: deprecated_member_use_from_same_package mediaChunks?.map((e) => e.toMediaChunkJson()).toList(), + if (audio != null) 'audio': audio!.toMediaChunkJson(), + if (video != null) 'video': video!.toMediaChunkJson(), + if (text != null) 'text': text, }, }; } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index 20e700bc82bf..f136a644d03d 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -76,9 +76,47 @@ class LiveSession { _ws.sink.add(clientJson); } + /// Sends audio data to the server in realtime. + /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. + /// [audio]: The audio data to send. + Future sendAudioRealtime(InlineDataPart audio) async { + _checkWsStatus(); + var clientMessage = LiveClientRealtimeInput.audio(audio); + var clientJson = jsonEncode(clientMessage.toJson()); + _ws.sink.add(clientJson); + } + + /// Sends video data to the server in realtime. + /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. + /// [video]: The video data to send. + Future sendVideoRealtime(InlineDataPart video) async { + _checkWsStatus(); + var clientMessage = LiveClientRealtimeInput.video(video); + var clientJson = jsonEncode(clientMessage.toJson()); + _ws.sink.add(clientJson); + } + + /// Sends text data to the server in realtime. + /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. + /// [text]: The text data to send. + Future sendTextRealtime(String text) async { + _checkWsStatus(); + var clientMessage = LiveClientRealtimeInput.text(text); + var clientJson = jsonEncode(clientMessage.toJson()); + _ws.sink.add(clientJson); + } + /// Sends realtime input (media chunks) to the server. /// /// [mediaChunks]: The list of media chunks to send. + @Deprecated( + 'Use sendAudioRealtime, sendVideoRealtime, or sendTextRealtime instead') Future sendMediaChunks({ required List mediaChunks, }) async { @@ -95,6 +133,7 @@ class LiveSession { /// /// Parameters: /// - [mediaChunkStream]: The stream of [InlineDataPart] objects to send to the server. + @Deprecated('Use sendAudio, sendVideo, or sendText with a stream instead') Future sendMediaStream(Stream mediaChunkStream) async { _checkWsStatus(); @@ -111,6 +150,7 @@ class LiveSession { Future _sendMediaChunk(InlineDataPart chunk) async { var clientMessage = LiveClientRealtimeInput( + // ignore: deprecated_member_use_from_same_package mediaChunks: [chunk]); // Create a list with the single chunk var clientJson = jsonEncode(clientMessage.toJson()); _ws.sink.add(clientJson); diff --git a/packages/firebase_ai/firebase_ai/test/live_test.dart b/packages/firebase_ai/firebase_ai/test/live_test.dart index a89c53e7901a..6b4b6b9c290f 100644 --- a/packages/firebase_ai/firebase_ai/test/live_test.dart +++ b/packages/firebase_ai/firebase_ai/test/live_test.dart @@ -103,6 +103,7 @@ void main() { test('LiveClientRealtimeInput toJson() returns correct JSON', () { final part = InlineDataPart('audio/pcm', Uint8List.fromList([1, 2, 3])); + // ignore: deprecated_member_use_from_same_package final message = LiveClientRealtimeInput(mediaChunks: [part]); expect(message.toJson(), { 'realtime_input': {