Skip to content
Open
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,13 @@ class _BidiPageState extends State<BidiPage> {
itemBuilder: (context, idx) {
return MessageWidget(
text: _messages[idx].text,
image: Image.memory(
_messages[idx].imageBytes!,
cacheWidth: 400,
cacheHeight: 400,
),
image: _messages[idx].imageBytes != null
? Image.memory(
_messages[idx].imageBytes!,
cacheWidth: 400,
cacheHeight: 400,
)
: null,
isFromUser: _messages[idx].fromUser ?? false,
);
},
Expand Down Expand Up @@ -277,11 +279,9 @@ class _BidiPageState extends State<BidiPage> {
await _audioOutput.playStream();
// Map the Uint8List stream to InlineDataPart stream
if (inputStream != null) {
final inlineDataStream = inputStream.map((data) {
return InlineDataPart('audio/pcm', data);
});

await _session.sendMediaStream(inlineDataStream);
await for (final data in inputStream) {
await _session.sendAudioRealtime(InlineDataPart('audio/pcm', data));
}
}
} catch (e) {
developer.log(e.toString());
Expand Down
44 changes: 41 additions & 3 deletions packages/firebase_ai/firebase_ai/lib/src/live_api.dart
Original file line number Diff line number Diff line change
Expand Up @@ -179,18 +179,56 @@ class LiveServerResponse {
/// Represents realtime input from the client in a live stream.
class LiveClientRealtimeInput {
/// Creates a [LiveClientRealtimeInput] instance.
///
/// [mediaChunks] (optional): The list of media chunks.
LiveClientRealtimeInput({this.mediaChunks});
LiveClientRealtimeInput({
@Deprecated('Use audio, video, or text instead') this.mediaChunks,
this.audio,
this.video,
this.text,
});

/// Creates a [LiveClientRealtimeInput] with audio data.
LiveClientRealtimeInput.audio(this.audio)
// ignore: deprecated_member_use_from_same_package
: mediaChunks = null,
video = null,
text = null;

/// Creates a [LiveClientRealtimeInput] with video data.
LiveClientRealtimeInput.video(this.video)
// ignore: deprecated_member_use_from_same_package
: mediaChunks = null,
audio = null,
text = null;

/// Creates a [LiveClientRealtimeInput] with text data.
LiveClientRealtimeInput.text(this.text)
// ignore: deprecated_member_use_from_same_package
: mediaChunks = null,
audio = null,
video = null;

/// The list of media chunks.
@Deprecated('Use audio, video, or text instead')
final List<InlineDataPart>? mediaChunks;

/// Audio data.
final InlineDataPart? audio;

/// Video data.
final InlineDataPart? video;

/// Text data.
final String? text;

// ignore: public_member_api_docs
Map<String, dynamic> toJson() => {
'realtime_input': {
'media_chunks':
// ignore: deprecated_member_use_from_same_package
mediaChunks?.map((e) => e.toMediaChunkJson()).toList(),
if (audio != null) 'audio': audio!.toMediaChunkJson(),
if (video != null) 'video': video!.toMediaChunkJson(),
if (text != null) 'text': text,
},
};
}
Expand Down
40 changes: 40 additions & 0 deletions packages/firebase_ai/firebase_ai/lib/src/live_session.dart
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,47 @@ class LiveSession {
_ws.sink.add(clientJson);
}

/// Sends audio data to the server in realtime.
///
/// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for
/// details about the realtime input usage.
/// [audio]: The audio data to send.
Future<void> sendAudioRealtime(InlineDataPart audio) async {
_checkWsStatus();
var clientMessage = LiveClientRealtimeInput.audio(audio);
var clientJson = jsonEncode(clientMessage.toJson());
_ws.sink.add(clientJson);
}

/// Sends video data to the server in realtime.
///
/// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for
/// details about the realtime input usage.
/// [video]: The video data to send.
Future<void> sendVideoRealtime(InlineDataPart video) async {
_checkWsStatus();
var clientMessage = LiveClientRealtimeInput.video(video);
var clientJson = jsonEncode(clientMessage.toJson());
_ws.sink.add(clientJson);
}

/// Sends text data to the server in realtime.
///
/// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for
/// details about the realtime input usage.
/// [text]: The text data to send.
Future<void> sendTextRealtime(String text) async {
_checkWsStatus();
var clientMessage = LiveClientRealtimeInput.text(text);
var clientJson = jsonEncode(clientMessage.toJson());
_ws.sink.add(clientJson);
}

/// Sends realtime input (media chunks) to the server.
///
/// [mediaChunks]: The list of media chunks to send.
@Deprecated(
'Use sendAudioRealtime, sendVideoRealtime, or sendTextRealtime instead')
Future<void> sendMediaChunks({
required List<InlineDataPart> mediaChunks,
}) async {
Expand All @@ -95,6 +133,7 @@ class LiveSession {
///
/// Parameters:
/// - [mediaChunkStream]: The stream of [InlineDataPart] objects to send to the server.
@Deprecated('Use sendAudio, sendVideo, or sendText with a stream instead')
Future<void> sendMediaStream(Stream<InlineDataPart> mediaChunkStream) async {
_checkWsStatus();

Expand All @@ -111,6 +150,7 @@ class LiveSession {

Future<void> _sendMediaChunk(InlineDataPart chunk) async {
var clientMessage = LiveClientRealtimeInput(
// ignore: deprecated_member_use_from_same_package
mediaChunks: [chunk]); // Create a list with the single chunk
var clientJson = jsonEncode(clientMessage.toJson());
_ws.sink.add(clientJson);
Expand Down
1 change: 1 addition & 0 deletions packages/firebase_ai/firebase_ai/test/live_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ void main() {

test('LiveClientRealtimeInput toJson() returns correct JSON', () {
final part = InlineDataPart('audio/pcm', Uint8List.fromList([1, 2, 3]));
// ignore: deprecated_member_use_from_same_package
final message = LiveClientRealtimeInput(mediaChunks: [part]);
expect(message.toJson(), {
'realtime_input': {
Expand Down
Loading