feat(firebaseai): make Live API working with developer API (#17503)

cynthiajoan · google-labs-jules[bot] · web-flow · commit 467eaa181025 · 2025-08-08T10:16:34.000-07:00
* add developer api to live

* make the whole process going through

* feat(firebase_ai): handle unknown parts when parsing content

* tweak the content test

* remove the extra exception

* Make Live API sending tool response for function calling

* update bidi model for googleAI in example

* fix analyzer and test

* more test fixing

---------

Co-authored-by: google-labs-jules[bot] &lt;161369871+google-labs-jules[bot]@users.noreply.github.com&gt;
diff --git a/packages/firebase_ai/firebase_ai/example/lib/main.dart b/packages/firebase_ai/firebase_ai/example/lib/main.dart
@@ -154,10 +154,6 @@ class HomeScreen extends StatefulWidget {
 
 class _HomeScreenState extends State<HomeScreen> {
   void _onItemTapped(int index) {
-    if (index == 9 && !widget.useVertexBackend) {
-      // Live Stream feature only works with Vertex AI now.
-      return;
-    }
     widget.onSelectedIndexChanged(index);
   }
 
@@ -192,12 +188,12 @@ class _HomeScreenState extends State<HomeScreen> {
       case 8:
         return VideoPage(title: 'Video Prompt', model: currentModel);
       case 9:
-        if (useVertexBackend) {
-          return BidiPage(title: 'Live Stream', model: currentModel);
-        } else {
-          // Fallback to the first page in case of an unexpected index
-          return ChatPage(title: 'Chat', model: currentModel);
-        }
+        return BidiPage(
+          title: 'Live Stream',
+          model: currentModel,
+          useVertexBackend: useVertexBackend,
+        );
+
       default:
         // Fallback to the first page in case of an unexpected index
         return ChatPage(title: 'Chat', model: currentModel);
@@ -270,61 +266,58 @@ class _HomeScreenState extends State<HomeScreen> {
         unselectedItemColor: widget.useVertexBackend
             ? Theme.of(context).colorScheme.onSurface.withValues(alpha: 0.7)
             : Colors.grey,
-        items: <BottomNavigationBarItem>[
-          const BottomNavigationBarItem(
+        items: const <BottomNavigationBarItem>[
+          BottomNavigationBarItem(
             icon: Icon(Icons.chat),
             label: 'Chat',
             tooltip: 'Chat',
           ),
-          const BottomNavigationBarItem(
+          BottomNavigationBarItem(
             icon: Icon(Icons.mic),
             label: 'Audio',
             tooltip: 'Audio Prompt',
           ),
-          const BottomNavigationBarItem(
+          BottomNavigationBarItem(
             icon: Icon(Icons.numbers),
             label: 'Tokens',
             tooltip: 'Token Count',
           ),
-          const BottomNavigationBarItem(
+          BottomNavigationBarItem(
             icon: Icon(Icons.functions),
             label: 'Functions',
             tooltip: 'Function Calling',
           ),
-          const BottomNavigationBarItem(
+          BottomNavigationBarItem(
             icon: Icon(Icons.image),
             label: 'Image',
             tooltip: 'Image Prompt',
           ),
-          const BottomNavigationBarItem(
+          BottomNavigationBarItem(
             icon: Icon(Icons.image_search),
             label: 'Imagen',
             tooltip: 'Imagen Model',
           ),
-          const BottomNavigationBarItem(
+          BottomNavigationBarItem(
             icon: Icon(Icons.schema),
             label: 'Schema',
             tooltip: 'Schema Prompt',
           ),
-          const BottomNavigationBarItem(
+          BottomNavigationBarItem(
             icon: Icon(Icons.edit_document),
             label: 'Document',
             tooltip: 'Document Prompt',
           ),
-          const BottomNavigationBarItem(
+          BottomNavigationBarItem(
             icon: Icon(Icons.video_collection),
             label: 'Video',
             tooltip: 'Video Prompt',
           ),
           BottomNavigationBarItem(
             icon: Icon(
               Icons.stream,
-              color: widget.useVertexBackend ? null : Colors.grey,
             ),
             label: 'Live',
-            tooltip: widget.useVertexBackend
-                ? 'Live Stream'
-                : 'Live Stream (Currently Disabled)',
+            tooltip: 'Live Stream',
           ),
         ],
         currentIndex: widget.selectedIndex,
diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart
@@ -22,10 +22,16 @@ import '../utils/audio_output.dart';
 import '../widgets/message_widget.dart';
 
 class BidiPage extends StatefulWidget {
-  const BidiPage({super.key, required this.title, required this.model});
+  const BidiPage({
+    super.key,
+    required this.title,
+    required this.model,
+    required this.useVertexBackend,
+  });
 
   final String title;
   final GenerativeModel model;
+  final bool useVertexBackend;
 
   @override
   State<BidiPage> createState() => _BidiPageState();
@@ -64,13 +70,21 @@ class _BidiPageState extends State<BidiPage> {
     );
 
     // ignore: deprecated_member_use
-    _liveModel = FirebaseAI.vertexAI().liveGenerativeModel(
-      model: 'gemini-2.0-flash-exp',
-      liveGenerationConfig: config,
-      tools: [
-        Tool.functionDeclarations([lightControlTool]),
-      ],
-    );
+    _liveModel = widget.useVertexBackend
+        ? FirebaseAI.vertexAI().liveGenerativeModel(
+            model: 'gemini-2.0-flash-exp',
+            liveGenerationConfig: config,
+            tools: [
+              Tool.functionDeclarations([lightControlTool]),
+            ],
+          )
+        : FirebaseAI.googleAI().liveGenerativeModel(
+            model: 'gemini-live-2.5-flash-preview',
+            liveGenerationConfig: config,
+            tools: [
+              Tool.functionDeclarations([lightControlTool]),
+            ],
+          );
     _initAudio();
   }
 
@@ -389,9 +403,13 @@ class _BidiPageState extends State<BidiPage> {
           brightness: brightness,
           colorTemperature: color,
         );
-        await _session.send(
-          input: Content.functionResponse(functionCall.name, functionResult),
-        );
+        await _session.sendToolResponse([
+          FunctionResponse(
+            functionCall.name,
+            functionResult,
+            id: functionCall.id,
+          ),
+        ]);
       } else {
         throw UnimplementedError(
           'Function not declared to the model: ${functionCall.name}',
diff --git a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart
@@ -56,6 +56,7 @@ enum Task {
 
 abstract interface class _ModelUri {
   String get baseAuthority;
+  String get apiVersion;
   Uri taskUri(Task task);
   ({String prefix, String name}) get model;
 }
@@ -96,6 +97,9 @@ final class _VertexUri implements _ModelUri {
   @override
   String get baseAuthority => _baseAuthority;
 
+  @override
+  String get apiVersion => _apiVersion;
+
   @override
   Uri taskUri(Task task) {
     return _projectUri.replace(
@@ -135,6 +139,9 @@ final class _GoogleAIUri implements _ModelUri {
   @override
   String get baseAuthority => _baseAuthority;
 
+  @override
+  String get apiVersion => _apiVersion;
+
   @override
   Uri taskUri(Task task) => _baseUri.replace(
       pathSegments: _baseUri.pathSegments
diff --git a/packages/firebase_ai/firebase_ai/lib/src/content.dart b/packages/firebase_ai/firebase_ai/lib/src/content.dart
@@ -48,8 +48,9 @@ final class Content {
   static Content model(Iterable<Part> parts) => Content('model', [...parts]);
 
   /// Return a [Content] with [FunctionResponse].
-  static Content functionResponse(String name, Map<String, Object?> response) =>
-      Content('function', [FunctionResponse(name, response)]);
+  static Content functionResponse(String name, Map<String, Object?> response,
+          {String? id}) =>
+      Content('function', [FunctionResponse(name, response, id: id)]);
 
   /// Return a [Content] with multiple [FunctionResponse].
   static Content functionResponses(Iterable<FunctionResponse> responses) =>
diff --git a/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart b/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart
@@ -175,14 +175,11 @@ class FirebaseAI extends FirebasePluginPlatform {
     List<Tool>? tools,
     Content? systemInstruction,
   }) {
-    if (!_useVertexBackend) {
-      throw FirebaseAISdkException(
-          'LiveGenerativeModel is currently only supported with the VertexAI backend.');
-    }
     return createLiveGenerativeModel(
       app: app,
       location: location,
       model: model,
+      useVertexBackend: _useVertexBackend,
       liveGenerationConfig: liveGenerationConfig,
       tools: tools,
       systemInstruction: systemInstruction,
diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart
@@ -234,7 +234,15 @@ class LiveClientToolResponse {
   final List<FunctionResponse>? functionResponses;
   // ignore: public_member_api_docs
   Map<String, dynamic> toJson() => {
-        'functionResponses': functionResponses?.map((e) => e.toJson()).toList(),
+        'toolResponse': {
+          'functionResponses': functionResponses
+              ?.map((e) => {
+                    'name': e.name,
+                    'response': e.response,
+                    if (e.id != null) 'id': e.id,
+                  })
+              .toList(),
+        },
       };
 }
 
diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart
@@ -15,7 +15,8 @@
 part of 'base_model.dart';
 
 const _apiUrl = 'ws/google.firebase.vertexai';
-const _apiUrlSuffix = 'LlmBidiService/BidiGenerateContent/locations';
+const _apiUrlSuffixVertexAI = 'LlmBidiService/BidiGenerateContent/locations';
+const _apiUrlSuffixGoogleAI = 'GenerativeService/BidiGenerateContent';
 
 /// A live, generative AI model for real-time interaction.
 ///
@@ -32,36 +33,56 @@ final class LiveGenerativeModel extends BaseModel {
       {required String model,
       required String location,
       required FirebaseApp app,
+      required bool useVertexBackend,
       FirebaseAppCheck? appCheck,
       FirebaseAuth? auth,
       LiveGenerationConfig? liveGenerationConfig,
       List<Tool>? tools,
       Content? systemInstruction})
       : _app = app,
         _location = location,
+        _useVertexBackend = useVertexBackend,
         _appCheck = appCheck,
         _auth = auth,
         _liveGenerationConfig = liveGenerationConfig,
         _tools = tools,
         _systemInstruction = systemInstruction,
         super._(
           serializationStrategy: VertexSerialization(),
-          modelUri: _VertexUri(
-            model: model,
-            app: app,
-            location: location,
-          ),
+          modelUri: useVertexBackend
+              ? _VertexUri(
+                  model: model,
+                  app: app,
+                  location: location,
+                )
+              : _GoogleAIUri(
+                  model: model,
+                  app: app,
+                ),
         );
-  static const _apiVersion = 'v1beta';
 
   final FirebaseApp _app;
   final String _location;
+  final bool _useVertexBackend;
   final FirebaseAppCheck? _appCheck;
   final FirebaseAuth? _auth;
   final LiveGenerationConfig? _liveGenerationConfig;
   final List<Tool>? _tools;
   final Content? _systemInstruction;
 
+  String _vertexAIUri() => 'wss://${_modelUri.baseAuthority}/'
+      '$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixVertexAI/'
+      '$_location?key=${_app.options.apiKey}';
+
+  String _vertexAIModelString() => 'projects/${_app.options.projectId}/'
+      'locations/$_location/publishers/google/models/${model.name}';
+
+  String _googleAIUri() => 'wss://${_modelUri.baseAuthority}/'
+      '$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixGoogleAI?key=${_app.options.apiKey}';
+
+  String _googleAIModelString() =>
+      'projects/${_app.options.projectId}/models/${model.name}';
+
   /// Establishes a connection to a live generation service.
   ///
   /// This function handles the WebSocket connection setup and returns an [LiveSession]
@@ -70,11 +91,9 @@ final class LiveGenerativeModel extends BaseModel {
   /// Returns a [Future] that resolves to an [LiveSession] object upon successful
   /// connection.
   Future<LiveSession> connect() async {
-    final uri = 'wss://${_modelUri.baseAuthority}/'
-        '$_apiUrl.$_apiVersion.$_apiUrlSuffix/'
-        '$_location?key=${_app.options.apiKey}';
-    final modelString = 'projects/${_app.options.projectId}/'
-        'locations/$_location/publishers/google/models/${model.name}';
+    final uri = _useVertexBackend ? _vertexAIUri() : _googleAIUri();
+    final modelString =
+        _useVertexBackend ? _vertexAIModelString() : _googleAIModelString();
 
     final setupJson = {
       'setup': {
@@ -96,6 +115,7 @@ final class LiveGenerativeModel extends BaseModel {
     await ws.ready;
 
     ws.sink.add(request);
+
     return LiveSession(ws);
   }
 }
@@ -105,6 +125,7 @@ LiveGenerativeModel createLiveGenerativeModel({
   required FirebaseApp app,
   required String location,
   required String model,
+  required bool useVertexBackend,
   FirebaseAppCheck? appCheck,
   FirebaseAuth? auth,
   LiveGenerationConfig? liveGenerationConfig,
@@ -117,6 +138,7 @@ LiveGenerativeModel createLiveGenerativeModel({
       appCheck: appCheck,
       auth: auth,
       location: location,
+      useVertexBackend: useVertexBackend,
       liveGenerationConfig: liveGenerationConfig,
       tools: tools,
       systemInstruction: systemInstruction,
diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart
@@ -61,7 +61,18 @@ class LiveSession {
         ? LiveClientContent(turns: [input], turnComplete: turnComplete)
         : LiveClientContent(turnComplete: turnComplete);
     var clientJson = jsonEncode(clientMessage.toJson());
+    _ws.sink.add(clientJson);
+  }
 
+  /// Sends tool responses for function calling to the server.
+  ///
+  /// [functionResponses] (optional): The list of function responses.
+  Future<void> sendToolResponse(
+      List<FunctionResponse>? functionResponses) async {
+    final toolResponse =
+        LiveClientToolResponse(functionResponses: functionResponses);
+    _checkWsStatus();
+    var clientJson = jsonEncode(toolResponse.toJson());
     _ws.sink.add(clientJson);
   }
 
diff --git a/packages/firebase_ai/firebase_ai/test/live_test.dart b/packages/firebase_ai/firebase_ai/test/live_test.dart
@@ -154,15 +154,17 @@ void main() {
       final response = FunctionResponse('test', {});
       final message = LiveClientToolResponse(functionResponses: [response]);
       expect(message.toJson(), {
-        'functionResponses': [
-          {
-            'functionResponse': {'name': 'test', 'response': {}}
-          }
-        ]
+        'toolResponse': {
+          'functionResponses': [
+            {'name': 'test', 'response': {}}
+          ]
+        }
       });
 
       final message2 = LiveClientToolResponse();
-      expect(message2.toJson(), {'functionResponses': null});
+      expect(message2.toJson(), {
+        'toolResponse': {'functionResponses': null}
+      });
     });
 
     test('parseServerMessage parses serverContent message correctly', () {
diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart
diff --git a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart
