Update

Author: daymxn

FirebaseAI/Sources/AILog.swift

@@ -66,12 +66,21 @@ enum AILog {
     case codeExecutionResultUnrecognizedOutcome = 3015
     case executableCodeUnrecognizedLanguage = 3016
     case fallbackValueUsed = 3017
+    case liveSessionUnsupportedMessage = 3018
+    case liveSessionFailedToEncodeClientMessage = 3019
+    case liveSessionFailedToEncodeClientMessagePayload = 3020
+    case liveSessionFailedToSendClientMessage = 3021
+    case liveSessionUnexpectedResponse = 3022
+
 
     // SDK State Errors
     case generateContentResponseNoCandidates = 4000
     case generateContentResponseNoText = 4001
     case appCheckTokenFetchFailed = 4002
     case generateContentResponseEmptyCandidates = 4003
+    case invalidWebsocketURL = 4004
+    case duplicateLiveSessionSetupComplete = 4005
+
 
     // SDK Debugging
     case loadRequestStreamResponseLine = 5000

FirebaseAI/Sources/FirebaseAI.swift

@@ -141,15 +141,38 @@ public final class FirebaseAI: Sendable {
     )
   }
 
+  /// **[Public Preview]** Initializes a ``LiveGenerativeModel`` with the given parameters.
+  ///
+  /// > Warning: For Firebase AI SDK, bidirectional streaming using Live models is in Public
+  /// Preview, which means that the feature is not subject to any SLA or deprecation policy and
+  /// could change in backwards-incompatible ways.
+  ///
+  /// > Important: Only Live models (typically containing `live-*` in the name) are supported.
+  ///
+  /// - Parameters:
+  ///   - modelName: The name of the Livemodel to use, for example
+  /// `"gemini-live-2.5-flash-preview"`;
+  ///     see [model versions](https://firebase.google.com/docs/ai-logic/live-api?api=dev#models-that-support-capability)
+  ///     for a list of supported Live models.
+  ///   - generationConfig: The content generation parameters your model should use.
+  ///   - tools: A list of ``Tool`` objects that the model may use to generate the next response.
+  ///   - toolConfig: Tool configuration for any ``Tool`` specified in the request.
+  ///   - systemInstruction: Instructions that direct the model to behave a certain way; currently
+  ///     only text content is supported.
   public func liveModel(modelName: String,
                         generationConfig: LiveGenerationConfig? = nil,
+                        tools: [Tool]? = nil,
+                        toolConfig: ToolConfig? = nil,
+                        systemInstruction: ModelContent? = nil,
                         requestOptions: RequestOptions = RequestOptions()) -> LiveGenerativeModel {
     return LiveGenerativeModel(
       modelResourceName: modelResourceName(modelName: modelName),
       firebaseInfo: firebaseInfo,
       apiConfig: apiConfig,
       generationConfig: generationConfig,
-      requestOptions: requestOptions
+      tools: tools,
+      toolConfig: toolConfig,
+      systemInstruction: systemInstruction
     )
   }
 

FirebaseAI/Sources/GenerativeAIService.swift

@@ -177,7 +177,10 @@ struct GenerativeAIService {
     urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type")
 
     if let appCheck = firebaseInfo.appCheck {
-      let tokenResult = try await fetchAppCheckToken(appCheck: appCheck)
+      let tokenResult = try await appCheck.fetchAppCheckToken(
+        limitedUse: firebaseInfo.useLimitedUseAppCheckTokens,
+        domain: "GenerativeAIService"
+      )
       urlRequest.setValue(tokenResult.token, forHTTPHeaderField: "X-Firebase-AppCheck")
       if let error = tokenResult.error {
         AILog.error(
@@ -207,53 +210,6 @@ struct GenerativeAIService {
     return urlRequest
   }
 
-  private func fetchAppCheckToken(appCheck: AppCheckInterop) async throws
-    -> FIRAppCheckTokenResultInterop {
-    if firebaseInfo.useLimitedUseAppCheckTokens {
-      if let token = await getLimitedUseAppCheckToken(appCheck: appCheck) {
-        return token
-      }
-
-      let errorMessage =
-        "The provided App Check token provider doesn't implement getLimitedUseToken(), but requireLimitedUseTokens was enabled."
-
-      #if Debug
-        fatalError(errorMessage)
-      #else
-        throw NSError(
-          domain: "\(Constants.baseErrorDomain).\(Self.self)",
-          code: AILog.MessageCode.appCheckTokenFetchFailed.rawValue,
-          userInfo: [NSLocalizedDescriptionKey: errorMessage]
-        )
-      #endif
-    }
-
-    return await appCheck.getToken(forcingRefresh: false)
-  }
-
-  private func getLimitedUseAppCheckToken(appCheck: AppCheckInterop) async
-    -> FIRAppCheckTokenResultInterop? {
-    // At the moment, `await` doesn’t get along with Objective-C’s optional protocol methods.
-    await withCheckedContinuation { (continuation: CheckedContinuation<
-      FIRAppCheckTokenResultInterop?,
-      Never
-    >) in
-      guard
-        firebaseInfo.useLimitedUseAppCheckTokens,
-        // `getLimitedUseToken(completion:)` is an optional protocol method. Optional binding
-        // is performed to make sure `continuation` is called even if the method’s not implemented.
-        let limitedUseTokenClosure = appCheck.getLimitedUseToken
-      else {
-        return continuation.resume(returning: nil)
-      }
-
-      limitedUseTokenClosure { tokenResult in
-        // The placeholder token should be used in the case of App Check error.
-        continuation.resume(returning: tokenResult)
-      }
-    }
-  }
-
   private func httpResponse(urlResponse: URLResponse) throws -> HTTPURLResponse {
     // The following condition should always be true: "Whenever you make HTTP URL load requests, any
     // response objects you get back from the URLSession, NSURLConnection, or NSURLDownload class

FirebaseAI/Sources/Types/Internal/AppCheck.swift

@@ -0,0 +1,65 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import FirebaseAppCheckInterop
+
+// TODO: document
+internal extension AppCheckInterop {
+  // TODO: Document
+  func fetchAppCheckToken(limitedUse: Bool,
+                          domain: String) async throws -> FIRAppCheckTokenResultInterop {
+    if limitedUse {
+      if let token = await getLimitedUseTokenAsync() {
+        return token
+      }
+
+      let errorMessage =
+        "The provided App Check token provider doesn't implement getLimitedUseToken(), but requireLimitedUseTokens was enabled."
+
+      #if Debug
+        fatalError(errorMessage)
+      #else
+        throw NSError(
+          domain: "\(Constants.baseErrorDomain).\(domain)",
+          code: AILog.MessageCode.appCheckTokenFetchFailed.rawValue,
+          userInfo: [NSLocalizedDescriptionKey: errorMessage]
+        )
+      #endif
+    }
+
+    return await getToken(forcingRefresh: false)
+  }
+
+  private func getLimitedUseTokenAsync() async
+    -> FIRAppCheckTokenResultInterop? {
+    // At the moment, `await` doesn’t get along with Objective-C’s optional protocol methods.
+    await withCheckedContinuation { (continuation: CheckedContinuation<
+      FIRAppCheckTokenResultInterop?,
+      Never
+    >) in
+      guard
+        // `getLimitedUseToken(completion:)` is an optional protocol method. Optional binding
+        // is performed to make sure `continuation` is called even if the method’s not implemented.
+        let limitedUseTokenClosure = getLimitedUseToken
+      else {
+        return continuation.resume(returning: nil)
+      }
+
+      limitedUseTokenClosure { tokenResult in
+        // The placeholder token should be used in the case of App Check error.
+        continuation.resume(returning: tokenResult)
+      }
+    }
+  }
+}

FirebaseAI/Sources/Types/Internal/InternalPart.swift

@@ -56,10 +56,12 @@ struct FunctionCall: Equatable, Sendable {
 struct FunctionResponse: Codable, Equatable, Sendable {
   let name: String
   let response: JSONObject
+  let id: String?
 
-  init(name: String, response: JSONObject) {
+  init(name: String, response: JSONObject, id: String? = nil) {
     self.name = name
     self.response = response
+    self.id = id
   }
 }
 

FirebaseAI/Sources/Types/Internal/Live/AsyncWebSocket.swift

@@ -21,11 +21,11 @@ final class AsyncWebSocket: NSObject, @unchecked Sendable, URLSessionWebSocketDe
   private var continuationFinished = false
   private let continuationLock = NSLock()
 
-  private var _isConnected = false
-  private let isConnectedLock = NSLock()
-  private(set) var isConnected: Bool {
-    get { isConnectedLock.withLock { _isConnected } }
-    set { isConnectedLock.withLock { _isConnected = newValue } }
+  private var _closeError: WebSocketClosedError? = nil
+  private let closeErrorLock = NSLock()
+  private(set) var closeError: WebSocketClosedError? {
+    get { closeErrorLock.withLock { _closeError } }
+    set { closeErrorLock.withLock { _closeError = newValue } }
   }
 
   init(urlSession: URLSession = GenAIURLSession.default, urlRequest: URLRequest) {
@@ -40,44 +40,55 @@ final class AsyncWebSocket: NSObject, @unchecked Sendable, URLSessionWebSocketDe
 
   func connect() -> AsyncThrowingStream<URLSessionWebSocketTask.Message, Error> {
     webSocketTask.resume()
-    isConnected = true
+    closeError = nil
     startReceiving()
     return stream
   }
 
   func disconnect() {
-    webSocketTask.cancel(with: .goingAway, reason: nil)
-    isConnected = false
-    continuationLock.withLock {
-      self.continuation.finish()
-      self.continuationFinished = true
-    }
+    if let closeError { return }
+
+    close(code: .goingAway, reason: nil)
   }
 
   func send(_ message: URLSessionWebSocketTask.Message) async throws {
-    // TODO: Throw error if socket already closed
+    if let closeError {
+      throw closeError
+    }
     try await webSocketTask.send(message)
   }
 
   private func startReceiving() {
     Task {
-      while !Task.isCancelled && self.webSocketTask.isOpen && self.isConnected {
-        let message = try await webSocketTask.receive()
-        // TODO: Check continuationFinished before yielding. Use the same thread for NSLock.
-        continuation.yield(message)
+      while !Task.isCancelled && self.webSocketTask.isOpen && self.closeError == nil {
+        do {
+          let message = try await webSocketTask.receive()
+          continuation.yield(message)
+        } catch {
+          close(code: webSocketTask.closeCode, reason: webSocketTask.closeReason)
+        }
       }
     }
   }
 
+  private func close(code: URLSessionWebSocketTask.CloseCode, reason: Data?) {
+    let error = WebSocketClosedError(closeCode: code, closeReason: reason)
+    closeError = error
+
+    webSocketTask.cancel(with: code, reason: reason)
+
+    continuationLock.withLock {
+      guard !continuationFinished else { return }
+      self.continuation.finish(throwing: error)
+      self.continuationFinished = true
+    }
+  }
+
   func urlSession(_ session: URLSession,
                   webSocketTask: URLSessionWebSocketTask,
                   didCloseWith closeCode: URLSessionWebSocketTask.CloseCode,
                   reason: Data?) {
-    continuationLock.withLock {
-      guard !continuationFinished else { return }
-      continuation.finish()
-      continuationFinished = true
-    }
+    close(code: closeCode, reason: reason)
   }
 }
 

FirebaseAI/Sources/Types/Internal/Live/BidiGenerateContentRealtimeInput.swift

@@ -43,7 +43,7 @@ struct BidiGenerateContentRealtimeInput: Encodable {
   let audioStreamEnd: Bool?
 
   /// These form the realtime video input stream.
-  let video: Data?
+  let video: InlineData?
 
   /// These form the realtime text input stream.
   let text: String?
@@ -61,4 +61,15 @@ struct BidiGenerateContentRealtimeInput: Encodable {
   /// Marks the end of user activity. This can only be sent if automatic (i.e.
   // server-side) activity detection is disabled.
   let activityEnd: ActivityEnd?
+
+  init(audio: InlineData? = nil, video: InlineData? = nil, text: String? = nil,
+       activityStart: ActivityStart? = nil, activityEnd: ActivityEnd? = nil,
+       audioStreamEnd: Bool? = nil) {
+    self.audio = audio
+    self.video = video
+    self.text = text
+    self.activityStart = activityStart
+    self.activityEnd = activityEnd
+    self.audioStreamEnd = audioStreamEnd
+  }
 }

FirebaseAI/Sources/Types/Internal/Live/BidiGenerateContentServerContent.swift

@@ -20,7 +20,7 @@ import Foundation
 /// Content is generated as quickly as possible, and not in realtime. Clients
 /// may choose to buffer and play it out in realtime.
 @available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
-struct BidiGenerateContentServerContent: Decodable {
+struct BidiGenerateContentServerContent: Decodable, Sendable {
   /// The content that the model has generated as part of the current
   /// conversation with the user.
   let modelTurn: ModelContent?
@@ -50,4 +50,6 @@ struct BidiGenerateContentServerContent: Decodable {
 
   /// Metadata specifies sources used to ground generated content.
   let groundingMetadata: GroundingMetadata?
+
+  let outputTranscription: BidiGenerateContentTranscription?
 }

FirebaseAI/Sources/Types/Internal/Live/BidiGenerateContentServerMessage.swift

@@ -16,11 +16,11 @@ import Foundation
 
 /// Response message for BidiGenerateContent RPC call.
 @available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
-public struct BidiGenerateContentServerMessage: Sendable {
+struct BidiGenerateContentServerMessage: Sendable {
   // TODO: Make this type `internal`
 
   /// The type of the message.
-  enum MessageType {
+  enum MessageType: Sendable {
     /// Sent in response to a `BidiGenerateContentSetup` message from the client.
     case setupComplete(BidiGenerateContentSetupComplete)
 

FirebaseAI/Sources/Types/Internal/Live/BidiGenerateContentSetup.swift

@@ -43,18 +43,32 @@ struct BidiGenerateContentSetup: Encodable {
   /// knowledge and scope of the model.
   let tools: [Tool]?
 
+  let toolConfig: ToolConfig?
+
   /// Configures the handling of realtime input.
   let realtimeInputConfig: RealtimeInputConfig?
 
+  let inputAudioTranscription: AudioTranscriptionConfig?
+
+  let outputAudioTranscription: AudioTranscriptionConfig?
+
   init(model: String,
        generationConfig: LiveGenerationConfig? = nil,
        systemInstruction: ModelContent? = nil,
        tools: [Tool]? = nil,
-       realtimeInputConfig: RealtimeInputConfig? = nil) {
+       toolConfig: ToolConfig? = nil,
+       realtimeInputConfig: RealtimeInputConfig? = nil,
+       inputAudioTranscription: AudioTranscriptionConfig? = nil,
+       outputAudioTranscription: AudioTranscriptionConfig? = nil) {
     self.model = model
     self.generationConfig = generationConfig
     self.systemInstruction = systemInstruction
     self.tools = tools
+    self.toolConfig = toolConfig
     self.realtimeInputConfig = realtimeInputConfig
+    self.inputAudioTranscription = inputAudioTranscription
+    self.outputAudioTranscription = outputAudioTranscription
   }
 }
+
+struct AudioTranscriptionConfig: Encodable {}