Skip to content

Commit 22e527b

Browse files
daymxnandrewheardncooke3gemini-code-assist[bot]
authored
feat(ai): Live API (#15309)
Co-authored-by: Andrew Heard <[email protected]> Co-authored-by: Nick Cooke <[email protected]> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
1 parent ead5eb4 commit 22e527b

39 files changed

+2387
-52
lines changed

FirebaseAI/CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,12 @@
1111
- [fixed] Fixed a decoding error when generating images with the
1212
`gemini-2.5-flash-image-preview` model using `generateContentStream` or
1313
`sendMessageStream` with the Gemini Developer API. (#15262)
14+
- [feature] Added support for the Live API, which allows bidirectional
15+
communication with the model in realtime.
16+
17+
To get started with the Live API, see the Firebase docs on
18+
[Bidirectional streaming using the Gemini Live API](https://firebase.google.com/docs/ai-logic/live-api).
19+
(#15309)
1420

1521
# 12.2.0
1622
- [feature] Added support for returning thought summaries, which are synthesized

FirebaseAI/Sources/AILog.swift

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,25 @@ enum AILog {
6767
case executableCodeUnrecognizedLanguage = 3016
6868
case fallbackValueUsed = 3017
6969
case urlMetadataUnrecognizedURLRetrievalStatus = 3018
70+
case liveSessionUnsupportedMessage = 3019
71+
case liveSessionUnsupportedMessagePayload = 3020
72+
case liveSessionFailedToEncodeClientMessage = 3021
73+
case liveSessionFailedToEncodeClientMessagePayload = 3022
74+
case liveSessionFailedToSendClientMessage = 3023
75+
case liveSessionUnexpectedResponse = 3024
76+
case liveSessionGoingAwaySoon = 3025
77+
case decodedMissingProtoDurationSuffix = 3026
78+
case decodedInvalidProtoDurationString = 3027
79+
case decodedInvalidProtoDurationSeconds = 3028
80+
case decodedInvalidProtoDurationNanoseconds = 3029
7081

7182
// SDK State Errors
7283
case generateContentResponseNoCandidates = 4000
7384
case generateContentResponseNoText = 4001
7485
case appCheckTokenFetchFailed = 4002
7586
case generateContentResponseEmptyCandidates = 4003
87+
case invalidWebsocketURL = 4004
88+
case duplicateLiveSessionSetupComplete = 4005
7689

7790
// SDK Debugging
7891
case loadRequestStreamResponseLine = 5000

FirebaseAI/Sources/FirebaseAI.swift

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,46 @@ public final class FirebaseAI: Sendable {
137137
)
138138
}
139139

140+
/// **[Public Preview]** Initializes a ``LiveGenerativeModel`` with the given parameters.
141+
///
142+
/// > Warning: Using the Firebase AI Logic SDKs with the Gemini Live API is in Public
143+
/// Preview, which means that the feature is not subject to any SLA or deprecation policy and
144+
/// could change in backwards-incompatible ways.
145+
///
146+
/// > Important: Only models that support the Gemini Live API (typically containing `live-*` in
147+
/// the name) are supported.
148+
///
149+
/// - Parameters:
150+
/// - modelName: The name of the model to use, for example
151+
/// `"gemini-live-2.5-flash-preview"`;
152+
/// see [model versions](https://firebase.google.com/docs/ai-logic/live-api?api=dev#models-that-support-capability)
153+
/// for a list of supported models.
154+
/// - generationConfig: The content generation parameters your model should use.
155+
/// - tools: A list of ``Tool`` objects that the model may use to generate the next response.
156+
/// - toolConfig: Tool configuration for any ``Tool`` specified in the request.
157+
/// - systemInstruction: Instructions that direct the model to behave a certain way; currently
158+
/// only text content is supported.
159+
/// - requestOptions: Configuration parameters for sending requests to the backend.
160+
@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, *)
161+
@available(watchOS, unavailable)
162+
public func liveModel(modelName: String,
163+
generationConfig: LiveGenerationConfig? = nil,
164+
tools: [Tool]? = nil,
165+
toolConfig: ToolConfig? = nil,
166+
systemInstruction: ModelContent? = nil,
167+
requestOptions: RequestOptions = RequestOptions()) -> LiveGenerativeModel {
168+
return LiveGenerativeModel(
169+
modelResourceName: modelResourceName(modelName: modelName),
170+
firebaseInfo: firebaseInfo,
171+
apiConfig: apiConfig,
172+
generationConfig: generationConfig,
173+
tools: tools,
174+
toolConfig: toolConfig,
175+
systemInstruction: systemInstruction,
176+
requestOptions: requestOptions
177+
)
178+
}
179+
140180
/// Class to enable FirebaseAI to register via the Objective-C based Firebase component system
141181
/// to include FirebaseAI in the userAgent.
142182
@objc(FIRVertexAIComponent) class FirebaseVertexAIComponent: NSObject {}

FirebaseAI/Sources/GenerativeAIService.swift

Lines changed: 4 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,10 @@ struct GenerativeAIService {
177177
urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type")
178178

179179
if let appCheck = firebaseInfo.appCheck {
180-
let tokenResult = try await fetchAppCheckToken(appCheck: appCheck)
180+
let tokenResult = try await appCheck.fetchAppCheckToken(
181+
limitedUse: firebaseInfo.useLimitedUseAppCheckTokens,
182+
domain: "GenerativeAIService"
183+
)
181184
urlRequest.setValue(tokenResult.token, forHTTPHeaderField: "X-Firebase-AppCheck")
182185
if let error = tokenResult.error {
183186
AILog.error(
@@ -207,53 +210,6 @@ struct GenerativeAIService {
207210
return urlRequest
208211
}
209212

210-
private func fetchAppCheckToken(appCheck: AppCheckInterop) async throws
211-
-> FIRAppCheckTokenResultInterop {
212-
if firebaseInfo.useLimitedUseAppCheckTokens {
213-
if let token = await getLimitedUseAppCheckToken(appCheck: appCheck) {
214-
return token
215-
}
216-
217-
let errorMessage =
218-
"The provided App Check token provider doesn't implement getLimitedUseToken(), but requireLimitedUseTokens was enabled."
219-
220-
#if Debug
221-
fatalError(errorMessage)
222-
#else
223-
throw NSError(
224-
domain: "\(Constants.baseErrorDomain).\(Self.self)",
225-
code: AILog.MessageCode.appCheckTokenFetchFailed.rawValue,
226-
userInfo: [NSLocalizedDescriptionKey: errorMessage]
227-
)
228-
#endif
229-
}
230-
231-
return await appCheck.getToken(forcingRefresh: false)
232-
}
233-
234-
private func getLimitedUseAppCheckToken(appCheck: AppCheckInterop) async
235-
-> FIRAppCheckTokenResultInterop? {
236-
// At the moment, `await` doesn’t get along with Objective-C’s optional protocol methods.
237-
await withCheckedContinuation { (continuation: CheckedContinuation<
238-
FIRAppCheckTokenResultInterop?,
239-
Never
240-
>) in
241-
guard
242-
firebaseInfo.useLimitedUseAppCheckTokens,
243-
// `getLimitedUseToken(completion:)` is an optional protocol method. Optional binding
244-
// is performed to make sure `continuation` is called even if the method’s not implemented.
245-
let limitedUseTokenClosure = appCheck.getLimitedUseToken
246-
else {
247-
return continuation.resume(returning: nil)
248-
}
249-
250-
limitedUseTokenClosure { tokenResult in
251-
// The placeholder token should be used in the case of App Check error.
252-
continuation.resume(returning: tokenResult)
253-
}
254-
}
255-
}
256-
257213
private func httpResponse(urlResponse: URLResponse) throws -> HTTPURLResponse {
258214
// The following condition should always be true: "Whenever you make HTTP URL load requests, any
259215
// response objects you get back from the URLSession, NSURLConnection, or NSURLDownload class

FirebaseAI/Sources/Types/Internal/APIConfig.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ extension APIConfig {
6868

6969
extension APIConfig.Service {
7070
/// Network addresses for generative AI API services.
71+
// TODO: maybe remove the https:// prefix and just add it as needed? websockets use these too.
7172
enum Endpoint: String, Encodable {
7273
/// The Firebase proxy production endpoint.
7374
///
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import FirebaseAppCheckInterop
16+
17+
/// Internal helper extension for fetching app check tokens.
18+
///
19+
/// Provides a common means for fetching limited use tokens, and falling back to standard tokens
20+
/// when it's disabled (or in debug mode). This also centrializes the error, since this method is
21+
/// used in multiple places.
22+
extension AppCheckInterop {
23+
/// Fetch the appcheck token.
24+
///
25+
/// - Parameters:
26+
/// - limitedUse: Should the token be a limited-use token, or a standard token.
27+
/// - domain: A string dictating where this method is being called from. Used in any thrown
28+
/// errors, to avoid hard-to-parse traces.
29+
func fetchAppCheckToken(limitedUse: Bool,
30+
domain: String) async throws -> FIRAppCheckTokenResultInterop {
31+
if limitedUse {
32+
if let token = await getLimitedUseTokenAsync() {
33+
return token
34+
}
35+
36+
let errorMessage =
37+
"The provided App Check token provider doesn't implement getLimitedUseToken(), but requireLimitedUseTokens was enabled."
38+
39+
#if Debug
40+
fatalError(errorMessage)
41+
#else
42+
throw NSError(
43+
domain: "\(Constants.baseErrorDomain).\(domain)",
44+
code: AILog.MessageCode.appCheckTokenFetchFailed.rawValue,
45+
userInfo: [NSLocalizedDescriptionKey: errorMessage]
46+
)
47+
#endif
48+
}
49+
50+
return await getToken(forcingRefresh: false)
51+
}
52+
53+
private func getLimitedUseTokenAsync() async
54+
-> FIRAppCheckTokenResultInterop? {
55+
// At the moment, `await` doesn’t get along with Objective-C’s optional protocol methods.
56+
await withCheckedContinuation { (continuation: CheckedContinuation<
57+
FIRAppCheckTokenResultInterop?,
58+
Never
59+
>) in
60+
guard
61+
// `getLimitedUseToken(completion:)` is an optional protocol method. Optional binding
62+
// is performed to make sure `continuation` is called even if the method’s not implemented.
63+
let limitedUseTokenClosure = getLimitedUseToken
64+
else {
65+
return continuation.resume(returning: nil)
66+
}
67+
68+
limitedUseTokenClosure { tokenResult in
69+
// The placeholder token should be used in the case of App Check error.
70+
continuation.resume(returning: tokenResult)
71+
}
72+
}
73+
}
74+
}

FirebaseAI/Sources/Types/Internal/InternalPart.swift

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,21 +45,25 @@ struct FileData: Codable, Equatable, Sendable {
4545
struct FunctionCall: Equatable, Sendable {
4646
let name: String
4747
let args: JSONObject
48+
let id: String?
4849

49-
init(name: String, args: JSONObject) {
50+
init(name: String, args: JSONObject, id: String?) {
5051
self.name = name
5152
self.args = args
53+
self.id = id
5254
}
5355
}
5456

5557
@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
5658
struct FunctionResponse: Codable, Equatable, Sendable {
5759
let name: String
5860
let response: JSONObject
61+
let id: String?
5962

60-
init(name: String, response: JSONObject) {
63+
init(name: String, response: JSONObject, id: String? = nil) {
6164
self.name = name
6265
self.response = response
66+
self.id = id
6367
}
6468
}
6569

@@ -135,6 +139,7 @@ extension FunctionCall: Codable {
135139
} else {
136140
args = JSONObject()
137141
}
142+
id = try container.decodeIfPresent(String.self, forKey: .id)
138143
}
139144
}
140145

0 commit comments

Comments
 (0)