Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
126 commits
Select commit Hold shift + click to select a range
441b4a7
[Firebase AI] Add starter types for Live API
andrewheard Aug 9, 2025
af76cbc
Add placeholder types for `LiveGenerativeModel` and `LiveSession`
andrewheard Aug 9, 2025
1b5d791
Fix `BidiGenerateContentClientMessage` encoding
andrewheard Aug 10, 2025
da8c21b
Fix `BidiGenerateContentServerMessage` decoding
andrewheard Aug 10, 2025
aa3d148
Add `LiveGenerationConfig` and add to setup
andrewheard Aug 10, 2025
60003c1
Add temporary state machine in `LiveSession`
andrewheard Aug 10, 2025
3f285b8
Add `liveModel` static method to construct `LiveGenerativeModel`
andrewheard Aug 10, 2025
737d823
Emit `responses` from `LiveSession`
andrewheard Aug 10, 2025
91dd102
Temporarily display text Bidi responses in TestApp
andrewheard Aug 10, 2025
dd89e20
Refactor to use async/await and remove `URLSessionWebSocketDelegate`
andrewheard Aug 10, 2025
962b57f
Add platform availability annotations
andrewheard Aug 10, 2025
95e1908
Add `BidiGenerateContentServerMessage` availability annotation
andrewheard Aug 11, 2025
c6f3ba8
Add `AsyncWebSocket` wrapper for `URLSessionWebSocketTask`
andrewheard Aug 12, 2025
a9eb3d8
Update
daymxn Sep 9, 2025
b5b0107
Add support for SpeechConfig
daymxn Sep 9, 2025
9b83676
Minor doc fixes
daymxn Sep 9, 2025
d5e375f
Add missing ID field
daymxn Sep 9, 2025
543869e
Remove old todo comment
daymxn Sep 9, 2025
d342829
Add missing docs
daymxn Sep 9, 2025
465006e
Add docs for app check extension
daymxn Sep 9, 2025
e51cc63
Add bug for session resumption
daymxn Sep 9, 2025
3e58503
Use boolean to avoid warning
daymxn Sep 10, 2025
55b8425
Log going away instead of error
daymxn Sep 10, 2025
fa4cf24
Merge branch 'main' into dlr/firebaseai-live-api
daymxn Sep 10, 2025
0c23c7d
Add additional docs for tasks
daymxn Sep 10, 2025
71acb55
Revert andrew's prototyping changes
daymxn Sep 10, 2025
2ccca17
Add support for request options
daymxn Sep 10, 2025
00ebe13
Minor doc fixes
daymxn Sep 10, 2025
3c442a5
Revert "Analytics 12.3.0 (#15310)"
daymxn Sep 10, 2025
dba2bc0
Remove default from video realtime
daymxn Sep 17, 2025
7902349
Add the close method
daymxn Sep 17, 2025
24ef7f0
Add a todo about adding start
daymxn Sep 17, 2025
c252b0a
Fix missing id usage
daymxn Sep 17, 2025
b469e9c
Formatting
daymxn Sep 17, 2025
12f9f27
Temporarily add support for function behavior
daymxn Sep 17, 2025
c4339fa
Add official support for transcripts
daymxn Sep 25, 2025
9527026
Formatting
daymxn Sep 25, 2025
48ce789
Add changelog entry
daymxn Sep 25, 2025
240b94e
Add protoduration support
daymxn Sep 26, 2025
5670c09
Remove unnecessary structs
daymxn Sep 26, 2025
a9f4961
Use unfair lock for closeError
daymxn Sep 26, 2025
e216c19
Use unfair lock for continuation finished
daymxn Sep 26, 2025
1f545da
Make closeErorr immutable
daymxn Sep 26, 2025
279b733
Merge branch 'main' into dlr/firebaseai-live-api
daymxn Sep 26, 2025
6553b5a
Update some doc formatting
daymxn Sep 26, 2025
8b32ed4
Fix duration compile error
daymxn Sep 26, 2025
6678be2
Add docs to AsyncWebSocket
daymxn Sep 26, 2025
58aab91
Remove realtime input config
daymxn Sep 26, 2025
4c742d5
Add an error for lost connection
daymxn Sep 26, 2025
2cd1544
Formatting
daymxn Sep 26, 2025
4fce708
Remove function behavior stuff
daymxn Sep 26, 2025
dd81a84
Revert analytics changes from testing
daymxn Sep 26, 2025
a574eb4
Fix id decoding
daymxn Sep 29, 2025
712b8b4
Rename id to functionId to avoid identifiable collision
daymxn Sep 29, 2025
6ec278b
Fix available version to match usage
daymxn Sep 29, 2025
ad58739
Update reference to id field in docs
daymxn Sep 29, 2025
0f4c55b
Update other doc references to id field
daymxn Sep 29, 2025
6855efc
formatting
daymxn Sep 29, 2025
d1d878a
Fix old available version
daymxn Sep 29, 2025
e3fb8e8
Possibly fix concurrency issue
daymxn Sep 29, 2025
866b32e
Allow params to be unnamed in realtime apis
daymxn Sep 30, 2025
3b896bd
Add audio modifier to transcription names
daymxn Sep 30, 2025
34f4275
Update FirebaseAI/Sources/Types/Internal/Live/LiveSessionService.swift
daymxn Sep 30, 2025
30d9896
Update FirebaseAI/Sources/Types/Internal/Live/LiveSessionService.swift
daymxn Sep 30, 2025
082ccbd
Update FirebaseAI/Sources/Types/Internal/Live/LiveSessionService.swift
daymxn Sep 30, 2025
956942e
Update FirebaseAI/Sources/Types/Internal/Live/LiveSessionService.swift
daymxn Sep 30, 2025
319ebc2
Update FirebaseAI/Sources/Types/Internal/Live/LiveSessionService.swift
daymxn Sep 30, 2025
01f18f2
Add comments explaining nested task
daymxn Sep 30, 2025
029ee0d
Use close instead on connect
daymxn Sep 30, 2025
88fa653
Move live session error to public folder
daymxn Sep 30, 2025
6332009
Use failable initializers
daymxn Sep 30, 2025
bba76c6
Formatting on comment
daymxn Sep 30, 2025
7dacdc2
Update FirebaseAI/Sources/Types/Public/Live/LiveServerContent.swift
daymxn Sep 30, 2025
e8e6370
Update FirebaseAI/Sources/Types/Public/Live/LiveServerContent.swift
daymxn Sep 30, 2025
57e2d40
Update docs for boolean value changes
daymxn Sep 30, 2025
ed06c34
Update FirebaseAI/Sources/Types/Public/Live/LiveServerContent.swift
daymxn Sep 30, 2025
68472c4
Update docs for turn complete
daymxn Sep 30, 2025
70c9f19
Update FirebaseAI/Sources/Types/Internal/Live/AsyncWebSocket.swift
daymxn Sep 30, 2025
d487279
Update FirebaseAI/Sources/Types/Internal/Live/BidiGenerateContentClie…
daymxn Sep 30, 2025
474c646
Update FirebaseAI/Sources/Types/Internal/Live/BidiGenerateContentClie…
daymxn Sep 30, 2025
29dfd73
Update FirebaseAI/Sources/Types/Internal/Live/BidiGenerateContentClie…
daymxn Sep 30, 2025
a7f5028
Update FirebaseAI/Sources/Types/Public/Live/LiveSessionErrors.swift
daymxn Sep 30, 2025
2429404
Update FirebaseAI/Sources/Types/Public/Live/LiveSessionErrors.swift
daymxn Sep 30, 2025
0939793
Update FirebaseAI/Sources/Types/Public/Live/LiveTranscription.swift
daymxn Sep 30, 2025
407ee64
Update FirebaseAI/Sources/Types/Public/Live/SpeechConfig.swift
daymxn Sep 30, 2025
fe885aa
Update available versions to exclude watchos
daymxn Sep 30, 2025
5977fe1
Add docs for audio response modality.
daymxn Sep 30, 2025
6a6b2be
Link to refdocs instead
daymxn Sep 30, 2025
9970582
Slight rewording + fix formatting on docs
daymxn Sep 30, 2025
0ada63e
Add back encodable for client message
daymxn Sep 30, 2025
814b4d4
Renaming GoAway to GoingAwayNotice for public api
daymxn Sep 30, 2025
84adb36
lint
daymxn Sep 30, 2025
166be93
Add workaround for `available(watchOS, unavailable)` being ignored
andrewheard Sep 30, 2025
9228d9b
Add `available(watchOS, unavailable)` annotations to remaining types
andrewheard Sep 30, 2025
140bfb8
Update FirebaseAI/Sources/FirebaseAI.swift
daymxn Sep 30, 2025
5a1d836
Fix doc links + default to false for nullable bools
daymxn Sep 30, 2025
86df993
Merge branch 'dlr/firebaseai-live-api' of github.com:firebase/firebas…
daymxn Sep 30, 2025
c5b1567
Rephrase live models in docs
daymxn Sep 30, 2025
850914e
Migrate to protocol instead of enum
daymxn Oct 1, 2025
ba868a2
Update doc comments per rachel's input
daymxn Oct 1, 2025
3788525
formatting
daymxn Oct 1, 2025
31570e3
Update FirebaseAI/Sources/Types/Public/Live/LiveServerContent.swift
daymxn Oct 1, 2025
12750b5
Merge branch 'dlr/firebaseai-live-api' of github.com:firebase/firebas…
daymxn Oct 1, 2025
feb838a
Update FirebaseAI/Sources/Types/Public/Live/LiveServerContent.swift
daymxn Oct 1, 2025
91e13b8
Update FirebaseAI/Sources/Types/Public/Live/LiveServerContent.swift
daymxn Oct 1, 2025
c1de7cd
Update FirebaseAI/Sources/Types/Public/Live/LiveGenerationConfig.swift
daymxn Oct 1, 2025
ca3ebb9
Update FirebaseAI/Sources/Types/Public/Live/LiveGenerationConfig.swift
daymxn Oct 1, 2025
6c999aa
Update FirebaseAI/Sources/Types/Public/Live/LiveGenerationConfig.swift
daymxn Oct 1, 2025
6e80ac0
Update FirebaseAI/Sources/Types/Internal/Live/LiveSessionService.swift
daymxn Oct 1, 2025
2b3e3c3
Only add underlying error if it's present
daymxn Oct 1, 2025
dc5942f
Merge branch 'dlr/firebaseai-live-api' of github.com:firebase/firebas…
daymxn Oct 1, 2025
66ca3ca
minor comment fix
daymxn Oct 1, 2025
74ea37d
Make sendVideo internal
daymxn Oct 1, 2025
adc35d2
Revert to enum
daymxn Oct 1, 2025
27ce3fa
Rename live server message type
daymxn Oct 1, 2025
d2f48fd
Rename to LiveAudioTranscription
daymxn Oct 1, 2025
a589589
Rename function responses
daymxn Oct 1, 2025
a8c24ba
Minor doc formatting
daymxn Oct 1, 2025
2ea027d
Log json payload if we don't have a mapping
daymxn Oct 1, 2025
85f4c77
Surface setup task errors
daymxn Oct 1, 2025
7d6b4de
Remove config suffix from speech param
daymxn Oct 1, 2025
72c33d7
Use apiConfig endpoint
daymxn Oct 1, 2025
339d5f3
Rename event to payload
daymxn Oct 1, 2025
dbcf303
Remove unnecessary conformance on AsyncWebSocket
daymxn Oct 1, 2025
50aac8f
Add todo for location
daymxn Oct 1, 2025
55cbf8d
Add hanging todo for service endpoints
daymxn Oct 1, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions FirebaseAI/Sources/Types/Internal/Live/AsyncWebSocket.swift
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ private import FirebaseCoreInternal
/// Also surfaces a more fine-grained ``WebSocketClosedError`` for when the web socket is closed.
@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, *)
@available(watchOS, unavailable)
final class AsyncWebSocket: NSObject, @unchecked Sendable, URLSessionWebSocketDelegate {
final class AsyncWebSocket: Sendable {
private let webSocketTask: URLSessionWebSocketTask
private let stream: AsyncThrowingStream<URLSessionWebSocketTask.Message, Error>
private let continuation: AsyncThrowingStream<URLSessionWebSocketTask.Message, Error>.Continuation
Expand Down Expand Up @@ -107,13 +107,6 @@ final class AsyncWebSocket: NSObject, @unchecked Sendable, URLSessionWebSocketDe
isFinished = true
}
}

func urlSession(_ session: URLSession,
webSocketTask: URLSessionWebSocketTask,
didCloseWith closeCode: URLSessionWebSocketTask.CloseCode,
reason: Data?) {
close(code: closeCode, reason: reason)
}
}

private extension URLSessionWebSocketTask {
Expand Down
18 changes: 16 additions & 2 deletions FirebaseAI/Sources/Types/Internal/Live/LiveSessionService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -308,11 +308,12 @@ actor LiveSessionService {
///
/// Will apply the required app check and auth headers, as the backend expects them.
private nonisolated func createWebsocket() async throws -> AsyncWebSocket {
let host = apiConfig.service.endpoint.rawValue.withoutPrefix("https://")
let urlString = switch apiConfig.service {
case .vertexAI:
"wss://firebasevertexai.googleapis.com/ws/google.firebase.vertexai.v1beta.LlmBidiService/BidiGenerateContent/locations/us-central1"
"wss://\(host)/ws/google.firebase.vertexai.v1beta.LlmBidiService/BidiGenerateContent/locations/us-central1"
case .googleAI:
"wss://firebasevertexai.googleapis.com/ws/google.firebase.vertexai.v1beta.GenerativeService/BidiGenerateContent"
"wss://\(host)/ws/google.firebase.vertexai.v1beta.GenerativeService/BidiGenerateContent"
}
guard let url = URL(string: urlString) else {
throw NSError(
Expand Down Expand Up @@ -378,3 +379,16 @@ private extension Data {
}
}
}

private extension String {
/// Create a new string with the given prefix removed, if it's present.
///
/// If the prefix isn't present, this string will be returned instead.
func withoutPrefix(_ prefix: String) -> String {
if let index = range(of: prefix, options: .anchored) {
return String(self[index.upperBound...])
} else {
return self
}
}
}
8 changes: 4 additions & 4 deletions FirebaseAI/Sources/Types/Public/Live/LiveServerMessage.swift
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public struct LiveServerMessage: Sendable {
let serverMessage: BidiGenerateContentServerMessage

/// The type of message sent from the server.
public enum Event: Sendable {
public enum Payload: Sendable {
/// Content generated by the model in response to client messages.
case content(LiveServerContent)

Expand All @@ -35,7 +35,7 @@ public struct LiveServerMessage: Sendable {
}

/// The message sent from the server.
public let payload: Event
public let payload: Payload

/// Metadata on the usage of the cached content.
public var usageMetadata: GenerateContentResponse.UsageMetadata? { serverMessage.usageMetadata }
Expand All @@ -47,7 +47,7 @@ public struct LiveServerMessage: Sendable {
@available(watchOS, unavailable)
extension LiveServerMessage {
init?(from serverMessage: BidiGenerateContentServerMessage) {
guard let payload = LiveServerMessage.Event(from: serverMessage.messageType) else {
guard let payload = LiveServerMessage.Payload(from: serverMessage.messageType) else {
return nil
}

Expand All @@ -58,7 +58,7 @@ extension LiveServerMessage {

@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, *)
@available(watchOS, unavailable)
extension LiveServerMessage.Event {
extension LiveServerMessage.Payload {
init?(from serverMessage: BidiGenerateContentServerMessage.MessageType) {
switch serverMessage {
case .setupComplete:
Expand Down
Loading