[FirebaseAI] Add Multimodal Analysis demos

.github/workflows/firebaseai.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: macOS-15
     strategy:
       matrix:
-        xcode: ["26_beta_5"]
+        xcode: ["26.0"]
         os: [iOS]
         include:
           - os: iOS
@@ -45,5 +45,10 @@ jobs:
           gem install xcpretty
       - name: Placeholder GoogleService-Info.plist good enough for build only testing.
         run: cp ./mock-GoogleService-Info.plist ./firebaseai/GoogleService-Info.plist
+      - uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: '26.0-beta'
+      - name: Install iOS
+        run: xcodebuild -downloadPlatform iOS
       - name: Build and Test SwiftUI (${{ matrix.os }})
         run: ./scripts/test.sh

firebaseai/ChatExample/Models/ChatMessage.swift

@@ -15,27 +15,34 @@
 import FirebaseAI
 import Foundation
 import ConversationKit
+import UIKit
 
 public struct ChatMessage: Message {
   public let id: UUID = .init()
   public var content: String?
-  public let imageURL: String?
   public let participant: Participant
   public let error: (any Error)?
   public var pending = false
   public var groundingMetadata: GroundingMetadata?
+  public var attachments: [MultimodalAttachment] = []
+  public var image: UIImage?
+  // required by the Message protocol, but not used in this app
+  public var imageURL: String?
 
   public init(content: String? = nil, imageURL: String? = nil, participant: Participant,
-              error: (any Error)? = nil, pending: Bool = false) {
+              error: (any Error)? = nil, pending: Bool = false,
+              attachments: [MultimodalAttachment] = [], image: UIImage? = nil) {
     self.content = content
     self.imageURL = imageURL
     self.participant = participant
     self.error = error
     self.pending = pending
+    self.attachments = attachments
+    self.image = image
   }
 
   // Protocol-required initializer
-  public init(content: String?, imageURL: String?, participant: Participant) {
+  public init(content: String?, imageURL: String? = nil, participant: Participant) {
     self.content = content
     self.imageURL = imageURL
     self.participant = participant
@@ -54,16 +61,18 @@ extension ChatMessage {
   public static func == (lhs: ChatMessage, rhs: ChatMessage) -> Bool {
     lhs.id == rhs.id &&
       lhs.content == rhs.content &&
-      lhs.imageURL == rhs.imageURL &&
-      lhs.participant == rhs.participant
+      lhs.participant == rhs.participant &&
+      lhs.image == rhs.image &&
+      lhs.attachments == rhs.attachments
     // intentionally ignore `error`
   }
 
   public func hash(into hasher: inout Hasher) {
     hasher.combine(id)
     hasher.combine(content)
-    hasher.combine(imageURL)
     hasher.combine(participant)
+    hasher.combine(image)
+    hasher.combine(attachments)
     // intentionally ignore `error`
   }
 }

firebaseai/ChatExample/Screens/ChatScreen.swift

@@ -17,13 +17,13 @@ import SwiftUI
 import ConversationKit
 
 struct ChatScreen: View {
-  let firebaseService: FirebaseAI
+  let backendType: BackendOption
   @StateObject var viewModel: ChatViewModel
 
-  init(firebaseService: FirebaseAI, sample: Sample? = nil) {
-    self.firebaseService = firebaseService
+  init(backendType: BackendOption, sample: Sample? = nil) {
+    self.backendType = backendType
     _viewModel =
-      StateObject(wrappedValue: ChatViewModel(firebaseService: firebaseService,
+      StateObject(wrappedValue: ChatViewModel(backendType: backendType,
                                               sample: sample))
   }
 
@@ -35,9 +35,7 @@ struct ChatScreen: View {
       }
       .disableAttachments()
       .onSendMessage { message in
-        Task {
-          await viewModel.sendMessage(message.content ?? "", streaming: true)
-        }
+        await viewModel.sendMessage(message.content ?? "", streaming: true)
       }
       .onError { error in
         viewModel.presentErrorDetails = true
@@ -65,5 +63,5 @@ struct ChatScreen: View {
 }
 
 #Preview {
-  ChatScreen(firebaseService: FirebaseAI.firebaseAI())
+  ChatScreen(backendType: .googleAI)
 }

firebaseai/ChatExample/ViewModels/ChatViewModel.swift

@@ -36,24 +36,27 @@ class ChatViewModel: ObservableObject {
 
   private var model: GenerativeModel
   private var chat: Chat
-  private var stopGenerating = false
 
   private var chatTask: Task<Void, Never>?
 
   private var sample: Sample?
+  private var backendType: BackendOption
 
-  init(firebaseService: FirebaseAI, sample: Sample? = nil) {
+  init(backendType: BackendOption, sample: Sample? = nil) {
     self.sample = sample
+    self.backendType = backendType
+
+    let firebaseService = backendType == .googleAI
+      ? FirebaseAI.firebaseAI(backend: .googleAI())
+      : FirebaseAI.firebaseAI(backend: .vertexAI())
 
-    // create a generative model with sample data
     model = firebaseService.generativeModel(
-      modelName: "gemini-2.0-flash-001",
-      tools: sample?.tools,
+      modelName: sample?.modelName ?? "gemini-2.5-flash",
+      generationConfig: sample?.generationConfig,
       systemInstruction: sample?.systemInstruction
     )
 
     if let chatHistory = sample?.chatHistory, !chatHistory.isEmpty {
-      // Initialize with sample chat history if it's available
       messages = ChatMessage.from(chatHistory)
       chat = model.startChat(history: chatHistory)
     } else {
@@ -112,13 +115,14 @@ class ChatViewModel: ObservableObject {
               .content = (messages[messages.count - 1].content ?? "") + text
           }
 
-          if let candidate = chunk.candidates.first {
-            if let groundingMetadata = candidate.groundingMetadata {
-              self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata
+          if let inlineDataPart = chunk.inlineDataParts.first {
+            if let uiImage = UIImage(data: inlineDataPart.data) {
+              messages[messages.count - 1].image = uiImage
+            } else {
+              print("Failed to convert inline data to UIImage")
             }
           }
         }
-
       } catch {
         self.error = error
         print(error.localizedDescription)
@@ -156,11 +160,13 @@ class ChatViewModel: ObservableObject {
           // replace pending message with backend response
           messages[messages.count - 1].content = responseText
           messages[messages.count - 1].pending = false
+        }
 
-          if let candidate = response?.candidates.first {
-            if let groundingMetadata = candidate.groundingMetadata {
-              self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata
-            }
+        if let inlineDataPart = response?.inlineDataParts.first {
+          if let uiImage = UIImage(data: inlineDataPart.data) {
+            messages[messages.count - 1].image = uiImage
+          } else {
+            print("Failed to convert inline data to UIImage")
           }
         }
       } catch {

firebaseai/ChatExample/Views/MessageView.swift

@@ -53,16 +53,28 @@ struct MessageContentView: View {
           }
           .labelStyle(.iconOnly)
         }
-      }
+      } else {
+        VStack(alignment: .leading, spacing: 8) {
+          if message.participant == .user && !message.attachments.isEmpty {
+            AttachmentPreviewScrollView(attachments: message.attachments)
+          }
 
-      // Grounded Response
-      else if let groundingMetadata = message.groundingMetadata {
-        GroundedResponseView(message: message, groundingMetadata: groundingMetadata)
-      }
+          if let image = message.image {
+            Image(uiImage: image)
+              .resizable()
+              .aspectRatio(contentMode: .fit)
+              .frame(maxWidth: 300, maxHeight: 300)
+              .clipShape(RoundedRectangle(cornerRadius: 8))
+          }
 
-      // Non-grounded response
-      else {
-        ResponseTextView(message: message)
+          // Grounded Response
+          if let groundingMetadata = message.groundingMetadata {
+            GroundedResponseView(message: message, groundingMetadata: groundingMetadata)
+          } else {
+            // Non-grounded response
+            ResponseTextView(message: message)
+          }
+        }
       }
     }
   }