[FirebaseAI] Add Multimodal Analysis demos (#1750)

Author: YoungHypo

firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj

@@ -19,27 +19,34 @@
 #endif
 import Foundation
 import ConversationKit
+import UIKit
 
 public struct ChatMessage: Message {
   public let id: UUID = .init()
   public var content: String?
-  public let imageURL: String?
   public let participant: Participant
   public let error: (any Error)?
   public var pending = false
   public var groundingMetadata: GroundingMetadata?
+  public var attachments: [MultimodalAttachment] = []
+  public var image: UIImage?
+  // required by the Message protocol, but not used in this app
+  public var imageURL: String?
 
   public init(content: String? = nil, imageURL: String? = nil, participant: Participant,
-              error: (any Error)? = nil, pending: Bool = false) {
+              error: (any Error)? = nil, pending: Bool = false,
+              attachments: [MultimodalAttachment] = [], image: UIImage? = nil) {
     self.content = content
     self.imageURL = imageURL
     self.participant = participant
     self.error = error
     self.pending = pending
+    self.attachments = attachments
+    self.image = image
   }
 
   // Protocol-required initializer
-  public init(content: String?, imageURL: String?, participant: Participant) {
+  public init(content: String?, imageURL: String? = nil, participant: Participant) {
     self.content = content
     self.imageURL = imageURL
     self.participant = participant
@@ -58,16 +65,18 @@ extension ChatMessage {
   public static func == (lhs: ChatMessage, rhs: ChatMessage) -> Bool {
     lhs.id == rhs.id &&
       lhs.content == rhs.content &&
-      lhs.imageURL == rhs.imageURL &&
-      lhs.participant == rhs.participant
+      lhs.participant == rhs.participant &&
+      lhs.image == rhs.image &&
+      lhs.attachments == rhs.attachments
     // intentionally ignore `error`
   }
 
   public func hash(into hasher: inout Hasher) {
     hasher.combine(id)
     hasher.combine(content)
-    hasher.combine(imageURL)
     hasher.combine(participant)
+    hasher.combine(image)
+    hasher.combine(attachments)
     // intentionally ignore `error`
   }
 }

firebaseai/FirebaseAIExample/ChatExample/Models/ChatMessage.swift

@@ -17,13 +17,13 @@ import SwiftUI
 import ConversationKit
 
 struct ChatScreen: View {
-  let firebaseService: FirebaseAI
+  let backendType: BackendOption
   @StateObject var viewModel: ChatViewModel
 
-  init(firebaseService: FirebaseAI, sample: Sample? = nil) {
-    self.firebaseService = firebaseService
+  init(backendType: BackendOption, sample: Sample? = nil) {
+    self.backendType = backendType
     _viewModel =
-      StateObject(wrappedValue: ChatViewModel(firebaseService: firebaseService,
+      StateObject(wrappedValue: ChatViewModel(backendType: backendType,
                                               sample: sample))
   }
 
@@ -35,9 +35,7 @@ struct ChatScreen: View {
       }
       .disableAttachments()
       .onSendMessage { message in
-        Task {
-          await viewModel.sendMessage(message.content ?? "", streaming: true)
-        }
+        await viewModel.sendMessage(message.content ?? "", streaming: true)
       }
       .onError { error in
         viewModel.presentErrorDetails = true
@@ -65,5 +63,5 @@ struct ChatScreen: View {
 }
 
 #Preview {
-  ChatScreen(firebaseService: FirebaseAI.firebaseAI())
+  ChatScreen(backendType: .googleAI)
 }

firebaseai/FirebaseAIExample/ChatExample/Screens/ChatScreen.swift

@@ -40,24 +40,27 @@ class ChatViewModel: ObservableObject {
 
   private var model: GenerativeModel
   private var chat: Chat
-  private var stopGenerating = false
 
   private var chatTask: Task<Void, Never>?
 
   private var sample: Sample?
+  private var backendType: BackendOption
 
-  init(firebaseService: FirebaseAI, sample: Sample? = nil) {
+  init(backendType: BackendOption, sample: Sample? = nil) {
     self.sample = sample
+    self.backendType = backendType
+
+    let firebaseService = backendType == .googleAI
+      ? FirebaseAI.firebaseAI(backend: .googleAI())
+      : FirebaseAI.firebaseAI(backend: .vertexAI())
 
-    // create a generative model with sample data
     model = firebaseService.generativeModel(
-      modelName: "gemini-2.0-flash-001",
-      tools: sample?.tools,
+      modelName: sample?.modelName ?? "gemini-2.5-flash",
+      generationConfig: sample?.generationConfig,
       systemInstruction: sample?.systemInstruction
     )
 
     if let chatHistory = sample?.chatHistory, !chatHistory.isEmpty {
-      // Initialize with sample chat history if it's available
       messages = ChatMessage.from(chatHistory)
       chat = model.startChat(history: chatHistory)
     } else {
@@ -116,13 +119,14 @@ class ChatViewModel: ObservableObject {
               .content = (messages[messages.count - 1].content ?? "") + text
           }
 
-          if let candidate = chunk.candidates.first {
-            if let groundingMetadata = candidate.groundingMetadata {
-              self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata
+          if let inlineDataPart = chunk.inlineDataParts.first {
+            if let uiImage = UIImage(data: inlineDataPart.data) {
+              messages[messages.count - 1].image = uiImage
+            } else {
+              print("Failed to convert inline data to UIImage")
             }
           }
         }
-
       } catch {
         self.error = error
         print(error.localizedDescription)
@@ -160,11 +164,13 @@ class ChatViewModel: ObservableObject {
           // replace pending message with backend response
           messages[messages.count - 1].content = responseText
           messages[messages.count - 1].pending = false
+        }
 
-          if let candidate = response?.candidates.first {
-            if let groundingMetadata = candidate.groundingMetadata {
-              self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata
-            }
+        if let inlineDataPart = response?.inlineDataParts.first {
+          if let uiImage = UIImage(data: inlineDataPart.data) {
+            messages[messages.count - 1].image = uiImage
+          } else {
+            print("Failed to convert inline data to UIImage")
           }
         }
       } catch {

firebaseai/FirebaseAIExample/ChatExample/ViewModels/ChatViewModel.swift

@@ -57,16 +57,28 @@ struct MessageContentView: View {
           }
           .labelStyle(.iconOnly)
         }
-      }
+      } else {
+        VStack(alignment: .leading, spacing: 8) {
+          if message.participant == .user && !message.attachments.isEmpty {
+            AttachmentPreviewScrollView(attachments: message.attachments)
+          }
 
-      // Grounded Response
-      else if let groundingMetadata = message.groundingMetadata {
-        GroundedResponseView(message: message, groundingMetadata: groundingMetadata)
-      }
+          if let image = message.image {
+            Image(uiImage: image)
+              .resizable()
+              .aspectRatio(contentMode: .fit)
+              .frame(maxWidth: 300, maxHeight: 300)
+              .clipShape(RoundedRectangle(cornerRadius: 8))
+          }
 
-      // Non-grounded response
-      else {
-        ResponseTextView(message: message)
+          // Grounded Response
+          if let groundingMetadata = message.groundingMetadata {
+            GroundedResponseView(message: message, groundingMetadata: groundingMetadata)
+          } else {
+            // Non-grounded response
+            ResponseTextView(message: message)
+          }
+        }
       }
     }
   }

firebaseai/FirebaseAIExample/ChatExample/Views/MessageView.swift

@@ -18,25 +18,20 @@ import FirebaseAI
 enum BackendOption: String, CaseIterable, Identifiable {
   case googleAI = "Gemini Developer API"
   case vertexAI = "Vertex AI Gemini API"
-  var id: String { rawValue }
 
-  var backendValue: FirebaseAI {
-    switch self {
-    case .googleAI:
-      return FirebaseAI.firebaseAI(backend: .googleAI())
-    case .vertexAI:
-      return FirebaseAI.firebaseAI(backend: .vertexAI())
-    }
-  }
+  var id: String { rawValue }
 }
 
 struct ContentView: View {
   @State private var selectedBackend: BackendOption = .googleAI
-  @State private var firebaseService: FirebaseAI = FirebaseAI.firebaseAI(backend: .googleAI())
-  @State private var selectedUseCase: UseCase = .text
+  @State private var selectedUseCase: UseCase = .all
 
   var filteredSamples: [Sample] {
-    Sample.samples.filter { $0.useCases.contains(selectedUseCase) }
+    if selectedUseCase == .all {
+      return Sample.samples
+    } else {
+      return Sample.samples.filter { $0.useCases.contains(selectedUseCase) }
+    }
   }
 
   let columns = [
@@ -102,23 +97,22 @@ struct ContentView: View {
       }
       .background(Color(.systemGroupedBackground))
       .navigationTitle("Firebase AI Logic")
-      .onChange(of: selectedBackend) { newBackend in
-        firebaseService = newBackend.backendValue
-      }
     }
   }
 
   @ViewBuilder
   private func destinationView(for sample: Sample) -> some View {
     switch sample.navRoute {
     case "ChatScreen":
-      ChatScreen(firebaseService: firebaseService, sample: sample)
+      ChatScreen(backendType: selectedBackend, sample: sample)
     case "ImagenScreen":
-      ImagenScreen(firebaseService: firebaseService, sample: sample)
-    case "PhotoReasoningScreen":
-      PhotoReasoningScreen(firebaseService: firebaseService)
+      ImagenScreen(backendType: selectedBackend, sample: sample)
+    case "MultimodalScreen":
+      MultimodalScreen(backendType: selectedBackend, sample: sample)
     case "FunctionCallingScreen":
-      FunctionCallingScreen(firebaseService: firebaseService, sample: sample)
+      FunctionCallingScreen(backendType: selectedBackend, sample: sample)
+    case "GroundingScreen":
+      GroundingScreen(backendType: selectedBackend, sample: sample)
     default:
       EmptyView()
     }

firebaseai/FirebaseAIExample/ContentView.swift

@@ -17,13 +17,13 @@ import SwiftUI
 import ConversationKit
 
 struct FunctionCallingScreen: View {
-  let firebaseService: FirebaseAI
+  let backendType: BackendOption
   @StateObject var viewModel: FunctionCallingViewModel
 
-  init(firebaseService: FirebaseAI, sample: Sample? = nil) {
-    self.firebaseService = firebaseService
+  init(backendType: BackendOption, sample: Sample? = nil) {
+    self.backendType = backendType
     _viewModel =
-      StateObject(wrappedValue: FunctionCallingViewModel(firebaseService: firebaseService,
+      StateObject(wrappedValue: FunctionCallingViewModel(backendType: backendType,
                                                          sample: sample))
   }
 
@@ -35,9 +35,7 @@ struct FunctionCallingScreen: View {
       }
       .disableAttachments()
       .onSendMessage { message in
-        Task {
-          await viewModel.sendMessage(message.content ?? "", streaming: true)
-        }
+        await viewModel.sendMessage(message.content ?? "", streaming: true)
       }
       .onError { error in
         viewModel.presentErrorDetails = true
@@ -65,5 +63,5 @@ struct FunctionCallingScreen: View {
 }
 
 #Preview {
-  FunctionCallingScreen(firebaseService: FirebaseAI.firebaseAI())
+  FunctionCallingScreen(backendType: .googleAI)
 }

firebaseai/FirebaseAIExample/FunctionCallingExample/Screens/FunctionCallingScreen.swift

@@ -44,13 +44,19 @@ class FunctionCallingViewModel: ObservableObject {
   private var chatTask: Task<Void, Never>?
 
   private var sample: Sample?
+  private var backendType: BackendOption
 
-  init(firebaseService: FirebaseAI, sample: Sample? = nil) {
+  init(backendType: BackendOption, sample: Sample? = nil) {
     self.sample = sample
+    self.backendType = backendType
+
+    let firebaseService = backendType == .googleAI
+      ? FirebaseAI.firebaseAI(backend: .googleAI())
+      : FirebaseAI.firebaseAI(backend: .vertexAI())
 
     // create a generative model with sample data
     model = firebaseService.generativeModel(
-      modelName: "gemini-2.0-flash-001",
+      modelName: sample?.modelName ?? "gemini-2.0-flash-001",
       tools: sample?.tools,
       systemInstruction: sample?.systemInstruction
     )
@@ -103,18 +109,24 @@ class FunctionCallingViewModel: ObservableObject {
       do {
         let responseStream = try chat.sendMessageStream(text)
 
+        var functionCalls = [FunctionCallPart]()
+
         for try await chunk in responseStream {
           if !chunk.functionCalls.isEmpty {
-            try await handleFunctionCallsStreaming(chunk)
-          } else {
-            if let text = chunk.text {
-              messages[messages.count - 1]
-                .content = (messages[messages.count - 1].content ?? "") + text
-              messages[messages.count - 1].pending = false
-            }
+            functionCalls.append(contentsOf: chunk.functionCalls)
+          }
+          if let text = chunk.text {
+            messages[messages.count - 1]
+              .content = (messages[messages.count - 1].content ?? "") + text
+            messages[messages.count - 1].pending = false
           }
         }
 
+        // On functionCalls, never keep reading the old stream or call the second API inside the first for-loop.
+        // Start a NEW stream only after the function response turn is sent.
+        if !functionCalls.isEmpty {
+          try await handleFunctionCallsStreaming(functionCalls)
+        }
       } catch {
         self.error = error
         print(error.localizedDescription)
@@ -168,10 +180,10 @@ class FunctionCallingViewModel: ObservableObject {
     }
   }
 
-  private func handleFunctionCallsStreaming(_ response: GenerateContentResponse) async throws {
+  private func handleFunctionCallsStreaming(_ functionCalls: [FunctionCallPart]) async throws {
     var functionResponses = [FunctionResponsePart]()
 
-    for functionCall in response.functionCalls {
+    for functionCall in functionCalls {
       switch functionCall.name {
       case "fetchWeather":
         guard case let .string(city) = functionCall.args["city"],
@@ -198,7 +210,7 @@ class FunctionCallingViewModel: ObservableObject {
     }
 
     if !functionResponses.isEmpty {
-      let finalResponse = try await chat
+      let finalResponse = try chat
         .sendMessageStream([ModelContent(role: "function", parts: functionResponses)])
 
       for try await chunk in finalResponse {