From d2d4f87c971776d1817631c6a6fe1312d0d2d013 Mon Sep 17 00:00:00 2001 From: haibo Date: Sun, 17 Aug 2025 10:02:46 -0700 Subject: [PATCH 01/13] add multimodal analysis --- .../AccentColor.colorset/Contents.json | 11 - .../AppIcon.appiconset/Contents.json | 13 - .../ChatExample/Models/ChatMessage.swift | 17 +- .../ViewModels/ChatViewModel.swift | 26 +- .../ChatExample/Views/MessageView.swift | 28 ++- .../project.pbxproj | 223 ++++++++++-------- .../FirebaseAIExample/ContentView.swift | 16 +- .../Views/SampleCardView.swift | 2 + .../ViewModels/FunctionCallingViewModel.swift | 4 +- .../AccentColor.colorset/Contents.json | 11 - .../AppIcon.appiconset/Contents.json | 13 - .../Assets.xcassets/Contents.json | 6 - .../Preview Assets.xcassets/Contents.json | 6 - .../Screens/PhotoReasoningScreen.swift | 88 ------- .../ViewModels/PhotoReasoningViewModel.swift | 119 ---------- .../AccentColor.colorset/Contents.json | 11 - .../AppIcon.appiconset/Contents.json | 13 - .../Assets.xcassets/Contents.json | 6 - .../Preview Assets.xcassets/Contents.json | 6 - .../Screens/GenerateContentScreen.swift | 85 ------- .../ViewModels/GenerateContentViewModel.swift | 66 ------ .../Screens/GroundingScreen.swift | 69 ++++++ .../ViewModels/GroundingViewMoel.swift | 170 +++++++++++++ .../Views}/GoogleSearchSuggestionView.swift | 0 .../Views}/GroundedResponseView.swift | 0 .../ImagenScreen.swift | 2 +- .../ImagenViewModel.swift | 0 .../Models/MultimodalAttachment.swift | 204 ++++++++++++++++ .../Preview Assets.xcassets}/Contents.json | 0 .../Screens/MultimodalScreen.swift | 185 +++++++++++++++ .../ViewModels/MultimodalViewModel.swift | 192 +++++++++++++++ .../Views/AttachmentPreviewCard.swift | 212 +++++++++++++++++ firebaseai/UIComponents/Models/Sample.swift | 138 +++++++---- firebaseai/UIComponents/Models/UseCase.swift | 1 + .../UIComponents/Views/InputField.swift | 83 ------- .../Views/MultimodalInputField.swift | 183 -------------- 36 files changed, 1308 insertions(+), 901 deletions(-) delete mode 100644 firebaseai/ChatExample/Assets.xcassets/AccentColor.colorset/Contents.json delete mode 100644 firebaseai/ChatExample/Assets.xcassets/AppIcon.appiconset/Contents.json delete mode 100644 firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AccentColor.colorset/Contents.json delete mode 100644 firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AppIcon.appiconset/Contents.json delete mode 100644 firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/Contents.json delete mode 100644 firebaseai/GenerativeAIMultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json delete mode 100644 firebaseai/GenerativeAIMultimodalExample/Screens/PhotoReasoningScreen.swift delete mode 100644 firebaseai/GenerativeAIMultimodalExample/ViewModels/PhotoReasoningViewModel.swift delete mode 100644 firebaseai/GenerativeAITextExample/Assets.xcassets/AccentColor.colorset/Contents.json delete mode 100644 firebaseai/GenerativeAITextExample/Assets.xcassets/AppIcon.appiconset/Contents.json delete mode 100644 firebaseai/GenerativeAITextExample/Assets.xcassets/Contents.json delete mode 100644 firebaseai/GenerativeAITextExample/Preview Content/Preview Assets.xcassets/Contents.json delete mode 100644 firebaseai/GenerativeAITextExample/Screens/GenerateContentScreen.swift delete mode 100644 firebaseai/GenerativeAITextExample/ViewModels/GenerateContentViewModel.swift create mode 100644 firebaseai/GroundingExample/Screens/GroundingScreen.swift create mode 100644 firebaseai/GroundingExample/ViewModels/GroundingViewMoel.swift rename firebaseai/{ChatExample/Views/Grounding => GroundingExample/Views}/GoogleSearchSuggestionView.swift (100%) rename firebaseai/{ChatExample/Views/Grounding => GroundingExample/Views}/GroundedResponseView.swift (100%) rename firebaseai/{ImagenScreen => ImagenExample}/ImagenScreen.swift (98%) rename firebaseai/{ImagenScreen => ImagenExample}/ImagenViewModel.swift (100%) create mode 100644 firebaseai/MultimodalExample/Models/MultimodalAttachment.swift rename firebaseai/{ChatExample/Assets.xcassets => MultimodalExample/Preview Content/Preview Assets.xcassets}/Contents.json (100%) create mode 100644 firebaseai/MultimodalExample/Screens/MultimodalScreen.swift create mode 100644 firebaseai/MultimodalExample/ViewModels/MultimodalViewModel.swift create mode 100644 firebaseai/MultimodalExample/Views/AttachmentPreviewCard.swift delete mode 100644 firebaseai/UIComponents/Views/InputField.swift delete mode 100644 firebaseai/UIComponents/Views/MultimodalInputField.swift diff --git a/firebaseai/ChatExample/Assets.xcassets/AccentColor.colorset/Contents.json b/firebaseai/ChatExample/Assets.xcassets/AccentColor.colorset/Contents.json deleted file mode 100644 index eb8789700..000000000 --- a/firebaseai/ChatExample/Assets.xcassets/AccentColor.colorset/Contents.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "colors" : [ - { - "idiom" : "universal" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/ChatExample/Assets.xcassets/AppIcon.appiconset/Contents.json b/firebaseai/ChatExample/Assets.xcassets/AppIcon.appiconset/Contents.json deleted file mode 100644 index 13613e3ee..000000000 --- a/firebaseai/ChatExample/Assets.xcassets/AppIcon.appiconset/Contents.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "images" : [ - { - "idiom" : "universal", - "platform" : "ios", - "size" : "1024x1024" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/ChatExample/Models/ChatMessage.swift b/firebaseai/ChatExample/Models/ChatMessage.swift index f1171c748..45d54eb23 100644 --- a/firebaseai/ChatExample/Models/ChatMessage.swift +++ b/firebaseai/ChatExample/Models/ChatMessage.swift @@ -15,29 +15,32 @@ import FirebaseAI import Foundation import ConversationKit +import UIKit public struct ChatMessage: Message { public let id: UUID = .init() public var content: String? - public let imageURL: String? public let participant: Participant public let error: (any Error)? public var pending = false public var groundingMetadata: GroundingMetadata? + public var attachments: [MultimodalAttachment] = [] + public var image: UIImage? - public init(content: String? = nil, imageURL: String? = nil, participant: Participant, - error: (any Error)? = nil, pending: Bool = false) { + public init(content: String? = nil, participant: Participant, + error: (any Error)? = nil, pending: Bool = false, + attachments: [MultimodalAttachment] = [], image: UIImage? = nil) { self.content = content - self.imageURL = imageURL self.participant = participant self.error = error self.pending = pending + self.attachments = attachments + self.image = image } // Protocol-required initializer - public init(content: String?, imageURL: String?, participant: Participant) { + public init(content: String?, participant: Participant) { self.content = content - self.imageURL = imageURL self.participant = participant error = nil } @@ -54,7 +57,6 @@ extension ChatMessage { public static func == (lhs: ChatMessage, rhs: ChatMessage) -> Bool { lhs.id == rhs.id && lhs.content == rhs.content && - lhs.imageURL == rhs.imageURL && lhs.participant == rhs.participant // intentionally ignore `error` } @@ -62,7 +64,6 @@ extension ChatMessage { public func hash(into hasher: inout Hasher) { hasher.combine(id) hasher.combine(content) - hasher.combine(imageURL) hasher.combine(participant) // intentionally ignore `error` } diff --git a/firebaseai/ChatExample/ViewModels/ChatViewModel.swift b/firebaseai/ChatExample/ViewModels/ChatViewModel.swift index ad1077407..7384b5b53 100644 --- a/firebaseai/ChatExample/ViewModels/ChatViewModel.swift +++ b/firebaseai/ChatExample/ViewModels/ChatViewModel.swift @@ -36,7 +36,6 @@ class ChatViewModel: ObservableObject { private var model: GenerativeModel private var chat: Chat - private var stopGenerating = false private var chatTask: Task? @@ -45,15 +44,13 @@ class ChatViewModel: ObservableObject { init(firebaseService: FirebaseAI, sample: Sample? = nil) { self.sample = sample - // create a generative model with sample data model = firebaseService.generativeModel( - modelName: "gemini-2.0-flash-001", - tools: sample?.tools, + modelName: sample?.modelName ?? "gemini-2.5-flash", + generationConfig: sample?.generationConfig, systemInstruction: sample?.systemInstruction ) if let chatHistory = sample?.chatHistory, !chatHistory.isEmpty { - // Initialize with sample chat history if it's available messages = ChatMessage.from(chatHistory) chat = model.startChat(history: chatHistory) } else { @@ -112,13 +109,14 @@ class ChatViewModel: ObservableObject { .content = (messages[messages.count - 1].content ?? "") + text } - if let candidate = chunk.candidates.first { - if let groundingMetadata = candidate.groundingMetadata { - self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata + if let inlineDataPart = chunk.inlineDataParts.first { + if let uiImage = UIImage(data: inlineDataPart.data) { + messages[messages.count - 1].image = (messages[messages.count - 1].image ?? uiImage) + } else { + print("Failed to convert inline data to UIImage") } } } - } catch { self.error = error print(error.localizedDescription) @@ -156,11 +154,13 @@ class ChatViewModel: ObservableObject { // replace pending message with backend response messages[messages.count - 1].content = responseText messages[messages.count - 1].pending = false + } - if let candidate = response?.candidates.first { - if let groundingMetadata = candidate.groundingMetadata { - self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata - } + if let inlineDataPart = response?.inlineDataParts.first { + if let uiImage = UIImage(data: inlineDataPart.data) { + messages[messages.count - 1].image = uiImage + } else { + print("Failed to convert inline data to UIImage") } } } catch { diff --git a/firebaseai/ChatExample/Views/MessageView.swift b/firebaseai/ChatExample/Views/MessageView.swift index 2242a02cd..5eb3ac6b0 100644 --- a/firebaseai/ChatExample/Views/MessageView.swift +++ b/firebaseai/ChatExample/Views/MessageView.swift @@ -53,16 +53,28 @@ struct MessageContentView: View { } .labelStyle(.iconOnly) } - } + } else { + VStack(alignment: .leading, spacing: 8) { + if message.participant == .user && !message.attachments.isEmpty { + AttachmentPreviewScrollView(attachments: message.attachments) + } - // Grounded Response - else if let groundingMetadata = message.groundingMetadata { - GroundedResponseView(message: message, groundingMetadata: groundingMetadata) - } + if let image = message.image { + Image(uiImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxWidth: 300, maxHeight: 300) + .clipShape(RoundedRectangle(cornerRadius: 8)) + } - // Non-grounded response - else { - ResponseTextView(message: message) + // Grounded Response + if let groundingMetadata = message.groundingMetadata { + GroundedResponseView(message: message, groundingMetadata: groundingMetadata) + } else { + // Non-grounded response + ResponseTextView(message: message) + } + } } } } diff --git a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj index 2217a26aa..cd599c53c 100644 --- a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj +++ b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj @@ -7,25 +7,27 @@ objects = { /* Begin PBXBuildFile section */ + 7210F4A22E52317E002FE9F2 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 7210F4972E52317E002FE9F2 /* Preview Assets.xcassets */; }; + 7210F4A32E52317E002FE9F2 /* MultimodalViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F49B2E52317E002FE9F2 /* MultimodalViewModel.swift */; }; + 7210F4A42E52317E002FE9F2 /* AttachmentPreviewCard.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F49D2E52317E002FE9F2 /* AttachmentPreviewCard.swift */; }; + 7210F4A52E52317E002FE9F2 /* MultimodalAttachment.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F4952E52317E002FE9F2 /* MultimodalAttachment.swift */; }; + 7210F4A62E52317E002FE9F2 /* MultimodalScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F4992E52317E002FE9F2 /* MultimodalScreen.swift */; }; + 7210F4B12E525A64002FE9F2 /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 7210F4B02E525A64002FE9F2 /* ConversationKit */; }; + 7210F4BA2E526AA1002FE9F2 /* GroundingScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F4B92E526A9B002FE9F2 /* GroundingScreen.swift */; }; + 7210F4BC2E526AB2002FE9F2 /* GroundingViewMoel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F4BB2E526AAA002FE9F2 /* GroundingViewMoel.swift */; }; + 7210F4C82E527A39002FE9F2 /* GoogleService-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = 7210F4C72E527A39002FE9F2 /* GoogleService-Info.plist */; }; 726490D92E3F39E000A92700 /* Sample.swift in Sources */ = {isa = PBXBuildFile; fileRef = 726490D22E3F39D200A92700 /* Sample.swift */; }; 726490DA2E3F39E000A92701 /* UseCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 726490D32E3F39D200A92700 /* UseCase.swift */; }; - 726490DC2E3F39E000A92703 /* InputField.swift in Sources */ = {isa = PBXBuildFile; fileRef = 726490D72E3F39D900A92700 /* InputField.swift */; }; - 726490DD2E3F39E000A92704 /* MultimodalInputField.swift in Sources */ = {isa = PBXBuildFile; fileRef = 726490D82E3F39DC00A92700 /* MultimodalInputField.swift */; }; 72DA044F2E385DF3004FED7D /* ChatMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 72DA044E2E385DF3004FED7D /* ChatMessage.swift */; }; 72E040752E448731003D4135 /* WeatherService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 72E040742E44872C003D4135 /* WeatherService.swift */; }; - 869200B32B879C4F00482873 /* GoogleService-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = 869200B22B879C4F00482873 /* GoogleService-Info.plist */; }; 86C1F4832BC726150026816F /* FunctionCallingScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 86C1F47E2BC726150026816F /* FunctionCallingScreen.swift */; }; 86C1F4842BC726150026816F /* FunctionCallingViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 86C1F4802BC726150026816F /* FunctionCallingViewModel.swift */; }; 88263BF12B239C11008AB09B /* ErrorDetailsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 889873842B208563005B4896 /* ErrorDetailsView.swift */; }; - 884298E12E4B8110005F535F /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 884298E02E4B8110005F535F /* ConversationKit */; }; 8848C8332B0D04BC007B434F /* FirebaseAIExampleApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8848C8322B0D04BC007B434F /* FirebaseAIExampleApp.swift */; }; 8848C8352B0D04BC007B434F /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8848C8342B0D04BC007B434F /* ContentView.swift */; }; 8848C8372B0D04BD007B434F /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8848C8362B0D04BD007B434F /* Assets.xcassets */; }; 8848C83A2B0D04BD007B434F /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8848C8392B0D04BD007B434F /* Preview Assets.xcassets */; }; - 885D0CA12E4CB7CD00A217A0 /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 885D0CA02E4CB7CD00A217A0 /* ConversationKit */; }; 886F95D82B17BA420036F07A /* MarkdownUI in Frameworks */ = {isa = PBXBuildFile; productRef = 886F95D72B17BA420036F07A /* MarkdownUI */; }; - 886F95DB2B17BAEF0036F07A /* PhotoReasoningViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8802666F2B0FC39000CF7CB6 /* PhotoReasoningViewModel.swift */; }; - 886F95DC2B17BAEF0036F07A /* PhotoReasoningScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 880266752B0FC39000CF7CB6 /* PhotoReasoningScreen.swift */; }; 886F95DD2B17D5010036F07A /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88E10F5A2B11133E00C08E95 /* MessageView.swift */; }; 886F95DF2B17D5010036F07A /* BouncingDots.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88E10F5C2B11135000C08E95 /* BouncingDots.swift */; }; 886F95E02B17D5010036F07A /* ChatViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88E10F562B1112F600C08E95 /* ChatViewModel.swift */; }; @@ -40,26 +42,26 @@ /* End PBXBuildFile section */ /* Begin PBXFileReference section */ + 7210F4952E52317E002FE9F2 /* MultimodalAttachment.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultimodalAttachment.swift; sourceTree = ""; }; + 7210F4972E52317E002FE9F2 /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; + 7210F4992E52317E002FE9F2 /* MultimodalScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultimodalScreen.swift; sourceTree = ""; }; + 7210F49B2E52317E002FE9F2 /* MultimodalViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultimodalViewModel.swift; sourceTree = ""; }; + 7210F49D2E52317E002FE9F2 /* AttachmentPreviewCard.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AttachmentPreviewCard.swift; sourceTree = ""; }; + 7210F4B92E526A9B002FE9F2 /* GroundingScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GroundingScreen.swift; sourceTree = ""; }; + 7210F4BB2E526AAA002FE9F2 /* GroundingViewMoel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GroundingViewMoel.swift; sourceTree = ""; }; + 7210F4C72E527A39002FE9F2 /* GoogleService-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = "GoogleService-Info.plist"; sourceTree = ""; }; 726490D22E3F39D200A92700 /* Sample.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Sample.swift; sourceTree = ""; }; 726490D32E3F39D200A92700 /* UseCase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UseCase.swift; sourceTree = ""; }; - 726490D72E3F39D900A92700 /* InputField.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InputField.swift; sourceTree = ""; }; - 726490D82E3F39DC00A92700 /* MultimodalInputField.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultimodalInputField.swift; sourceTree = ""; }; 72DA044E2E385DF3004FED7D /* ChatMessage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatMessage.swift; sourceTree = ""; }; 72E040742E44872C003D4135 /* WeatherService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WeatherService.swift; sourceTree = ""; }; - 869200B22B879C4F00482873 /* GoogleService-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "GoogleService-Info.plist"; sourceTree = ""; }; 86C1F47E2BC726150026816F /* FunctionCallingScreen.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FunctionCallingScreen.swift; sourceTree = ""; }; 86C1F4802BC726150026816F /* FunctionCallingViewModel.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FunctionCallingViewModel.swift; sourceTree = ""; }; - 8802666F2B0FC39000CF7CB6 /* PhotoReasoningViewModel.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PhotoReasoningViewModel.swift; sourceTree = ""; }; - 880266752B0FC39000CF7CB6 /* PhotoReasoningScreen.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PhotoReasoningScreen.swift; sourceTree = ""; }; 8848C82F2B0D04BC007B434F /* FirebaseAIExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = FirebaseAIExample.app; sourceTree = BUILT_PRODUCTS_DIR; }; 8848C8322B0D04BC007B434F /* FirebaseAIExampleApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FirebaseAIExampleApp.swift; sourceTree = ""; }; 8848C8342B0D04BC007B434F /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; 8848C8362B0D04BD007B434F /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 8848C8392B0D04BD007B434F /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; - 8848C85C2B0D056D007B434F /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; - 8848C85F2B0D056D007B434F /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; 889873842B208563005B4896 /* ErrorDetailsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ErrorDetailsView.swift; sourceTree = ""; }; - 88E10F482B110D5400C08E95 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 88E10F4B2B110D5400C08E95 /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; 88E10F542B1112CA00C08E95 /* ChatScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatScreen.swift; sourceTree = ""; }; 88E10F562B1112F600C08E95 /* ChatViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatViewModel.swift; sourceTree = ""; }; @@ -78,9 +80,8 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - 884298E12E4B8110005F535F /* ConversationKit in Frameworks */, + 7210F4B12E525A64002FE9F2 /* ConversationKit in Frameworks */, DE26D95F2DBB3E9F007E6668 /* FirebaseAI in Frameworks */, - 885D0CA12E4CB7CD00A217A0 /* ConversationKit in Frameworks */, 886F95D82B17BA420036F07A /* MarkdownUI in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -88,6 +89,93 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 7210F4962E52317E002FE9F2 /* Models */ = { + isa = PBXGroup; + children = ( + 7210F4952E52317E002FE9F2 /* MultimodalAttachment.swift */, + ); + path = Models; + sourceTree = ""; + }; + 7210F4982E52317E002FE9F2 /* Preview Content */ = { + isa = PBXGroup; + children = ( + 7210F4972E52317E002FE9F2 /* Preview Assets.xcassets */, + ); + path = "Preview Content"; + sourceTree = ""; + }; + 7210F49A2E52317E002FE9F2 /* Screens */ = { + isa = PBXGroup; + children = ( + 7210F4992E52317E002FE9F2 /* MultimodalScreen.swift */, + ); + path = Screens; + sourceTree = ""; + }; + 7210F49C2E52317E002FE9F2 /* ViewModels */ = { + isa = PBXGroup; + children = ( + 7210F49B2E52317E002FE9F2 /* MultimodalViewModel.swift */, + ); + path = ViewModels; + sourceTree = ""; + }; + 7210F49E2E52317E002FE9F2 /* Views */ = { + isa = PBXGroup; + children = ( + 7210F49D2E52317E002FE9F2 /* AttachmentPreviewCard.swift */, + ); + path = Views; + sourceTree = ""; + }; + 7210F4A02E52317E002FE9F2 /* MultimodalExample */ = { + isa = PBXGroup; + children = ( + 7210F4962E52317E002FE9F2 /* Models */, + 7210F4982E52317E002FE9F2 /* Preview Content */, + 7210F49A2E52317E002FE9F2 /* Screens */, + 7210F49C2E52317E002FE9F2 /* ViewModels */, + 7210F49E2E52317E002FE9F2 /* Views */, + ); + path = MultimodalExample; + sourceTree = ""; + }; + 7210F4B42E526A5B002FE9F2 /* GroundingExample */ = { + isa = PBXGroup; + children = ( + 7210F4B82E526A82002FE9F2 /* Screens */, + 7210F4B62E526A69002FE9F2 /* ViewModels */, + 7210F4B52E526A64002FE9F2 /* Views */, + ); + path = GroundingExample; + sourceTree = ""; + }; + 7210F4B52E526A64002FE9F2 /* Views */ = { + isa = PBXGroup; + children = ( + AEE793DC2E256D3900708F02 /* GoogleSearchSuggestionView.swift */, + AEE793DD2E256D3900708F02 /* GroundedResponseView.swift */, + ); + path = Views; + sourceTree = ""; + }; + 7210F4B62E526A69002FE9F2 /* ViewModels */ = { + isa = PBXGroup; + children = ( + 7210F4BB2E526AAA002FE9F2 /* GroundingViewMoel.swift */, + ); + path = ViewModels; + sourceTree = ""; + }; + 7210F4B82E526A82002FE9F2 /* Screens */ = { + isa = PBXGroup; + children = ( + 7210F4B92E526A9B002FE9F2 /* GroundingScreen.swift */, + ); + path = Screens; + sourceTree = ""; + }; 726490D12E3F39C900A92700 /* UIComponents */ = { isa = PBXGroup; children = ( @@ -109,8 +197,6 @@ 726490D62E3F39D600A92700 /* Views */ = { isa = PBXGroup; children = ( - 726490D72E3F39D900A92700 /* InputField.swift */, - 726490D82E3F39DC00A92700 /* MultimodalInputField.swift */, ); path = Views; sourceTree = ""; @@ -157,22 +243,6 @@ path = FunctionCallingExample; sourceTree = ""; }; - 8802666E2B0FC39000CF7CB6 /* ViewModels */ = { - isa = PBXGroup; - children = ( - 8802666F2B0FC39000CF7CB6 /* PhotoReasoningViewModel.swift */, - ); - path = ViewModels; - sourceTree = ""; - }; - 880266742B0FC39000CF7CB6 /* Screens */ = { - isa = PBXGroup; - children = ( - 880266752B0FC39000CF7CB6 /* PhotoReasoningScreen.swift */, - ); - path = Screens; - sourceTree = ""; - }; 88209C222B0FBE1700F64795 /* Frameworks */ = { isa = PBXGroup; children = ( @@ -183,15 +253,16 @@ 8848C8262B0D04BC007B434F = { isa = PBXGroup; children = ( + 7210F4C72E527A39002FE9F2 /* GoogleService-Info.plist */, + 7210F4A02E52317E002FE9F2 /* MultimodalExample */, 726490D12E3F39C900A92700 /* UIComponents */, - DEFECAA82D7B4CCD00EF9621 /* ImagenScreen */, - 869200B22B879C4F00482873 /* GoogleService-Info.plist */, + DEFECAA82D7B4CCD00EF9621 /* ImagenExample */, 8848C8312B0D04BC007B434F /* FirebaseAIExample */, - 8848C8572B0D056C007B434F /* GenerativeAIMultimodalExample */, 88E10F432B110D5300C08E95 /* ChatExample */, 86C1F4822BC726150026816F /* FunctionCallingExample */, 8848C8302B0D04BC007B434F /* Products */, 88209C222B0FBE1700F64795 /* Frameworks */, + 7210F4B42E526A5B002FE9F2 /* GroundingExample */, ); sourceTree = ""; }; @@ -223,25 +294,6 @@ path = "Preview Content"; sourceTree = ""; }; - 8848C8572B0D056C007B434F /* GenerativeAIMultimodalExample */ = { - isa = PBXGroup; - children = ( - 8802666E2B0FC39000CF7CB6 /* ViewModels */, - 880266742B0FC39000CF7CB6 /* Screens */, - 8848C85C2B0D056D007B434F /* Assets.xcassets */, - 8848C85E2B0D056D007B434F /* Preview Content */, - ); - path = GenerativeAIMultimodalExample; - sourceTree = ""; - }; - 8848C85E2B0D056D007B434F /* Preview Content */ = { - isa = PBXGroup; - children = ( - 8848C85F2B0D056D007B434F /* Preview Assets.xcassets */, - ); - path = "Preview Content"; - sourceTree = ""; - }; 88E10F432B110D5300C08E95 /* ChatExample */ = { isa = PBXGroup; children = ( @@ -249,7 +301,6 @@ 88E10F502B11123600C08E95 /* ViewModels */, 88E10F512B11124100C08E95 /* Views */, 88E10F532B1112B900C08E95 /* Screens */, - 88E10F482B110D5400C08E95 /* Assets.xcassets */, 88E10F4A2B110D5400C08E95 /* Preview Content */, ); path = ChatExample; @@ -274,7 +325,6 @@ 88E10F512B11124100C08E95 /* Views */ = { isa = PBXGroup; children = ( - AEE793DE2E256D3900708F02 /* Grounding */, 88E10F5A2B11133E00C08E95 /* MessageView.swift */, 88E10F5C2B11135000C08E95 /* BouncingDots.swift */, 889873842B208563005B4896 /* ErrorDetailsView.swift */, @@ -299,22 +349,13 @@ path = Views; sourceTree = ""; }; - AEE793DE2E256D3900708F02 /* Grounding */ = { - isa = PBXGroup; - children = ( - AEE793DC2E256D3900708F02 /* GoogleSearchSuggestionView.swift */, - AEE793DD2E256D3900708F02 /* GroundedResponseView.swift */, - ); - path = Grounding; - sourceTree = ""; - }; - DEFECAA82D7B4CCD00EF9621 /* ImagenScreen */ = { + DEFECAA82D7B4CCD00EF9621 /* ImagenExample */ = { isa = PBXGroup; children = ( DEFECAA62D7B4CCD00EF9621 /* ImagenScreen.swift */, DEFECAA72D7B4CCD00EF9621 /* ImagenViewModel.swift */, ); - path = ImagenScreen; + path = ImagenExample; sourceTree = ""; }; /* End PBXGroup section */ @@ -336,8 +377,7 @@ packageProductDependencies = ( 886F95D72B17BA420036F07A /* MarkdownUI */, DE26D95E2DBB3E9F007E6668 /* FirebaseAI */, - 884298E02E4B8110005F535F /* ConversationKit */, - 885D0CA02E4CB7CD00A217A0 /* ConversationKit */, + 7210F4B02E525A64002FE9F2 /* ConversationKit */, ); productName = GenerativeAIExample; productReference = 8848C82F2B0D04BC007B434F /* FirebaseAIExample.app */; @@ -371,7 +411,7 @@ 88209C212B0FBDF700F64795 /* XCRemoteSwiftPackageReference "swift-markdown-ui" */, DEA09AC32B1FCE22001962D9 /* XCRemoteSwiftPackageReference "NetworkImage" */, DEFECAAB2D7BB49700EF9621 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */, - 885D0C9F2E4CB7CD00A217A0 /* XCRemoteSwiftPackageReference "ConversationKit" */, + 7210F4AF2E525A64002FE9F2 /* XCRemoteSwiftPackageReference "ConversationKit" */, ); productRefGroup = 8848C8302B0D04BC007B434F /* Products */; projectDirPath = ""; @@ -388,8 +428,9 @@ buildActionMask = 2147483647; files = ( 8848C83A2B0D04BD007B434F /* Preview Assets.xcassets in Resources */, + 7210F4C82E527A39002FE9F2 /* GoogleService-Info.plist in Resources */, 8848C8372B0D04BD007B434F /* Assets.xcassets in Resources */, - 869200B32B879C4F00482873 /* GoogleService-Info.plist in Resources */, + 7210F4A22E52317E002FE9F2 /* Preview Assets.xcassets in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -408,10 +449,13 @@ 8848C8332B0D04BC007B434F /* FirebaseAIExampleApp.swift in Sources */, 886F95E02B17D5010036F07A /* ChatViewModel.swift in Sources */, 886F95DD2B17D5010036F07A /* MessageView.swift in Sources */, - 886F95DC2B17BAEF0036F07A /* PhotoReasoningScreen.swift in Sources */, + 7210F4BC2E526AB2002FE9F2 /* GroundingViewMoel.swift in Sources */, DEFECAA92D7B4CCD00EF9621 /* ImagenViewModel.swift in Sources */, + 7210F4A32E52317E002FE9F2 /* MultimodalViewModel.swift in Sources */, + 7210F4A42E52317E002FE9F2 /* AttachmentPreviewCard.swift in Sources */, + 7210F4A52E52317E002FE9F2 /* MultimodalAttachment.swift in Sources */, + 7210F4A62E52317E002FE9F2 /* MultimodalScreen.swift in Sources */, DEFECAAA2D7B4CCD00EF9621 /* ImagenScreen.swift in Sources */, - 886F95DB2B17BAEF0036F07A /* PhotoReasoningViewModel.swift in Sources */, 72E040752E448731003D4135 /* WeatherService.swift in Sources */, 886F95E12B17D5010036F07A /* ChatScreen.swift in Sources */, 72DA044F2E385DF3004FED7D /* ChatMessage.swift in Sources */, @@ -419,10 +463,9 @@ A5E8E3CA2C3B4F388A7A4A1A /* SampleCardView.swift in Sources */, AEE793DF2E256D3900708F02 /* GoogleSearchSuggestionView.swift in Sources */, AEE793E02E256D3900708F02 /* GroundedResponseView.swift in Sources */, + 7210F4BA2E526AA1002FE9F2 /* GroundingScreen.swift in Sources */, 726490D92E3F39E000A92700 /* Sample.swift in Sources */, 726490DA2E3F39E000A92701 /* UseCase.swift in Sources */, - 726490DC2E3F39E000A92703 /* InputField.swift in Sources */, - 726490DD2E3F39E000A92704 /* MultimodalInputField.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -632,20 +675,20 @@ /* End XCConfigurationList section */ /* Begin XCRemoteSwiftPackageReference section */ - 88209C212B0FBDF700F64795 /* XCRemoteSwiftPackageReference "swift-markdown-ui" */ = { + 7210F4AF2E525A64002FE9F2 /* XCRemoteSwiftPackageReference "ConversationKit" */ = { isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/gonzalezreal/swift-markdown-ui"; + repositoryURL = "https://github.com/YoungHypo/ConversationKit"; requirement = { - kind = revision; - revision = 55441810c0f678c78ed7e2ebd46dde89228e02fc; + branch = main; + kind = branch; }; }; - 885D0C9F2E4CB7CD00A217A0 /* XCRemoteSwiftPackageReference "ConversationKit" */ = { + 88209C212B0FBDF700F64795 /* XCRemoteSwiftPackageReference "swift-markdown-ui" */ = { isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/peterfriese/ConversationKit"; + repositoryURL = "https://github.com/gonzalezreal/swift-markdown-ui"; requirement = { - kind = upToNextMajorVersion; - minimumVersion = 0.0.2; + kind = revision; + revision = 55441810c0f678c78ed7e2ebd46dde89228e02fc; }; }; DEA09AC32B1FCE22001962D9 /* XCRemoteSwiftPackageReference "NetworkImage" */ = { @@ -667,13 +710,9 @@ /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ - 884298E02E4B8110005F535F /* ConversationKit */ = { - isa = XCSwiftPackageProductDependency; - productName = ConversationKit; - }; - 885D0CA02E4CB7CD00A217A0 /* ConversationKit */ = { + 7210F4B02E525A64002FE9F2 /* ConversationKit */ = { isa = XCSwiftPackageProductDependency; - package = 885D0C9F2E4CB7CD00A217A0 /* XCRemoteSwiftPackageReference "ConversationKit" */; + package = 7210F4AF2E525A64002FE9F2 /* XCRemoteSwiftPackageReference "ConversationKit" */; productName = ConversationKit; }; 886F95D72B17BA420036F07A /* MarkdownUI */ = { diff --git a/firebaseai/FirebaseAIExample/ContentView.swift b/firebaseai/FirebaseAIExample/ContentView.swift index 5af66fad7..0d004ee2f 100644 --- a/firebaseai/FirebaseAIExample/ContentView.swift +++ b/firebaseai/FirebaseAIExample/ContentView.swift @@ -33,10 +33,14 @@ enum BackendOption: String, CaseIterable, Identifiable { struct ContentView: View { @State private var selectedBackend: BackendOption = .googleAI @State private var firebaseService: FirebaseAI = FirebaseAI.firebaseAI(backend: .googleAI()) - @State private var selectedUseCase: UseCase = .text + @State private var selectedUseCase: UseCase = .all var filteredSamples: [Sample] { - Sample.samples.filter { $0.useCases.contains(selectedUseCase) } + if selectedUseCase == .all { + return Sample.samples + } else { + return Sample.samples.filter { $0.useCases.contains(selectedUseCase) } + } } let columns = [ @@ -102,7 +106,7 @@ struct ContentView: View { } .background(Color(.systemGroupedBackground)) .navigationTitle("Firebase AI Logic") - .onChange(of: selectedBackend) { newBackend in + .onChange(of: selectedBackend) { _, newBackend in firebaseService = newBackend.backendValue } } @@ -115,10 +119,12 @@ struct ContentView: View { ChatScreen(firebaseService: firebaseService, sample: sample) case "ImagenScreen": ImagenScreen(firebaseService: firebaseService, sample: sample) - case "PhotoReasoningScreen": - PhotoReasoningScreen(firebaseService: firebaseService) + case "MultimodalScreen": + MultimodalScreen(firebaseService: firebaseService, sample: sample) case "FunctionCallingScreen": FunctionCallingScreen(firebaseService: firebaseService, sample: sample) + case "GroundingScreen": + GroundingScreen(firebaseService: firebaseService, sample: sample) default: EmptyView() } diff --git a/firebaseai/FirebaseAIExample/Views/SampleCardView.swift b/firebaseai/FirebaseAIExample/Views/SampleCardView.swift index af4c4680b..58034475a 100644 --- a/firebaseai/FirebaseAIExample/Views/SampleCardView.swift +++ b/firebaseai/FirebaseAIExample/Views/SampleCardView.swift @@ -39,6 +39,7 @@ struct SampleCardView: View { private func systemName(for useCase: UseCase) -> String { switch useCase { + case .all: "square.grid.2x2.fill" case .text: "text.bubble.fill" case .image: "photo.fill" case .video: "video.fill" @@ -50,6 +51,7 @@ struct SampleCardView: View { private func color(for useCase: UseCase) -> Color { switch useCase { + case .all:.primary case .text:.blue case .image:.purple case .video:.red diff --git a/firebaseai/FunctionCallingExample/ViewModels/FunctionCallingViewModel.swift b/firebaseai/FunctionCallingExample/ViewModels/FunctionCallingViewModel.swift index e288ef585..078ab1547 100644 --- a/firebaseai/FunctionCallingExample/ViewModels/FunctionCallingViewModel.swift +++ b/firebaseai/FunctionCallingExample/ViewModels/FunctionCallingViewModel.swift @@ -46,7 +46,7 @@ class FunctionCallingViewModel: ObservableObject { // create a generative model with sample data model = firebaseService.generativeModel( - modelName: "gemini-2.0-flash-001", + modelName: sample?.modelName ?? "gemini-2.0-flash-001", tools: sample?.tools, systemInstruction: sample?.systemInstruction ) @@ -194,7 +194,7 @@ class FunctionCallingViewModel: ObservableObject { } if !functionResponses.isEmpty { - let finalResponse = try await chat + let finalResponse = try chat .sendMessageStream([ModelContent(role: "function", parts: functionResponses)]) for try await chunk in finalResponse { diff --git a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AccentColor.colorset/Contents.json b/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AccentColor.colorset/Contents.json deleted file mode 100644 index eb8789700..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AccentColor.colorset/Contents.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "colors" : [ - { - "idiom" : "universal" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AppIcon.appiconset/Contents.json b/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AppIcon.appiconset/Contents.json deleted file mode 100644 index 13613e3ee..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AppIcon.appiconset/Contents.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "images" : [ - { - "idiom" : "universal", - "platform" : "ios", - "size" : "1024x1024" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/Contents.json b/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/Contents.json deleted file mode 100644 index 73c00596a..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/Contents.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json b/firebaseai/GenerativeAIMultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json deleted file mode 100644 index 73c00596a..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/Screens/PhotoReasoningScreen.swift b/firebaseai/GenerativeAIMultimodalExample/Screens/PhotoReasoningScreen.swift deleted file mode 100644 index b1a992eae..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Screens/PhotoReasoningScreen.swift +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import MarkdownUI -import PhotosUI -import SwiftUI -import FirebaseAI - -struct PhotoReasoningScreen: View { - let firebaseService: FirebaseAI - @StateObject var viewModel: PhotoReasoningViewModel - - init(firebaseService: FirebaseAI) { - self.firebaseService = firebaseService - _viewModel = - StateObject(wrappedValue: PhotoReasoningViewModel(firebaseService: firebaseService)) - } - - enum FocusedField: Hashable { - case message - } - - @FocusState - var focusedField: FocusedField? - - var body: some View { - VStack { - MultimodalInputField(text: $viewModel.userInput, selection: $viewModel.selectedItems) - .focused($focusedField, equals: .message) - .onSubmit { - onSendTapped() - } - - ScrollViewReader { scrollViewProxy in - List { - if let outputText = viewModel.outputText { - HStack(alignment: .top) { - if viewModel.inProgress { - ProgressView() - } else { - Image(systemName: "cloud.circle.fill") - .font(.title2) - } - - Markdown("\(outputText)") - } - .listRowSeparator(.hidden) - } - } - .listStyle(.plain) - } - } - .onTapGesture { - focusedField = nil - } - .navigationTitle("Multimodal example") - .onAppear { - focusedField = .message - } - } - - // MARK: - Actions - - private func onSendTapped() { - focusedField = nil - - Task { - await viewModel.reason() - } - } -} - -#Preview { - NavigationStack { - PhotoReasoningScreen(firebaseService: FirebaseAI.firebaseAI()) - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/ViewModels/PhotoReasoningViewModel.swift b/firebaseai/GenerativeAIMultimodalExample/ViewModels/PhotoReasoningViewModel.swift deleted file mode 100644 index 24a2e96e2..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/ViewModels/PhotoReasoningViewModel.swift +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import FirebaseAI -import Foundation -import OSLog -import PhotosUI -import SwiftUI - -@MainActor -class PhotoReasoningViewModel: ObservableObject { - // Maximum value for the larger of the two image dimensions (height and width) in pixels. This is - // being used to reduce the image size in bytes. - private static let largestImageDimension = 768.0 - - private var logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "generative-ai") - - @Published - var userInput: String = "" - - @Published - var selectedItems = [PhotosPickerItem]() - - @Published - var outputText: String? = nil - - @Published - var errorMessage: String? - - @Published - var inProgress = false - - private var model: GenerativeModel? - - init(firebaseService: FirebaseAI) { - model = firebaseService.generativeModel(modelName: "gemini-2.0-flash-001") - } - - func reason() async { - defer { - inProgress = false - } - guard let model else { - return - } - - do { - inProgress = true - errorMessage = nil - outputText = "" - - let prompt = "Look at the image(s), and then answer the following question: \(userInput)" - - var images = [any PartsRepresentable]() - for item in selectedItems { - if let data = try? await item.loadTransferable(type: Data.self) { - guard let image = UIImage(data: data) else { - logger.error("Failed to parse data as an image, skipping.") - continue - } - if image.size.fits(largestDimension: PhotoReasoningViewModel.largestImageDimension) { - images.append(image) - } else { - guard let resizedImage = image - .preparingThumbnail(of: image.size - .aspectFit(largestDimension: PhotoReasoningViewModel.largestImageDimension)) else { - logger.error("Failed to resize image: \(image)") - continue - } - - images.append(resizedImage) - } - } - } - - let outputContentStream = try model.generateContentStream(prompt, images) - - // stream response - for try await outputContent in outputContentStream { - guard let line = outputContent.text else { - return - } - - outputText = (outputText ?? "") + line - } - } catch { - logger.error("\(error.localizedDescription)") - errorMessage = error.localizedDescription - } - } -} - -private extension CGSize { - func fits(largestDimension length: CGFloat) -> Bool { - return width <= length && height <= length - } - - func aspectFit(largestDimension length: CGFloat) -> CGSize { - let aspectRatio = width / height - if width > height { - let width = min(self.width, length) - return CGSize(width: width, height: round(width / aspectRatio)) - } else { - let height = min(self.height, length) - return CGSize(width: round(height * aspectRatio), height: height) - } - } -} diff --git a/firebaseai/GenerativeAITextExample/Assets.xcassets/AccentColor.colorset/Contents.json b/firebaseai/GenerativeAITextExample/Assets.xcassets/AccentColor.colorset/Contents.json deleted file mode 100644 index eb8789700..000000000 --- a/firebaseai/GenerativeAITextExample/Assets.xcassets/AccentColor.colorset/Contents.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "colors" : [ - { - "idiom" : "universal" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAITextExample/Assets.xcassets/AppIcon.appiconset/Contents.json b/firebaseai/GenerativeAITextExample/Assets.xcassets/AppIcon.appiconset/Contents.json deleted file mode 100644 index 13613e3ee..000000000 --- a/firebaseai/GenerativeAITextExample/Assets.xcassets/AppIcon.appiconset/Contents.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "images" : [ - { - "idiom" : "universal", - "platform" : "ios", - "size" : "1024x1024" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAITextExample/Assets.xcassets/Contents.json b/firebaseai/GenerativeAITextExample/Assets.xcassets/Contents.json deleted file mode 100644 index 73c00596a..000000000 --- a/firebaseai/GenerativeAITextExample/Assets.xcassets/Contents.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAITextExample/Preview Content/Preview Assets.xcassets/Contents.json b/firebaseai/GenerativeAITextExample/Preview Content/Preview Assets.xcassets/Contents.json deleted file mode 100644 index 73c00596a..000000000 --- a/firebaseai/GenerativeAITextExample/Preview Content/Preview Assets.xcassets/Contents.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAITextExample/Screens/GenerateContentScreen.swift b/firebaseai/GenerativeAITextExample/Screens/GenerateContentScreen.swift deleted file mode 100644 index 2d1648d5a..000000000 --- a/firebaseai/GenerativeAITextExample/Screens/GenerateContentScreen.swift +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import MarkdownUI -import SwiftUI -import FirebaseAI -import GenerativeAIUIComponents - -struct GenerateContentScreen: View { - let firebaseService: FirebaseAI - @StateObject var viewModel: GenerateContentViewModel - @State var userInput = "" - - init(firebaseService: FirebaseAI) { - self.firebaseService = firebaseService - _viewModel = - StateObject(wrappedValue: GenerateContentViewModel(firebaseService: firebaseService)) - } - - enum FocusedField: Hashable { - case message - } - - @FocusState - var focusedField: FocusedField? - - var body: some View { - VStack { - VStack(alignment: .leading) { - Text("Enter some text, then tap on _Go_ to run generateContent on it.") - .padding(.horizontal, 6) - InputField("Enter generate content input", text: $userInput) { - Text("Go") - } - .focused($focusedField, equals: .message) - .onSubmit { onGenerateContentTapped() } - } - .padding(.horizontal, 16) - - List { - HStack(alignment: .top) { - if viewModel.inProgress { - ProgressView() - } else { - Image(systemName: "cloud.circle.fill") - .font(.title2) - } - - Markdown("\(viewModel.outputText)") - } - .listRowSeparator(.hidden) - } - .listStyle(.plain) - } - .onTapGesture { - focusedField = nil - } - .navigationTitle("Text example") - } - - private func onGenerateContentTapped() { - focusedField = nil - - Task { - await viewModel.generateContent(inputText: userInput) - } - } -} - -#Preview { - NavigationStack { - GenerateContentScreen(firebaseService: FirebaseAI.firebaseAI()) - } -} diff --git a/firebaseai/GenerativeAITextExample/ViewModels/GenerateContentViewModel.swift b/firebaseai/GenerativeAITextExample/ViewModels/GenerateContentViewModel.swift deleted file mode 100644 index a9272ef5b..000000000 --- a/firebaseai/GenerativeAITextExample/ViewModels/GenerateContentViewModel.swift +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import FirebaseAI -import Foundation -import OSLog - -@MainActor -class GenerateContentViewModel: ObservableObject { - private var logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "generative-ai") - - @Published - var outputText = "" - - @Published - var errorMessage: String? - - @Published - var inProgress = false - - private var model: GenerativeModel? - - init(firebaseService: FirebaseAI) { - model = firebaseService.generativeModel(modelName: "gemini-2.0-flash-001") - } - - func generateContent(inputText: String) async { - defer { - inProgress = false - } - guard let model else { - return - } - - do { - inProgress = true - errorMessage = nil - outputText = "" - - let outputContentStream = try model.generateContentStream(inputText) - - // stream response - for try await outputContent in outputContentStream { - guard let line = outputContent.text else { - return - } - - outputText = outputText + line - } - } catch { - logger.error("\(error.localizedDescription)") - errorMessage = error.localizedDescription - } - } -} diff --git a/firebaseai/GroundingExample/Screens/GroundingScreen.swift b/firebaseai/GroundingExample/Screens/GroundingScreen.swift new file mode 100644 index 000000000..506199ec7 --- /dev/null +++ b/firebaseai/GroundingExample/Screens/GroundingScreen.swift @@ -0,0 +1,69 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseAI +import SwiftUI +import ConversationKit + +struct GroundingScreen: View { + let firebaseService: FirebaseAI + @StateObject var viewModel: GroundingViewModel + + init(firebaseService: FirebaseAI, sample: Sample? = nil) { + self.firebaseService = firebaseService + _viewModel = + StateObject(wrappedValue: GroundingViewModel(firebaseService: firebaseService, + sample: sample)) + } + + var body: some View { + NavigationStack { + ConversationView(messages: $viewModel.messages, + userPrompt: viewModel.initialPrompt) { message in + MessageView(message: message) + } + .disableAttachments() + .onSendMessage { message in + Task { + await viewModel.sendMessage(message.content ?? "", streaming: true) + } + } + .onError { error in + viewModel.presentErrorDetails = true + } + .sheet(isPresented: $viewModel.presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button(action: newChat) { + Image(systemName: "square.and.pencil") + } + } + } + .navigationTitle(viewModel.title) + .navigationBarTitleDisplayMode(.inline) + } + } + + private func newChat() { + viewModel.startNewChat() + } +} + +#Preview { + GroundingScreen(firebaseService: FirebaseAI.firebaseAI()) +} diff --git a/firebaseai/GroundingExample/ViewModels/GroundingViewMoel.swift b/firebaseai/GroundingExample/ViewModels/GroundingViewMoel.swift new file mode 100644 index 000000000..9977519be --- /dev/null +++ b/firebaseai/GroundingExample/ViewModels/GroundingViewMoel.swift @@ -0,0 +1,170 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseAI +import Foundation +import UIKit + +@MainActor +class GroundingViewModel: ObservableObject { + /// This array holds both the user's and the system's chat messages + @Published var messages = [ChatMessage]() + + /// Indicates we're waiting for the model to finish + @Published var busy = false + + @Published var error: Error? + var hasError: Bool { + return error != nil + } + + @Published var presentErrorDetails: Bool = false + + @Published var initialPrompt: String = "" + @Published var title: String = "" + + private var model: GenerativeModel + private var chat: Chat + + private var chatTask: Task? + + private var sample: Sample? + + init(firebaseService: FirebaseAI, sample: Sample? = nil) { + self.sample = sample + + model = firebaseService.generativeModel( + modelName: sample?.modelName ?? "gemini-2.5-flash", + tools: sample?.tools, + systemInstruction: sample?.systemInstruction + ) + + chat = model.startChat() + + initialPrompt = sample?.initialPrompt ?? "" + title = sample?.title ?? "" + } + + func sendMessage(_ text: String, streaming: Bool = true) async { + error = nil + if streaming { + await internalSendMessageStreaming(text) + } else { + await internalSendMessage(text) + } + } + + func startNewChat() { + stop() + error = nil + chat = model.startChat() + messages.removeAll() + initialPrompt = "" + } + + func stop() { + chatTask?.cancel() + error = nil + } + + private func internalSendMessageStreaming(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(content: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .other) + messages.append(systemMessage) + + do { + let responseStream = try chat.sendMessageStream(text) + for try await chunk in responseStream { + messages[messages.count - 1].pending = false + if let text = chunk.text { + messages[messages.count - 1] + .content = (messages[messages.count - 1].content ?? "") + text + } + + if let candidate = chunk.candidates.first { + if let groundingMetadata = candidate.groundingMetadata { + self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata + } + } + } + + } catch { + self.error = error + print(error.localizedDescription) + let errorMessage = ChatMessage(content: "An error occurred. Please try again.", + participant: .other, + error: error, + pending: false) + messages[messages.count - 1] = errorMessage + } + } + } + + private func internalSendMessage(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(content: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .other) + messages.append(systemMessage) + + do { + var response: GenerateContentResponse? + response = try await chat.sendMessage(text) + + if let responseText = response?.text { + // replace pending message with backend response + messages[messages.count - 1].content = responseText + messages[messages.count - 1].pending = false + + if let candidate = response?.candidates.first { + if let groundingMetadata = candidate.groundingMetadata { + self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata + } + } + } + + } catch { + self.error = error + print(error.localizedDescription) + let errorMessage = ChatMessage(content: "An error occurred. Please try again.", + participant: .other, + error: error, + pending: false) + messages[messages.count - 1] = errorMessage + } + } + } +} diff --git a/firebaseai/ChatExample/Views/Grounding/GoogleSearchSuggestionView.swift b/firebaseai/GroundingExample/Views/GoogleSearchSuggestionView.swift similarity index 100% rename from firebaseai/ChatExample/Views/Grounding/GoogleSearchSuggestionView.swift rename to firebaseai/GroundingExample/Views/GoogleSearchSuggestionView.swift diff --git a/firebaseai/ChatExample/Views/Grounding/GroundedResponseView.swift b/firebaseai/GroundingExample/Views/GroundedResponseView.swift similarity index 100% rename from firebaseai/ChatExample/Views/Grounding/GroundedResponseView.swift rename to firebaseai/GroundingExample/Views/GroundedResponseView.swift diff --git a/firebaseai/ImagenScreen/ImagenScreen.swift b/firebaseai/ImagenExample/ImagenScreen.swift similarity index 98% rename from firebaseai/ImagenScreen/ImagenScreen.swift rename to firebaseai/ImagenExample/ImagenScreen.swift index 4d546dc94..c4e53da16 100644 --- a/firebaseai/ImagenScreen/ImagenScreen.swift +++ b/firebaseai/ImagenExample/ImagenScreen.swift @@ -47,7 +47,7 @@ struct ImagenScreen: View { .disableAttachments() .onSubmitAction { sendOrStop() } - if let error = viewModel.error { + if viewModel.error != nil { HStack { Text("An error occurred.") Button("More information", systemImage: "info.circle") { diff --git a/firebaseai/ImagenScreen/ImagenViewModel.swift b/firebaseai/ImagenExample/ImagenViewModel.swift similarity index 100% rename from firebaseai/ImagenScreen/ImagenViewModel.swift rename to firebaseai/ImagenExample/ImagenViewModel.swift diff --git a/firebaseai/MultimodalExample/Models/MultimodalAttachment.swift b/firebaseai/MultimodalExample/Models/MultimodalAttachment.swift new file mode 100644 index 000000000..3fd707983 --- /dev/null +++ b/firebaseai/MultimodalExample/Models/MultimodalAttachment.swift @@ -0,0 +1,204 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import PhotosUI +import FirebaseAI + +public enum AttachmentType: String, CaseIterable { + case image = "IMAGE" + case video = "VIDEO" + case audio = "AUDIO" + case pdf = "PDF" + case link = "LINK" + case unknown = "UNKNOWN" +} + +public enum AttachmentLoadingState { + case idle + case loading + case loaded + case failed(Error) +} + +public struct MultimodalAttachment: Identifiable, Equatable { + public let id = UUID() + public let type: AttachmentType + public let fileName: String + public let mimeType: String + public let data: Data + public let url: URL? + public let thumbnailImage: UIImage? + public var loadingState: AttachmentLoadingState = .idle + + public static func == (lhs: MultimodalAttachment, rhs: MultimodalAttachment) -> Bool { + return lhs.id == rhs.id + } + + public init(type: AttachmentType, fileName: String, mimeType: String, data: Data, + url: URL? = nil, thumbnailImage: UIImage? = nil) { + self.type = type + self.fileName = fileName + self.mimeType = mimeType + self.data = data + self.url = url + self.thumbnailImage = thumbnailImage + } + + public static func fromPhotosPickerItem(_ item: PhotosPickerItem) async -> MultimodalAttachment? { + guard let data = try? await item.loadTransferable(type: Data.self) else { + return nil + } + + if let image = UIImage(data: data) { + return MultimodalAttachment( + type: .image, + fileName: "Local Image", + mimeType: "image/jpeg", + data: data, + thumbnailImage: image + ) + } else { + return MultimodalAttachment( + type: .video, + fileName: "Local Video", + mimeType: "video/mp4", + data: data + ) + } + } + + public static func fromFilePickerItem(from url: URL) async -> MultimodalAttachment? { + do { + let data = try await Task.detached(priority: .utility) { + try Data(contentsOf: url) + }.value + let fileName = url.lastPathComponent + let mimeType = Self.getMimeType(for: url) + let fileType = Self.getAttachmentType(for: mimeType) + return MultimodalAttachment( + type: fileType, + fileName: fileName, + mimeType: mimeType, + data: data, + url: url + ) + } catch { + return nil + } + } + + public static func fromURL(_ url: URL, mimeType: String) async -> MultimodalAttachment? { + do { + var data: Data + data = try await Task.detached(priority: .utility) { + try Data(contentsOf: url) + }.value + return MultimodalAttachment( + type: .link, + fileName: url.lastPathComponent, + mimeType: mimeType, + data: data, + url: url + ) + } catch { + return nil + } + } + + public func toInlineDataPart() -> InlineDataPart? { + guard !data.isEmpty else { return nil } + return InlineDataPart(data: data, mimeType: mimeType) + } + + private static func getMimeType(for url: URL) -> String { + let fileExtension = url.pathExtension.lowercased() + + switch fileExtension { + // Documents / text + case "pdf": + return "application/pdf" + case "txt", "text": + return "text/plain" + + // Images + case "jpg", "jpeg": + return "image/jpeg" + case "png": + return "image/png" + case "webp": + return "image/webp" + + // Video + case "flv": + return "video/x-flv" + case "mov", "qt": + return "video/quicktime" + case "mpeg": + return "video/mpeg" + case "mpg": + return "video/mpg" + case "ps": + return "video/mpegps" + case "mp4": + return "video/mp4" + case "webm": + return "video/webm" + case "wmv": + return "video/wmv" + case "3gp", "3gpp": + return "video/3gpp" + + // Audio + case "aac": + return "audio/aac" + case "flac": + return "audio/flac" + case "mp3": + return "audio/mpeg" + case "m4a": + return "audio/m4a" + case "mpga": + return "audio/mpga" + case "mp4a": + return "audio/mp4" + case "opus": + return "audio/opus" + case "pcm", "raw": + return "audio/pcm" + case "wav": + return "audio/wav" + case "weba": + return "audio/webm" + + default: + return "application/octet-stream" + } + } + + private static func getAttachmentType(for mimeType: String) -> AttachmentType { + if mimeType.starts(with: "image/") { + return .image + } else if mimeType.starts(with: "video/") { + return .video + } else if mimeType.starts(with: "audio/") { + return .audio + } else if mimeType == "application/pdf" { + return .pdf + } else { + return .unknown + } + } +} diff --git a/firebaseai/ChatExample/Assets.xcassets/Contents.json b/firebaseai/MultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json similarity index 100% rename from firebaseai/ChatExample/Assets.xcassets/Contents.json rename to firebaseai/MultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json diff --git a/firebaseai/MultimodalExample/Screens/MultimodalScreen.swift b/firebaseai/MultimodalExample/Screens/MultimodalScreen.swift new file mode 100644 index 000000000..fb79e9ce8 --- /dev/null +++ b/firebaseai/MultimodalExample/Screens/MultimodalScreen.swift @@ -0,0 +1,185 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseAI +import SwiftUI +import PhotosUI +import ConversationKit + +struct MultimodalScreen: View { + let firebaseService: FirebaseAI + @StateObject var viewModel: MultimodalViewModel + + @State private var showingPhotoPicker = false + @State private var showingFilePicker = false + @State private var showingLinkDialog = false + @State private var linkText = "" + @State private var linkMimeType = "" + @State private var selectedPhotoItems = [PhotosPickerItem]() + + init(firebaseService: FirebaseAI, sample: Sample? = nil) { + self.firebaseService = firebaseService + _viewModel = + StateObject(wrappedValue: MultimodalViewModel(firebaseService: firebaseService, + sample: sample)) + } + + private var attachmentPreviewScrollView: some View { + AttachmentPreviewScrollView( + attachments: viewModel.attachments, + onAttachmentRemove: viewModel.removeAttachment + ) + } + + var body: some View { + NavigationStack { + ConversationView(messages: $viewModel.messages, + userPrompt: viewModel.initialPrompt) { message in + MessageView(message: message) + } + .attachmentActions { + Button(action: showLinkDialog) { + Label("Link", systemImage: "link") + } + Button(action: showFilePicker) { + Label("File", systemImage: "doc.text") + } + Button(action: showPhotoPicker) { + Label("Photo", systemImage: "photo.on.rectangle.angled") + } + } + .attachmentPreview { attachmentPreviewScrollView } + .onSendMessage { message in + Task { + await viewModel.sendMessage(message.content ?? "", streaming: true) + } + } + .onError { error in + viewModel.presentErrorDetails = true + } + .sheet(isPresented: $viewModel.presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .photosPicker( + isPresented: $showingPhotoPicker, + selection: $selectedPhotoItems, + maxSelectionCount: 5, + matching: .any(of: [.images, .videos]) + ) + .fileImporter( + isPresented: $showingFilePicker, + allowedContentTypes: [.pdf, .audio], + allowsMultipleSelection: true + ) { result in + handleFileImport(result) + } + .alert("Add Web URL", isPresented: $showingLinkDialog) { + TextField("Enter URL", text: $linkText) + TextField("Enter mimeType", text: $linkMimeType) + Button("Add") { + handleLinkAttachment() + } + Button("Cancel", role: .cancel) { + linkText = "" + linkMimeType = "" + } + } + } + .onChange(of: selectedPhotoItems) { _, newItems in + handlePhotoSelection(newItems) + } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button(action: newChat) { + Image(systemName: "square.and.pencil") + } + } + } + .navigationTitle(viewModel.title) + .navigationBarTitleDisplayMode(.inline) + } + + private func newChat() { + viewModel.startNewChat() + } + + private func showPhotoPicker() { + showingPhotoPicker = true + } + + private func showFilePicker() { + showingFilePicker = true + } + + private func showLinkDialog() { + showingLinkDialog = true + } + + private func handlePhotoSelection(_ items: [PhotosPickerItem]) { + Task { + for item in items { + if let attachment = await MultimodalAttachment.fromPhotosPickerItem(item) { + await MainActor.run { + viewModel.addAttachment(attachment) + } + } + } + await MainActor.run { + selectedPhotoItems = [] + } + } + } + + private func handleFileImport(_ result: Result<[URL], Error>) { + switch result { + case let .success(urls): + Task { + for url in urls { + if let attachment = await MultimodalAttachment.fromFilePickerItem(from: url) { + await MainActor.run { + viewModel.addAttachment(attachment) + } + } + } + } + case let .failure(error): + viewModel.handleError(error) + } + } + + private func handleLinkAttachment() { + guard !linkText.isEmpty, let url = URL(string: linkText) else { + return + } + + let trimmedMime = linkMimeType.trimmingCharacters(in: .whitespacesAndNewlines) + Task { + if let attachment = await MultimodalAttachment.fromURL(url, mimeType: trimmedMime) { + await MainActor.run { + viewModel.addAttachment(attachment) + } + } + await MainActor.run { + linkText = "" + linkMimeType = "" + } + } + } +} + +#Preview { + MultimodalScreen(firebaseService: FirebaseAI.firebaseAI()) +} diff --git a/firebaseai/MultimodalExample/ViewModels/MultimodalViewModel.swift b/firebaseai/MultimodalExample/ViewModels/MultimodalViewModel.swift new file mode 100644 index 000000000..e8f5c69cc --- /dev/null +++ b/firebaseai/MultimodalExample/ViewModels/MultimodalViewModel.swift @@ -0,0 +1,192 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseAI +import Foundation +import OSLog +import PhotosUI +import SwiftUI +import AVFoundation +import UniformTypeIdentifiers + +@MainActor +class MultimodalViewModel: ObservableObject { + @Published var messages = [ChatMessage]() + @Published var initialPrompt: String = "" + @Published var title: String = "" + @Published var error: Error? + @Published var inProgress = false + + @Published var presentErrorDetails: Bool = false + + @Published var attachments = [MultimodalAttachment]() + + private var model: GenerativeModel + private var chat: Chat + private var chatTask: Task? + private var sample: Sample? + private let logger = Logger(subsystem: "com.example.firebaseai", category: "MultimodalViewModel") + + init(firebaseService: FirebaseAI, sample: Sample? = nil) { + self.sample = sample + + model = firebaseService.generativeModel( + modelName: "gemini-2.0-flash-001", + systemInstruction: sample?.systemInstruction + ) + + chat = model.startChat() + initialPrompt = sample?.initialPrompt ?? "" + title = sample?.title ?? "" + + if let urlMetas = sample?.attachedURLs { + Task { + for urlMeta in urlMetas { + if let attachment = await MultimodalAttachment.fromURL( + urlMeta.url, + mimeType: urlMeta.mimeType + ) { + self.addAttachment(attachment) + } + } + } + } + } + + func sendMessage(_ text: String, streaming: Bool = true) async { + error = nil + if streaming { + await internalSendMessageStreaming(text) + } else { + await internalSendMessage(text) + } + } + + func startNewChat() { + stop() + error = nil + chat = model.startChat() + messages.removeAll() + attachments.removeAll() + initialPrompt = "" + } + + func stop() { + chatTask?.cancel() + error = nil + } + + private func internalSendMessageStreaming(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + inProgress = true + defer { + inProgress = false + } + + let userMessage = ChatMessage(content: text, participant: .user, attachments: attachments) + messages.append(userMessage) + let systemMessage = ChatMessage.pending(participant: .other) + messages.append(systemMessage) + + do { + var parts: [any PartsRepresentable] = [text] + + for attachment in attachments { + if let inlineDataPart = attachment.toInlineDataPart() { + parts.append(inlineDataPart) + } + } + + attachments.removeAll() + + let responseStream = try chat.sendMessageStream(parts) + for try await chunk in responseStream { + messages[messages.count - 1].pending = false + if let text = chunk.text { + messages[messages.count - 1] + .content = (messages[messages.count - 1].content ?? "") + text + } + } + } catch { + self.error = error + logger.error("\(error.localizedDescription)") + let errorMessage = ChatMessage(content: "An error occurred. Please try again.", + participant: .other, + error: error, + pending: false) + messages[messages.count - 1] = errorMessage + } + } + } + + private func internalSendMessage(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + inProgress = true + defer { + inProgress = false + } + let userMessage = ChatMessage(content: text, participant: .user, attachments: attachments) + messages.append(userMessage) + + let systemMessage = ChatMessage.pending(participant: .other) + messages.append(systemMessage) + + do { + var parts: [any PartsRepresentable] = [text] + + for attachment in attachments { + if let inlineDataPart = attachment.toInlineDataPart() { + parts.append(inlineDataPart) + } + } + + attachments.removeAll() + + let response = try await chat.sendMessage(parts) + + if let responseText = response.text { + messages[messages.count - 1].content = responseText + messages[messages.count - 1].pending = false + } + } catch { + self.error = error + logger.error("\(error.localizedDescription)") + let errorMessage = ChatMessage(content: "An error occurred. Please try again.", + participant: .other, + error: error, + pending: false) + messages[messages.count - 1] = errorMessage + } + } + } + + func addAttachment(_ attachment: MultimodalAttachment) { + var newAttachment = attachment + newAttachment.loadingState = .loaded + attachments.append(newAttachment) + } + + func removeAttachment(_ attachment: MultimodalAttachment) { + attachments.removeAll { $0.id == attachment.id } + } + + func handleError(_ error: Error) { + self.error = error + logger.error("\(error.localizedDescription)") + } +} diff --git a/firebaseai/MultimodalExample/Views/AttachmentPreviewCard.swift b/firebaseai/MultimodalExample/Views/AttachmentPreviewCard.swift new file mode 100644 index 000000000..debbb76a0 --- /dev/null +++ b/firebaseai/MultimodalExample/Views/AttachmentPreviewCard.swift @@ -0,0 +1,212 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +struct AttachmentPreviewCard: View { + let attachment: MultimodalAttachment + let onRemove: (() -> Void)? + + var body: some View { + HStack(spacing: 12) { + Group { + if let thumbnailImage = attachment.thumbnailImage { + Image(uiImage: thumbnailImage) + .resizable() + .aspectRatio(contentMode: .fill) + .frame(width: 40, height: 40) + .clipShape(RoundedRectangle(cornerRadius: 6)) + } else { + Image(systemName: systemImageName) + .font(.system(size: 20)) + .foregroundColor(.blue) + .frame(width: 40, height: 40) + .background(Color.blue.opacity(0.1)) + .clipShape(RoundedRectangle(cornerRadius: 6)) + } + } + + VStack(alignment: .leading, spacing: 4) { + Text(displayName) + .font(.system(size: 14, weight: .medium)) + .lineLimit(1) + .truncationMode(.middle) + .foregroundColor(.primary) + + HStack(spacing: 8) { + Text(attachment.type.rawValue) + .font(.system(size: 10, weight: .semibold)) + .padding(.horizontal, 6) + .padding(.vertical, 2) + .background(typeTagColor) + .foregroundColor(.white) + .clipShape(Capsule()) + + if case .loading = attachment.loadingState { + ProgressView() + .scaleEffect(0.7) + .progressViewStyle(CircularProgressViewStyle(tint: .blue)) + } else if case .failed = attachment.loadingState { + Image(systemName: "exclamationmark.triangle.fill") + .font(.system(size: 12)) + .foregroundColor(.red) + } + Spacer() + } + } + + if let onRemove = onRemove { + Button(action: onRemove) { + Image(systemName: "xmark.circle.fill") + .font(.system(size: 16)) + .foregroundColor(.gray) + } + .buttonStyle(PlainButtonStyle()) + } + } + .padding(12) + .background(Color(.systemGray6)) + .clipShape(RoundedRectangle(cornerRadius: 12)) + .overlay( + RoundedRectangle(cornerRadius: 12) + .stroke(Color(.separator), lineWidth: 0.5) + ) + } + + private var typeTagColor: Color { + switch attachment.type { + case .image: + return .green + case .video: + return .purple + case .audio: + return .orange + case .pdf: + return .red + case .link: + return .blue + case .unknown: + return .gray + } + } + + private var systemImageName: String { + switch attachment.type { + case .image: + return "photo" + case .video: + return "video" + case .audio: + return "waveform" + case .pdf: + return "doc.text" + case .link: + return "link" + case .unknown: + return "questionmark" + } + } + + private var displayName: String { + if attachment.fileName.count > 30 { + let startIndex = attachment.fileName.startIndex + let endIndex = attachment.fileName.endIndex + let start = String(attachment + .fileName[startIndex ..< attachment.fileName.index(startIndex, offsetBy: 15)]) + let end = String(attachment + .fileName[attachment.fileName.index(endIndex, offsetBy: -10) ..< endIndex]) + return "\(start)...\(end)" + } + return attachment.fileName + } +} + +struct AttachmentPreviewScrollView: View { + let attachments: [MultimodalAttachment] + var onAttachmentRemove: ((MultimodalAttachment) -> Void)? = nil + + var body: some View { + if !attachments.isEmpty { + ScrollView(.horizontal, showsIndicators: false) { + LazyHStack(spacing: 8) { + ForEach(attachments) { attachment in + AttachmentPreviewCard( + attachment: attachment, + onRemove: onAttachmentRemove == nil ? nil : { onAttachmentRemove?(attachment) } + ) + .frame(width: 200) + } + } + .padding(.horizontal, 16) + } + .frame(height: 80) + } else { + EmptyView() + } + } +} + +#Preview { + VStack(spacing: 20) { + AttachmentPreviewCard( + attachment: MultimodalAttachment( + type: .image, + fileName: "IMG_1234_very_long_filename_example.jpg", + mimeType: "image/jpeg", + data: Data(), + thumbnailImage: UIImage(systemName: "photo"), + ), + onRemove: { print("Image removed") } + ) + + AttachmentPreviewCard( + attachment: MultimodalAttachment( + type: .pdf, + fileName: "Document.pdf", + mimeType: "application/pdf", + data: Data(), + ), + onRemove: { print("PDF removed") } + ) + + AttachmentPreviewCard( + attachment: { + var attachment = MultimodalAttachment( + type: .video, + fileName: "video.mp4", + mimeType: "video/mp4", + data: Data(), + ) + attachment.loadingState = .loading + return attachment + }(), + onRemove: { print("Video removed") } + ) + + AttachmentPreviewCard( + attachment: { + var attachment = MultimodalAttachment( + type: .audio, + fileName: "audio.mp3", + mimeType: "audio/mpeg", + data: Data(), + ) + attachment.loadingState = .failed(NSError(domain: "TestError", code: 1, userInfo: nil)) + return attachment + }(), + onRemove: { print("Audio removed") } + ) + } + .padding() +} diff --git a/firebaseai/UIComponents/Models/Sample.swift b/firebaseai/UIComponents/Models/Sample.swift index 65649235b..fe26c56fa 100644 --- a/firebaseai/UIComponents/Models/Sample.swift +++ b/firebaseai/UIComponents/Models/Sample.swift @@ -15,33 +15,51 @@ import Foundation import FirebaseAI +public struct URLMetadata { + public let mimeType: String + public let url: URL + public init(mimeType: String, url: URL) { + self.mimeType = mimeType + self.url = url + } +} + public struct Sample: Identifiable { public let id = UUID() public let title: String public let description: String public let useCases: [UseCase] public let navRoute: String + public let modelName: String public let chatHistory: [ModelContent]? public let initialPrompt: String? public let systemInstruction: ModelContent? public let tools: [Tool]? + public let generationConfig: GenerationConfig? + public let attachedURLs: [URLMetadata]? public init(title: String, description: String, useCases: [UseCase], navRoute: String, + modelName: String = "gemini-2.5-flash", chatHistory: [ModelContent]? = nil, initialPrompt: String? = nil, systemInstruction: ModelContent? = nil, - tools: [Tool]? = nil) { + tools: [Tool]? = nil, + generationConfig: GenerationConfig? = nil, + attachedURLs: [URLMetadata]? = nil) { self.title = title self.description = description self.useCases = useCases self.navRoute = navRoute + self.modelName = modelName self.chatHistory = chatHistory self.initialPrompt = initialPrompt self.systemInstruction = systemInstruction self.tools = tools + self.generationConfig = generationConfig + self.attachedURLs = attachedURLs } } @@ -90,94 +108,102 @@ extension Sample { title: "Blog post creator", description: "Create a blog post from an image file stored in Cloud Storage.", useCases: [.image], - navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you help me create a blog post about this image?"), - ModelContent( - role: "model", - parts: "I'd be happy to help you create a blog post! Please share the image you'd like me to analyze and write about." + navRoute: "MultimodalScreen", + initialPrompt: "Write a short, engaging blog post based on this picture." + + " It should include a description of the meal in the" + + " photo and talk about my journey meal prepping.", + attachedURLs: [ + URLMetadata( + mimeType: "image/jpeg", + url: URL( + string: "https://storage.googleapis.com/cloud-samples-data/generative-ai/image/meal-prep.jpeg" + )! ), - ], - initialPrompt: "Please analyze this image and create an engaging blog post" + ] ), Sample( - title: "Imagen 3 - image generation", + title: "Imagen - image generation", description: "Generate images using Imagen 3", useCases: [.image], navRoute: "ImagenScreen", initialPrompt: "A photo of a modern building with water in the background" ), Sample( - title: "Gemini 2.0 Flash - image generation", + title: "Gemini Flash - image generation", description: "Generate and/or edit images using Gemini 2.0 Flash", useCases: [.image], navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you edit this image to make it brighter?"), - ModelContent( - role: "model", - parts: "I can help you edit images using Gemini 2.0 Flash. Please share the image you'd like me to modify." - ), - ], - initialPrompt: "" + modelName: "gemini-2.0-flash-preview-image-generation", + initialPrompt: "Hi, can you create a 3d rendered image of a pig " + + "with wings and a top hat flying over a happy " + + "futuristic scifi city with lots of greenery?", + generationConfig: GenerationConfig(responseModalities: [.text, .image]), ), // Video Sample( title: "Hashtags for a video", description: "Generate hashtags for a video ad stored in Cloud Storage.", useCases: [.video], - navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you suggest hashtags for my product video?"), - ModelContent( - role: "model", - parts: "I'd be happy to help you generate relevant hashtags! Please share your video or describe what it's about so I can suggest appropriate hashtags." + navRoute: "MultimodalScreen", + initialPrompt: "Generate 5-10 hashtags that relate to the video content." + + " Try to use more popular and engaging terms," + + " e.g. #Viral. Do not add content not related to" + + " the video.\n Start the output with 'Tags:'", + attachedURLs: [ + URLMetadata( + mimeType: "video/mp4", + url: URL( + string: "https://storage.googleapis.com/cloud-samples-data/generative-ai/video/google_home_celebrity_ad.mp4" + )! ), - ], - initialPrompt: "" + ] ), Sample( title: "Summarize video", description: "Summarize a video and extract important dialogue.", useCases: [.video], - navRoute: "ChatScreen", + navRoute: "MultimodalScreen", chatHistory: [ - ModelContent(role: "user", parts: "Can you summarize this video for me?"), + ModelContent(role: "user", parts: "Can you help me with the description of a video file?"), ModelContent( role: "model", - parts: "I can help you summarize videos and extract key dialogue. Please share the video you'd like me to analyze." + parts: "Sure! Click on the attach button below and choose a video file for me to describe." ), ], - initialPrompt: "" + initialPrompt: "I have attached the video file. Provide a description of" + + " the video. The description should also contain" + + " anything important which people say in the video." ), // Audio Sample( title: "Audio Summarization", description: "Summarize an audio file", useCases: [.audio], - navRoute: "ChatScreen", + navRoute: "MultimodalScreen", chatHistory: [ - ModelContent(role: "user", parts: "Can you summarize this audio recording?"), + ModelContent(role: "user", parts: "Can you help me summarize an audio file?"), ModelContent( role: "model", - parts: "I can help you summarize audio files. Please share the audio recording you'd like me to analyze." + parts: "Of course! Click on the attach button below and choose an audio file for me to summarize." ), ], - initialPrompt: "" + initialPrompt: "I have attached the audio file. Please analyze it and summarize the contents" + + " of the audio as bullet points." ), Sample( title: "Translation from audio", description: "Translate an audio file stored in Cloud Storage", useCases: [.audio], - navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you translate this audio from Spanish to English?"), - ModelContent( - role: "model", - parts: "I can help you translate audio files. Please share the audio file you'd like me to translate." + navRoute: "MultimodalScreen", + initialPrompt: "Please translate the audio in Mandarin.", + attachedURLs: [ + URLMetadata( + mimeType: "audio/mp3", + url: URL( + string: "https://storage.googleapis.com/cloud-samples-data/generative-ai/audio/How_to_create_a_My_Map_in_Google_Maps.mp3" + )! ), - ], - initialPrompt: "" + ] ), // Document Sample( @@ -185,15 +211,23 @@ extension Sample { description: "Compare the contents of 2 documents." + " Only supported by the Vertex AI Gemini API because the documents are stored in Cloud Storage", useCases: [.document], - navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you compare these two documents for me?"), - ModelContent( - role: "model", - parts: "I can help you compare documents using the Vertex AI Gemini API. Please share the two documents you'd like me to compare." + navRoute: "MultimodalScreen", + initialPrompt: "The first document is from 2013, and the second document is" + + " from 2023. How did the standard deduction evolve?", + attachedURLs: [ + URLMetadata( + mimeType: "application/pdf", + url: URL( + string: "https://storage.googleapis.com/cloud-samples-data/generative-ai/pdf/form_1040_2013.pdf" + )!, ), - ], - initialPrompt: "" + URLMetadata( + mimeType: "application/pdf", + url: URL( + string: "https://storage.googleapis.com/cloud-samples-data/generative-ai/pdf/form_1040_2023.pdf" + )! + ), + ] ), // Function Calling Sample( @@ -221,7 +255,7 @@ extension Sample { title: "Grounding with Google Search", description: "Use Grounding with Google Search to get responses based on up-to-date information from the web.", useCases: [.text], - navRoute: "ChatScreen", + navRoute: "GroundingScreen", initialPrompt: "What's the weather in Chicago this weekend?", tools: [.googleSearch()] ), diff --git a/firebaseai/UIComponents/Models/UseCase.swift b/firebaseai/UIComponents/Models/UseCase.swift index 5448dc01b..ee4e80f8a 100644 --- a/firebaseai/UIComponents/Models/UseCase.swift +++ b/firebaseai/UIComponents/Models/UseCase.swift @@ -15,6 +15,7 @@ import Foundation public enum UseCase: String, CaseIterable, Identifiable { + case all = "All" case text = "Text" case image = "Image" case video = "Video" diff --git a/firebaseai/UIComponents/Views/InputField.swift b/firebaseai/UIComponents/Views/InputField.swift deleted file mode 100644 index 67941c370..000000000 --- a/firebaseai/UIComponents/Views/InputField.swift +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import SwiftUI - -public struct InputField