From 0e2082d055f82d26c47d9ca1f26f4f92e2a05c21 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 19 Aug 2025 18:15:26 -0400 Subject: [PATCH 1/3] Add App Check Limited-Use mode integration tests --- .../GenerateContentIntegrationTests.swift | 21 +++++++++++-------- .../Tests/Utilities/InstanceConfig.swift | 10 +++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift index 61bcf240442..d976d19ce76 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift @@ -51,7 +51,9 @@ struct GenerateContentIntegrationTests { (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2FlashLite), (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), + (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashLite), + (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemma3_4B), (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), @@ -165,15 +167,16 @@ struct GenerateContentIntegrationTests { (.googleAI_v1beta_freeTier, ModelNames.gemini2_5_Flash, ThinkingConfig( thinkingBudget: 24576, includeThoughts: true )), - (.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2_5_Flash, ThinkingConfig( - thinkingBudget: 0 - )), - (.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2_5_Flash, ThinkingConfig( - thinkingBudget: 24576 - )), - (.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2_5_Flash, ThinkingConfig( - thinkingBudget: 24576, includeThoughts: true - )), + // Note: The following configs are commented out for easy one-off manual testing. + // (.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2_5_Flash, ThinkingConfig( + // thinkingBudget: 0 + // )), + // (.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2_5_Flash, ThinkingConfig( + // thinkingBudget: 24576 + // )), + // (.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2_5_Flash, ThinkingConfig( + // thinkingBudget: 24576, includeThoughts: true + // )), ] as [(InstanceConfig, String, ThinkingConfig)] ) func generateContentThinking(_ config: InstanceConfig, modelName: String, diff --git a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift index 2c6dceb24fd..b5a59240fda 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift @@ -27,12 +27,20 @@ struct InstanceConfig: Equatable, Encodable { location: "global", apiConfig: APIConfig(service: .vertexAI(endpoint: .firebaseProxyProd), version: .v1beta) ) + static let vertexAI_v1beta_global_appCheckLimitedUse = InstanceConfig( + location: "global", + useLimitedUseAppCheckTokens: true, + apiConfig: APIConfig(service: .vertexAI(endpoint: .firebaseProxyProd), version: .v1beta) + ) static let vertexAI_v1beta_staging = InstanceConfig( apiConfig: APIConfig(service: .vertexAI(endpoint: .firebaseProxyStaging), version: .v1beta) ) static let googleAI_v1beta = InstanceConfig( apiConfig: APIConfig(service: .googleAI(endpoint: .firebaseProxyProd), version: .v1beta) ) + static let googleAI_v1beta_appCheckLimitedUse = InstanceConfig( + apiConfig: APIConfig(service: .googleAI(endpoint: .firebaseProxyProd), version: .v1beta) + ) static let googleAI_v1beta_staging = InstanceConfig( apiConfig: APIConfig(service: .googleAI(endpoint: .firebaseProxyStaging), version: .v1beta) ) @@ -49,9 +57,11 @@ struct InstanceConfig: Equatable, Encodable { vertexAI_v1beta, vertexAI_v1beta_global, vertexAI_v1beta_staging, + vertexAI_v1beta_global_appCheckLimitedUse, googleAI_v1beta, googleAI_v1beta_staging, googleAI_v1beta_freeTier_bypassProxy, + googleAI_v1beta_appCheckLimitedUse, ] static let vertexAI_v1beta_appCheckNotConfigured = InstanceConfig( From 281301c39e3d4769109ad9bda09364e709563015 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 19 Aug 2025 18:16:40 -0400 Subject: [PATCH 2/3] Disable staging and bypass proxy tests on CI to reduce utilization --- .../GenerateContentIntegrationTests.swift | 33 ++++++++++------ .../Tests/Utilities/InstanceConfig.swift | 39 +++++++++++++++---- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift index d976d19ce76..ef0f19be217 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift @@ -50,15 +50,18 @@ struct GenerateContentIntegrationTests { @Test(arguments: [ (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2FlashLite), - (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemma3_4B), - (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), - (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), - (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2FlashLite), - (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemma3_4B), + (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), + (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma3_4B), + // Note: The following configs are commented out for easy one-off manual testing. + // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), + // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), + // (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), + // (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2FlashLite), + // (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemma3_4B), ]) func generateContent(_ config: InstanceConfig, modelName: String) async throws { let model = FirebaseAI.componentInstance(config).generativeModel( @@ -322,8 +325,9 @@ struct GenerateContentIntegrationTests { InstanceConfig.vertexAI_v1beta, InstanceConfig.vertexAI_v1beta_global, InstanceConfig.googleAI_v1beta, - InstanceConfig.googleAI_v1beta_staging, - InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, + // Note: The following configs are commented out for easy one-off manual testing. + // InstanceConfig.googleAI_v1beta_staging, + // InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ]) func generateImage(_ config: InstanceConfig) async throws { let generationConfig = GenerationConfig( @@ -419,13 +423,18 @@ struct GenerateContentIntegrationTests { @Test(arguments: [ (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2FlashLite), (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2FlashLite), - (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), + (InstanceConfig.vertexAI_v1beta_global_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashLite), + (InstanceConfig.googleAI_v1beta_appCheckLimitedUse, ModelNames.gemini2FlashLite), (InstanceConfig.googleAI_v1beta, ModelNames.gemma3_4B), - (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), - (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), - (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2FlashLite), - (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemma3_4B), + (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashLite), + (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemma3_4B), + // Note: The following configs are commented out for easy one-off manual testing. + // (InstanceConfig.vertexAI_v1beta_staging, ModelNames.gemini2FlashLite), + // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashLite), + // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemma3_4B), + // (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemini2FlashLite), + // (InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, ModelNames.gemma3_4B), ]) func generateContentStream(_ config: InstanceConfig, modelName: String) async throws { let expectedResponse = [ diff --git a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift index b5a59240fda..afc5bfeac78 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift @@ -56,35 +56,56 @@ struct InstanceConfig: Equatable, Encodable { static let allConfigs = [ vertexAI_v1beta, vertexAI_v1beta_global, - vertexAI_v1beta_staging, vertexAI_v1beta_global_appCheckLimitedUse, googleAI_v1beta, - googleAI_v1beta_staging, - googleAI_v1beta_freeTier_bypassProxy, googleAI_v1beta_appCheckLimitedUse, + googleAI_v1beta_freeTier, + // Note: The following configs are commented out for easy one-off manual testing. + // vertexAI_v1beta_staging, + // googleAI_v1beta_staging, + // googleAI_v1beta_freeTier_bypassProxy, ] static let vertexAI_v1beta_appCheckNotConfigured = InstanceConfig( appName: FirebaseAppNames.appCheckNotConfigured, apiConfig: APIConfig(service: .vertexAI(endpoint: .firebaseProxyProd), version: .v1beta) ) + static let vertexAI_v1beta_appCheckNotConfigured_limitedUseTokens = InstanceConfig( + appName: FirebaseAppNames.appCheckNotConfigured, + useLimitedUseAppCheckTokens: true, + apiConfig: APIConfig(service: .vertexAI(endpoint: .firebaseProxyProd), version: .v1beta) + ) static let googleAI_v1beta_appCheckNotConfigured = InstanceConfig( appName: FirebaseAppNames.appCheckNotConfigured, apiConfig: APIConfig(service: .googleAI(endpoint: .firebaseProxyProd), version: .v1beta) ) + static let googleAI_v1beta_appCheckNotConfigured_limitedUseTokens = InstanceConfig( + appName: FirebaseAppNames.appCheckNotConfigured, + useLimitedUseAppCheckTokens: true, + apiConfig: APIConfig(service: .googleAI(endpoint: .firebaseProxyProd), version: .v1beta) + ) static let appCheckNotConfiguredConfigs = [ vertexAI_v1beta_appCheckNotConfigured, + vertexAI_v1beta_appCheckNotConfigured_limitedUseTokens, googleAI_v1beta_appCheckNotConfigured, + googleAI_v1beta_appCheckNotConfigured_limitedUseTokens ] let appName: String? let location: String? + let useLimitedUseAppCheckTokens: Bool let apiConfig: APIConfig - init(appName: String? = nil, location: String? = nil, apiConfig: APIConfig) { + init( + appName: String? = nil, + location: String? = nil, + useLimitedUseAppCheckTokens: Bool = false, + apiConfig: APIConfig + ) { self.appName = appName self.location = location + self.useLimitedUseAppCheckTokens = useLimitedUseAppCheckTokens self.apiConfig = apiConfig } @@ -118,8 +139,12 @@ extension InstanceConfig: CustomTestStringConvertible { " - Bypass Proxy" } let locationSuffix = location.map { " - \($0)" } ?? "" + let appCheckLimitedUseDesignator = useLimitedUseAppCheckTokens ? " - FAC Limited-Use" : "" - return "\(serviceName) (\(versionName))\(freeTierDesignator)\(endpointSuffix)\(locationSuffix)" + return """ + \(serviceName) (\(versionName))\(freeTierDesignator)\(endpointSuffix)\(locationSuffix)\ + \(appCheckLimitedUseDesignator) + """ } } @@ -132,7 +157,7 @@ extension FirebaseAI { app: instanceConfig.app, location: location, apiConfig: instanceConfig.apiConfig, - useLimitedUseAppCheckTokens: false + useLimitedUseAppCheckTokens: instanceConfig.useLimitedUseAppCheckTokens ) case .googleAI: assert( @@ -143,7 +168,7 @@ extension FirebaseAI { app: instanceConfig.app, location: nil, apiConfig: instanceConfig.apiConfig, - useLimitedUseAppCheckTokens: false + useLimitedUseAppCheckTokens: instanceConfig.useLimitedUseAppCheckTokens ) } } From 05ee97ac2760976aea1a4b49ae84f3b730f85d44 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 19 Aug 2025 18:18:05 -0400 Subject: [PATCH 3/3] Run `style.sh` --- .../TestApp/Tests/Utilities/InstanceConfig.swift | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift index afc5bfeac78..df06f43c91f 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift @@ -89,7 +89,7 @@ struct InstanceConfig: Equatable, Encodable { vertexAI_v1beta_appCheckNotConfigured, vertexAI_v1beta_appCheckNotConfigured_limitedUseTokens, googleAI_v1beta_appCheckNotConfigured, - googleAI_v1beta_appCheckNotConfigured_limitedUseTokens + googleAI_v1beta_appCheckNotConfigured_limitedUseTokens, ] let appName: String? @@ -97,12 +97,10 @@ struct InstanceConfig: Equatable, Encodable { let useLimitedUseAppCheckTokens: Bool let apiConfig: APIConfig - init( - appName: String? = nil, - location: String? = nil, - useLimitedUseAppCheckTokens: Bool = false, - apiConfig: APIConfig - ) { + init(appName: String? = nil, + location: String? = nil, + useLimitedUseAppCheckTokens: Bool = false, + apiConfig: APIConfig) { self.appName = appName self.location = location self.useLimitedUseAppCheckTokens = useLimitedUseAppCheckTokens