From 8b9c47b81711e7677b2e1d07a2394a7d3b90aa0b Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 20 Nov 2025 12:53:01 -0500 Subject: [PATCH 1/8] [CI] Generate report with `xcbeautify` --- .github/workflows/common.yml | 5 +++++ .github/workflows/firebaseai.yml | 5 +++++ scripts/build.sh | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index b2456256c9c..fa40aa67eab 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -123,3 +123,8 @@ jobs: ${{ inputs.target }} \ ${{ matrix.platform }} \ ${{ (contains(inputs.buildonly_platforms, matrix.platform) || contains(inputs.buildonly_platforms, 'all')) && 'spmbuildonly' || 'spm' }} + - name: Publish Test Report + uses: mikepenz/action-junit-report@v6 + if: success() || failure() + with: + report_paths: 'build/reports/junit.xml' diff --git a/.github/workflows/firebaseai.yml b/.github/workflows/firebaseai.yml index eab6b595610..3174e725695 100644 --- a/.github/workflows/firebaseai.yml +++ b/.github/workflows/firebaseai.yml @@ -69,6 +69,11 @@ jobs: name: xcodebuild-${{ matrix.target }}-${{ matrix.os }}-${{ matrix.xcode }}.log path: xcodebuild-*.log retention-days: 2 + - name: Publish Test Report + uses: mikepenz/action-junit-report@v6 + if: success() || failure() + with: + report_paths: 'build/reports/junit.xml' pod_lib_lint: strategy: diff --git a/scripts/build.sh b/scripts/build.sh index f9a79ef0960..aea073a6258 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -113,7 +113,7 @@ function RunXcodebuild() { local buildaction="${xcodebuild_args[$# - 1]}" # buildaction is the last arg local log_filename="xcodebuild-${buildaction}.log" - local xcbeautify_cmd=(xcbeautify --renderer github-actions --disable-logging) + local xcbeautify_cmd=(xcbeautify --renderer github-actions --disable-logging --quiet --is-ci --report junit) local result=0 NSUnbufferedIO=YES xcodebuild "$@" 2>&1 | tee "$log_filename" | \ From 6271061db3df8a0f3a940eef7cc2861c16a1fdd5 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 20 Nov 2025 13:11:23 -0500 Subject: [PATCH 2/8] Add `checks: write` to publish report --- .github/workflows/common.yml | 1 + .github/workflows/firebaseai.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index fa40aa67eab..1ad8ae39203 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -2,6 +2,7 @@ name: common permissions: contents: read + checks: write on: workflow_call: diff --git a/.github/workflows/firebaseai.yml b/.github/workflows/firebaseai.yml index 3174e725695..58287b60d4c 100644 --- a/.github/workflows/firebaseai.yml +++ b/.github/workflows/firebaseai.yml @@ -25,6 +25,7 @@ concurrency: permissions: contents: read # Needed for actions/checkout actions: write # Needed for actions/cache (save and restore) + checks: write jobs: spm: From b1f4ce50899b78f966bb07e86586fce037d17c8a Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 20 Nov 2025 17:13:15 -0500 Subject: [PATCH 3/8] Add `--preserve-unbeautified` --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index aea073a6258..ee8eea44c96 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -113,7 +113,7 @@ function RunXcodebuild() { local buildaction="${xcodebuild_args[$# - 1]}" # buildaction is the last arg local log_filename="xcodebuild-${buildaction}.log" - local xcbeautify_cmd=(xcbeautify --renderer github-actions --disable-logging --quiet --is-ci --report junit) + local xcbeautify_cmd=(xcbeautify --renderer github-actions --disable-logging --is-ci --preserve-unbeautified --report junit) local result=0 NSUnbufferedIO=YES xcodebuild "$@" 2>&1 | tee "$log_filename" | \ From 7bc4699f04bf1dc7b5a21506b95e4be678cbc487 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 20 Nov 2025 17:14:34 -0500 Subject: [PATCH 4/8] Temporarily remove `needs: spm` in `testapp-integration` job --- .github/workflows/firebaseai.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/firebaseai.yml b/.github/workflows/firebaseai.yml index 58287b60d4c..c3e19586d7b 100644 --- a/.github/workflows/firebaseai.yml +++ b/.github/workflows/firebaseai.yml @@ -49,7 +49,7 @@ jobs: target: iOS xcode: Xcode_26.0 runs-on: ${{ matrix.os }} - needs: spm + # needs: spm env: TEST_RUNNER_FIRAAppCheckDebugToken: ${{ secrets.VERTEXAI_INTEGRATION_FAC_DEBUG_TOKEN }} TEST_RUNNER_VTXIntegrationImagen: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} From ba6c7d5bb60d0e865c94ab07e332fe8f59eb219d Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 20 Nov 2025 17:17:03 -0500 Subject: [PATCH 5/8] Temporarily remove `actions/cache/restore` in `testapp-integration` job --- .github/workflows/firebaseai.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/firebaseai.yml b/.github/workflows/firebaseai.yml index c3e19586d7b..00850fd9180 100644 --- a/.github/workflows/firebaseai.yml +++ b/.github/workflows/firebaseai.yml @@ -57,10 +57,10 @@ jobs: secrets_passphrase: ${{ secrets.GHASecretsGPGPassphrase1 }} steps: - uses: actions/checkout@v4 - - uses: actions/cache/restore@v4 - with: - path: .build - key: ${{ needs.spm.outputs.cache_key }} + # - uses: actions/cache/restore@v4 + # with: + # path: .build + # key: ${{ needs.spm.outputs.cache_key }} - name: Run integration tests run: scripts/repo.sh tests run --secrets ./scripts/secrets/AI.json --platforms ${{ matrix.target }} --xcode ${{ matrix.xcode }} AI - name: Upload xcodebuild logs From 31de7c4f792b70a8f3739fb5a0ad6b57a7d084d5 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 20 Nov 2025 18:07:14 -0500 Subject: [PATCH 6/8] Remove deprecated models from tests --- .../Tests/TestApp/Sources/Constants.swift | 3 +- .../GenerateContentIntegrationTests.swift | 66 +++++-------------- 2 files changed, 16 insertions(+), 53 deletions(-) diff --git a/FirebaseAI/Tests/TestApp/Sources/Constants.swift b/FirebaseAI/Tests/TestApp/Sources/Constants.swift index 9bfc47a0e5a..4f88a39448b 100644 --- a/FirebaseAI/Tests/TestApp/Sources/Constants.swift +++ b/FirebaseAI/Tests/TestApp/Sources/Constants.swift @@ -23,10 +23,9 @@ public enum FirebaseAppNames { public enum ModelNames { public static let gemini2Flash = "gemini-2.0-flash-001" public static let gemini2FlashLite = "gemini-2.0-flash-lite-001" - public static let gemini2FlashPreviewImageGeneration = "gemini-2.0-flash-preview-image-generation" public static let gemini2FlashLive = "gemini-2.0-flash-live-001" public static let gemini2FlashLivePreview = "gemini-2.0-flash-live-preview-04-09" - public static let gemini2_5_FlashImagePreview = "gemini-2.5-flash-image-preview" + public static let gemini2_5_FlashImage = "gemini-2.5-flash-image" public static let gemini2_5_Flash = "gemini-2.5-flash" public static let gemini2_5_FlashLite = "gemini-2.5-flash-lite" public static let gemini2_5_FlashLivePreview = "gemini-live-2.5-flash-preview" diff --git a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift index d83a6b4a18d..046d49ff4c0 100644 --- a/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift +++ b/FirebaseAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift @@ -322,18 +322,11 @@ struct GenerateContentIntegrationTests { } @Test(arguments: [ - (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2FlashPreviewImageGeneration), - (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2FlashPreviewImageGeneration), - (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2_5_FlashImagePreview), - (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashPreviewImageGeneration), - (InstanceConfig.googleAI_v1beta, ModelNames.gemini2_5_FlashImagePreview), + (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2_5_FlashImage), + (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2_5_FlashImage), + (InstanceConfig.googleAI_v1beta, ModelNames.gemini2_5_FlashImage), // Note: The following configs are commented out for easy one-off manual testing. - // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashPreviewImageGeneration) - // (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashPreviewImageGeneration), - // ( - // InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, - // ModelNames.gemini2FlashPreviewImageGeneration - // ), + // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2_5_FlashImage) ]) func generateImage(_ config: InstanceConfig, modelName: String) async throws { let generationConfig = GenerationConfig( @@ -354,17 +347,8 @@ struct GenerateContentIntegrationTests { ) let prompt = "Generate an image of a cute cartoon kitten playing with a ball of yarn." - var response: GenerateContentResponse? - try await withKnownIssue( - "Backend may fail with a 503 - Service Unavailable error when overloaded", - isIntermittent: true - ) { - response = try await model.generateContent(prompt) - } matching: { issue in - (issue.error as? BackendError).map { $0.httpResponseCode == 503 } ?? false - } + let response = try await model.generateContent(prompt) - guard let response else { return } let candidate = try #require(response.candidates.first) let inlineDataPart = try #require(candidate.content.parts .first { $0 is InlineDataPart } as? InlineDataPart) @@ -372,16 +356,12 @@ struct GenerateContentIntegrationTests { #expect(inlineDataPartsViaAccessor.count == 1) let inlineDataPartViaAccessor = try #require(inlineDataPartsViaAccessor.first) #expect(inlineDataPart == inlineDataPartViaAccessor) - #expect(inlineDataPart.mimeType == "image/png") + #expect(inlineDataPart.mimeType.starts(with: "image/")) #expect(inlineDataPart.data.count > 0) #if canImport(UIKit) let uiImage = try #require(UIImage(data: inlineDataPart.data)) - // Gemini 2.0 Flash Experimental returns images sized to fit within a 1024x1024 pixel box but - // dimensions may vary depending on the aspect ratio. - #expect(uiImage.size.width <= 1024) - #expect(uiImage.size.width >= 500) - #expect(uiImage.size.height <= 1024) - #expect(uiImage.size.height >= 500) + #expect(uiImage.size.width > 0) + #expect(uiImage.size.height > 0) #endif // canImport(UIKit) } @@ -552,18 +532,11 @@ struct GenerateContentIntegrationTests { } @Test(arguments: [ - (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2FlashPreviewImageGeneration), - (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2FlashPreviewImageGeneration), - (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2_5_FlashImagePreview), - (InstanceConfig.googleAI_v1beta, ModelNames.gemini2FlashPreviewImageGeneration), - (InstanceConfig.googleAI_v1beta, ModelNames.gemini2_5_FlashImagePreview), + (InstanceConfig.vertexAI_v1beta, ModelNames.gemini2_5_FlashImage), + (InstanceConfig.vertexAI_v1beta_global, ModelNames.gemini2_5_FlashImage), + (InstanceConfig.googleAI_v1beta, ModelNames.gemini2_5_FlashImage), // Note: The following configs are commented out for easy one-off manual testing. - // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2FlashPreviewImageGeneration) - // (InstanceConfig.googleAI_v1beta_freeTier, ModelNames.gemini2FlashPreviewImageGeneration), - // ( - // InstanceConfig.googleAI_v1beta_freeTier_bypassProxy, - // ModelNames.gemini2FlashPreviewImageGeneration - // ), + // (InstanceConfig.googleAI_v1beta_staging, ModelNames.gemini2_5_FlashImage) ]) func generateImageStreaming(_ config: InstanceConfig, modelName: String) async throws { let generationConfig = GenerationConfig( @@ -572,11 +545,6 @@ struct GenerateContentIntegrationTests { topK: 1, responseModalities: [.text, .image] ) - let safetySettings = safetySettings.filter { - // HARM_CATEGORY_CIVIC_INTEGRITY is deprecated in Vertex AI but only rejected when using the - // 'gemini-2.0-flash-preview-image-generation' model. - $0.harmCategory != .civicIntegrity - } let model = FirebaseAI.componentInstance(config).generativeModel( modelName: modelName, generationConfig: generationConfig, @@ -605,16 +573,12 @@ struct GenerateContentIntegrationTests { #expect(inlineDataParts.count == 1) let inlineDataPart = try #require(inlineDataParts.first) - #expect(inlineDataPart.mimeType == "image/png") + #expect(inlineDataPart.mimeType.starts(with: "image/")) #expect(inlineDataPart.data.count > 0) #if canImport(UIKit) let uiImage = try #require(UIImage(data: inlineDataPart.data)) - // Gemini 2.0 Flash Experimental returns images sized to fit within a 1024x1024 pixel box but - // dimensions may vary depending on the aspect ratio. - #expect(uiImage.size.width <= 1024) - #expect(uiImage.size.width >= 500) - #expect(uiImage.size.height <= 1024) - #expect(uiImage.size.height >= 500) + #expect(uiImage.size.width > 0) + #expect(uiImage.size.height > 0) #endif // canImport(UIKit) } From 93974ad9d30ff3fe9a19fa67f6ee09366fbc24d8 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 20 Nov 2025 18:08:05 -0500 Subject: [PATCH 7/8] Revert "Add `checks: write` to publish report" This reverts commit 6271061db3df8a0f3a940eef7cc2861c16a1fdd5. --- .github/workflows/common.yml | 1 - .github/workflows/firebaseai.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index 1ad8ae39203..fa40aa67eab 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -2,7 +2,6 @@ name: common permissions: contents: read - checks: write on: workflow_call: diff --git a/.github/workflows/firebaseai.yml b/.github/workflows/firebaseai.yml index 00850fd9180..1d11cc97df0 100644 --- a/.github/workflows/firebaseai.yml +++ b/.github/workflows/firebaseai.yml @@ -25,7 +25,6 @@ concurrency: permissions: contents: read # Needed for actions/checkout actions: write # Needed for actions/cache (save and restore) - checks: write jobs: spm: From 5961f949ed9d796c28c4fccac33fce6a1b0454f2 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 20 Nov 2025 18:09:48 -0500 Subject: [PATCH 8/8] Revert "[CI] Generate report with `xcbeautify`" This reverts commit 8b9c47b81711e7677b2e1d07a2394a7d3b90aa0b. # Conflicts: # scripts/build.sh --- .github/workflows/common.yml | 5 ----- .github/workflows/firebaseai.yml | 5 ----- scripts/build.sh | 2 +- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index fa40aa67eab..b2456256c9c 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -123,8 +123,3 @@ jobs: ${{ inputs.target }} \ ${{ matrix.platform }} \ ${{ (contains(inputs.buildonly_platforms, matrix.platform) || contains(inputs.buildonly_platforms, 'all')) && 'spmbuildonly' || 'spm' }} - - name: Publish Test Report - uses: mikepenz/action-junit-report@v6 - if: success() || failure() - with: - report_paths: 'build/reports/junit.xml' diff --git a/.github/workflows/firebaseai.yml b/.github/workflows/firebaseai.yml index 1d11cc97df0..f04ca06e169 100644 --- a/.github/workflows/firebaseai.yml +++ b/.github/workflows/firebaseai.yml @@ -69,11 +69,6 @@ jobs: name: xcodebuild-${{ matrix.target }}-${{ matrix.os }}-${{ matrix.xcode }}.log path: xcodebuild-*.log retention-days: 2 - - name: Publish Test Report - uses: mikepenz/action-junit-report@v6 - if: success() || failure() - with: - report_paths: 'build/reports/junit.xml' pod_lib_lint: strategy: diff --git a/scripts/build.sh b/scripts/build.sh index ee8eea44c96..68f61b94019 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -113,7 +113,7 @@ function RunXcodebuild() { local buildaction="${xcodebuild_args[$# - 1]}" # buildaction is the last arg local log_filename="xcodebuild-${buildaction}.log" - local xcbeautify_cmd=(xcbeautify --renderer github-actions --disable-logging --is-ci --preserve-unbeautified --report junit) + local xcbeautify_cmd=(xcbeautify --renderer github-actions --disable-logging --is-ci --preserve-unbeautified) local result=0 NSUnbufferedIO=YES xcodebuild "$@" 2>&1 | tee "$log_filename" | \