Skip to content

Commit 901844a

Browse files
andrewheardandrewheard
authored
[Vertex AI] Add simple integration tests (#13075)
1 parent 17165d1 commit 901844a

File tree

6 files changed

+194
-3
lines changed

6 files changed

+194
-3
lines changed

‎.github/workflows/vertexai.yml

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@ concurrency:
1515
cancel-in-progress: true
1616

1717
jobs:
18-
spm:
18+
spm-unit:
1919
strategy:
2020
matrix:
2121
target: [iOS, macOS, catalyst]
22-
os: [macos-13]
22+
os: [macos-14]
2323
include:
24-
- os: macos-13
24+
- os: macos-14
2525
xcode: Xcode_15.2
2626
runs-on: ${{ matrix.os }}
2727
env:
@@ -40,6 +40,35 @@ jobs:
4040
retry_wait_seconds: 120
4141
command: scripts/build.sh FirebaseVertexAIUnit ${{ matrix.target }} spm
4242

43+
spm-integration:
44+
strategy:
45+
matrix:
46+
target: [macOS]
47+
os: [macos-14]
48+
include:
49+
- os: macos-14
50+
xcode: Xcode_15.2
51+
runs-on: ${{ matrix.os }}
52+
env:
53+
FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1
54+
plist_secret: ${{ secrets.GHASecretsGPGPassphrase1 }}
55+
steps:
56+
- uses: actions/checkout@v4
57+
- name: Install Secret GoogleService-Info.plist
58+
run: scripts/decrypt_gha_secret.sh scripts/gha-encrypted/vertexai-integration.plist.gpg \
59+
FirebaseVertexAI/Tests/Integration/Resources/GoogleService-Info.plist "$plist_secret"
60+
- name: Xcode
61+
run: sudo xcode-select -s /Applications/${{ matrix.xcode }}.app/Contents/Developer
62+
- name: Initialize xcodebuild
63+
run: scripts/setup_spm_tests.sh
64+
- uses: nick-fields/retry@v3
65+
with:
66+
timeout_minutes: 120
67+
max_attempts: 3
68+
retry_on: error
69+
retry_wait_seconds: 120
70+
command: scripts/build.sh FirebaseVertexAIIntegration ${{ matrix.target }} spm
71+
4372
sample:
4473
strategy:
4574
matrix:

‎FirebaseVertexAI/Tests/Integration/IntegrationTests.swift

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import FirebaseCore
16+
import FirebaseVertexAI
17+
import XCTest
18+
19+
// These tests are functional on other platforms but are compiled-out of non-macOS platforms to
20+
// avoid re-running them as part of `swift-build-run` job (iOS-only) in the `spm` workflow on CI:
21+
// https://github.com/firebase/firebase-ios-sdk/blob/0492e83cb22833ec548e61d854bb7b830e83b826/.github/workflows/spm.yml#L57
22+
// Since these requests are billed, we are running them more sparsely than the unit tests.
23+
#if os(macOS)
24+
25+
@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, *)
26+
final class IntegrationTests: XCTestCase {
27+
// Set temperature, topP and topK to lowest allowed values to make responses more deterministic.
28+
let generationConfig = GenerationConfig(temperature: 0.0, topP: 0.0, topK: 1)
29+
30+
var vertex: VertexAI!
31+
var model: GenerativeModel!
32+
33+
override func setUp() async throws {
34+
let plistPath = try XCTUnwrap(Bundle.module.path(
35+
forResource: "GoogleService-Info",
36+
ofType: "plist"
37+
))
38+
let options = try XCTUnwrap(FirebaseOptions(contentsOfFile: plistPath))
39+
FirebaseApp.configure(options: options)
40+
41+
vertex = VertexAI.vertexAI()
42+
model = vertex.generativeModel(
43+
modelName: "gemini-1.5-flash",
44+
generationConfig: generationConfig
45+
)
46+
}
47+
48+
override func tearDown() async throws {
49+
if let app = FirebaseApp.app() {
50+
await app.delete()
51+
}
52+
}
53+
54+
// MARK: - Generate Content
55+
56+
func testGenerateContent() async throws {
57+
let prompt = "Where is Google headquarters located? Answer with the city name only."
58+
59+
let response = try await model.generateContent(prompt)
60+
61+
let text = try XCTUnwrap(response.text).trimmingCharacters(in: .whitespacesAndNewlines)
62+
XCTAssertEqual(text, "Mountain View")
63+
}
64+
65+
// MARK: - Count Tokens
66+
67+
func testCountTokens() async throws {
68+
let prompt = "Why is the sky blue?"
69+
70+
let response = try await model.countTokens(prompt)
71+
72+
XCTAssertEqual(response.totalTokens, 6)
73+
XCTAssertEqual(response.totalBillableCharacters, 16)
74+
}
75+
}
76+
77+
#endif // os(macOS)

‎FirebaseVertexAI/Tests/Integration/Resources/placeholder.txt

Whitespace-only changes.

‎Package.swift

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1386,6 +1386,14 @@ let package = Package(
13861386
.headerSearchPath("../../../"),
13871387
]
13881388
),
1389+
.testTarget(
1390+
name: "FirebaseVertexAIIntegration",
1391+
dependencies: ["FirebaseVertexAI", "SharedTestUtilities"],
1392+
path: "FirebaseVertexAI/Tests/Integration",
1393+
resources: [
1394+
.process("Resources"),
1395+
]
1396+
),
13891397
] + firestoreTargets(),
13901398
cLanguageStandard: .c99,
13911399
cxxLanguageStandard: CXXLanguageStandard.gnucxx14

‎scripts/gha-encrypted/vertexai-integration.plist.gpg

619 Bytes
Binary file not shown.

‎scripts/spm_test_schemes/FirebaseVertexAIIntegration.xcscheme

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<Scheme
3+
LastUpgradeVersion = "1200"
4+
version = "1.3">
5+
<BuildAction
6+
parallelizeBuildables = "YES"
7+
buildImplicitDependencies = "YES">
8+
<BuildActionEntries>
9+
<BuildActionEntry
10+
buildForTesting = "YES"
11+
buildForRunning = "YES"
12+
buildForProfiling = "NO"
13+
buildForArchiving = "NO"
14+
buildForAnalyzing = "YES">
15+
<BuildableReference
16+
BuildableIdentifier = "primary"
17+
BlueprintIdentifier = "FirebaseVertexAIIntegration"
18+
BuildableName = "FirebaseVertexAIIntegration"
19+
BlueprintName = "FirebaseVertexAIIntegration"
20+
ReferencedContainer = "container:">
21+
</BuildableReference>
22+
</BuildActionEntry>
23+
</BuildActionEntries>
24+
</BuildAction>
25+
<TestAction
26+
buildConfiguration = "Debug"
27+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
28+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
29+
shouldUseLaunchSchemeArgsEnv = "YES">
30+
<Testables>
31+
<TestableReference
32+
skipped = "NO">
33+
<BuildableReference
34+
BuildableIdentifier = "primary"
35+
BlueprintIdentifier = "FirebaseVertexAIIntegration"
36+
BuildableName = "FirebaseVertexAIIntegration"
37+
BlueprintName = "FirebaseVertexAIIntegration"
38+
ReferencedContainer = "container:">
39+
</BuildableReference>
40+
</TestableReference>
41+
</Testables>
42+
</TestAction>
43+
<LaunchAction
44+
buildConfiguration = "Debug"
45+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
46+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
47+
launchStyle = "0"
48+
useCustomWorkingDirectory = "NO"
49+
ignoresPersistentStateOnLaunch = "NO"
50+
debugDocumentVersioning = "YES"
51+
debugServiceExtension = "internal"
52+
allowLocationSimulation = "YES">
53+
</LaunchAction>
54+
<ProfileAction
55+
buildConfiguration = "Release"
56+
shouldUseLaunchSchemeArgsEnv = "YES"
57+
savedToolIdentifier = ""
58+
useCustomWorkingDirectory = "NO"
59+
debugDocumentVersioning = "YES">
60+
<MacroExpansion>
61+
<BuildableReference
62+
BuildableIdentifier = "primary"
63+
BlueprintIdentifier = "FirebaseVertexAIIntegration"
64+
BuildableName = "FirebaseVertexAIIntegration"
65+
BlueprintName = "FirebaseVertexAIIntegration"
66+
ReferencedContainer = "container:">
67+
</BuildableReference>
68+
</MacroExpansion>
69+
</ProfileAction>
70+
<AnalyzeAction
71+
buildConfiguration = "Debug">
72+
</AnalyzeAction>
73+
<ArchiveAction
74+
buildConfiguration = "Release"
75+
revealArchiveInOrganizer = "YES">
76+
</ArchiveAction>
77+
</Scheme>

0 commit comments

Comments
 (0)