Skip to content
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
44ea204
[Firebase AI] Add `CodeExecution` tool support
andrewheard Sep 2, 2025
4e3f587
Implement `ExecutableCode` and `CodeExecutionResult` part coding
andrewheard Sep 3, 2025
a38e44c
Add checks for response content in integration test
andrewheard Sep 3, 2025
d617c93
Add handling for unexpected null values
andrewheard Sep 3, 2025
a7f68e4
Merge branch 'main' into ah/ai-code-execution
andrewheard Sep 3, 2025
69aef8e
Add documentation
andrewheard Sep 3, 2025
1301338
Add `CustomStringConvertible` conformance
andrewheard Sep 3, 2025
31d8dbd
Add `CodeExecution` tool encoding test
andrewheard Sep 3, 2025
b4b5676
Add `InternalPartTests` for `ExecutedCode` and `CodeExecutionResult`
andrewheard Sep 3, 2025
3199d94
Add decoding and encoding tests in `PartTests`
andrewheard Sep 3, 2025
79ba1ff
Add CHANGELOG entry
andrewheard Sep 3, 2025
14f07ad
Fix integration test
andrewheard Sep 3, 2025
d56d040
Rename `_isThought` parameter in `CodeExecutionResultPart` initializer
andrewheard Sep 3, 2025
d94c062
Add mock response tests in `GenerativeModelVertexAITests`
andrewheard Sep 3, 2025
dd26bf8
Merge branch 'main' into ah/ai-code-execution
andrewheard Sep 3, 2025
b52a408
Formatting
andrewheard Sep 3, 2025
735684f
Add `AILog.safeUnwrap` static method to log on use of fallback values
andrewheard Sep 5, 2025
6810309
Merge branch 'main' into ah/ai-code-execution
andrewheard Sep 5, 2025
7cac4b5
Add mock response tests in `GenerativeModelGoogleAITests`
andrewheard Sep 5, 2025
b78c4d2
Fix `output` nullability in integration test
andrewheard Sep 5, 2025
1162598
Add decoding tests for missing fields in `ExecutableCode` and `CodeEx…
andrewheard Sep 6, 2025
35866d5
Add decoding tests with and without `"thought": true`
andrewheard Sep 6, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions FirebaseAI/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
# 12.3.0
- [feature] Added support for the Code Execution tool, which enables the model
to generate and run code to perform complex tasks like solving mathematical
equations or visualizing data. (#15280)
- [fixed] Fixed a decoding error when generating images with the
`gemini-2.5-flash-image-preview` model using `generateContentStream` or
`sendMessageStream` with the Gemini Developer API. (#15262)
Expand Down
31 changes: 31 additions & 0 deletions FirebaseAI/Sources/AILog.swift
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ enum AILog {
case generateContentResponseUnrecognizedContentModality = 3012
case decodedUnsupportedImagenPredictionType = 3013
case decodedUnsupportedPartData = 3014
case codeExecutionResultUnrecognizedOutcome = 3015
case executableCodeUnrecognizedLanguage = 3016
case fallbackValueUsed = 3017

// SDK State Errors
case generateContentResponseNoCandidates = 4000
Expand Down Expand Up @@ -124,4 +127,32 @@ enum AILog {
static func additionalLoggingEnabled() -> Bool {
return ProcessInfo.processInfo.arguments.contains(enableArgumentKey)
}

/// Returns the unwrapped optional value if non-nil or returns the fallback value and logs.
///
/// This convenience method is intended for use in place of `optionalValue ?? fallbackValue` with
/// the addition of logging on use of the fallback value.
///
/// - Parameters:
/// - optionalValue: The value to unwrap.
/// - fallbackValue: The fallback (default) value to return when `optionalValue` is `nil`.
/// - level: The logging level to use for fallback messages; defaults to
/// `FirebaseLoggerLevel.warning`.
/// - code: The message code to use for fallback messages; defaults to
/// `MessageCode.fallbackValueUsed`.
/// - caller: The name of the unwrapped value; defaults to the name of the computed property or
/// function name from which the unwrapping occurred.
static func safeUnwrap<T>(_ optionalValue: T?,
fallback fallbackValue: T,
level: FirebaseLoggerLevel = .warning,
code: MessageCode = .fallbackValueUsed,
caller: String = #function) -> T {
guard let unwrappedValue = optionalValue else {
AILog.log(level: level, code: code, """
No value specified for '\(caller)' (\(T.self)); using fallback value '\(fallbackValue)'.
""")
return fallbackValue
}
return unwrappedValue
}
}
24 changes: 24 additions & 0 deletions FirebaseAI/Sources/ModelContent.swift
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ struct InternalPart: Equatable, Sendable {
case fileData(FileData)
case functionCall(FunctionCall)
case functionResponse(FunctionResponse)
case executableCode(ExecutableCode)
case codeExecutionResult(CodeExecutionResult)

struct UnsupportedDataError: Error {
let decodingError: DecodingError
Expand Down Expand Up @@ -93,6 +95,16 @@ public struct ModelContent: Equatable, Sendable {
return FunctionResponsePart(
functionResponse, isThought: part.isThought, thoughtSignature: part.thoughtSignature
)
case let .executableCode(executableCode):
return ExecutableCodePart(
executableCode, isThought: part.isThought, thoughtSignature: part.thoughtSignature
)
case let .codeExecutionResult(codeExecutionResult):
return CodeExecutionResultPart(
codeExecutionResult: codeExecutionResult,
isThought: part.isThought,
thoughtSignature: part.thoughtSignature
)
case .none:
// Filter out parts that contain missing or unrecognized data
return nil
Expand Down Expand Up @@ -212,6 +224,8 @@ extension InternalPart.OneOfData: Codable {
case fileData
case functionCall
case functionResponse
case executableCode
case codeExecutionResult
}

public func encode(to encoder: Encoder) throws {
Expand All @@ -227,6 +241,10 @@ extension InternalPart.OneOfData: Codable {
try container.encode(functionCall, forKey: .functionCall)
case let .functionResponse(functionResponse):
try container.encode(functionResponse, forKey: .functionResponse)
case let .executableCode(executableCode):
try container.encode(executableCode, forKey: .executableCode)
case let .codeExecutionResult(codeExecutionResult):
try container.encode(codeExecutionResult, forKey: .codeExecutionResult)
}
}

Expand All @@ -242,6 +260,12 @@ extension InternalPart.OneOfData: Codable {
self = try .functionCall(values.decode(FunctionCall.self, forKey: .functionCall))
} else if values.contains(.functionResponse) {
self = try .functionResponse(values.decode(FunctionResponse.self, forKey: .functionResponse))
} else if values.contains(.executableCode) {
self = try .executableCode(values.decode(ExecutableCode.self, forKey: .executableCode))
} else if values.contains(.codeExecutionResult) {
self = try .codeExecutionResult(
values.decode(CodeExecutionResult.self, forKey: .codeExecutionResult)
)
} else {
let unexpectedKeys = values.allKeys.map { $0.stringValue }
throw UnsupportedDataError(decodingError: DecodingError.dataCorrupted(
Expand Down
20 changes: 14 additions & 6 deletions FirebaseAI/Sources/Tool.swift
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,18 @@ public struct GoogleSearch: Sendable {
public struct Tool: Sendable {
/// A list of `FunctionDeclarations` available to the model.
let functionDeclarations: [FunctionDeclaration]?

/// Specifies the Google Search configuration.
let googleSearch: GoogleSearch?

init(functionDeclarations: [FunctionDeclaration]?) {
self.functionDeclarations = functionDeclarations
googleSearch = nil
}
let codeExecution: CodeExecution?

init(googleSearch: GoogleSearch) {
init(functionDeclarations: [FunctionDeclaration]? = nil,
googleSearch: GoogleSearch? = nil,
codeExecution: CodeExecution? = nil) {
self.functionDeclarations = functionDeclarations
self.googleSearch = googleSearch
functionDeclarations = nil
self.codeExecution = codeExecution
}

/// Creates a tool that allows the model to perform function calling.
Expand Down Expand Up @@ -126,6 +127,13 @@ public struct Tool: Sendable {
public static func googleSearch(_ googleSearch: GoogleSearch = GoogleSearch()) -> Tool {
return self.init(googleSearch: googleSearch)
}

/// Creates a tool that allows the model to execute code.
///
/// For more details, see ``CodeExecution``.
public static func codeExecution() -> Tool {
return self.init(codeExecution: CodeExecution())
}
}

/// Configuration for specifying function calling behavior.
Expand Down
48 changes: 48 additions & 0 deletions FirebaseAI/Sources/Types/Internal/InternalPart.swift
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,54 @@ struct FunctionResponse: Codable, Equatable, Sendable {
}
}

@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
struct ExecutableCode: Codable, Equatable, Sendable {
struct Language: CodableProtoEnum, Sendable, Equatable {
enum Kind: String {
case unspecified = "LANGUAGE_UNSPECIFIED"
case python = "PYTHON"
}

let rawValue: String

static let unrecognizedValueMessageCode =
AILog.MessageCode.executableCodeUnrecognizedLanguage
}

let language: Language?
let code: String?

init(language: Language, code: String) {
self.language = language
self.code = code
}
}

@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
struct CodeExecutionResult: Codable, Equatable, Sendable {
struct Outcome: CodableProtoEnum, Sendable, Equatable {
enum Kind: String {
case unspecified = "OUTCOME_UNSPECIFIED"
case ok = "OUTCOME_OK"
case failed = "OUTCOME_FAILED"
case deadlineExceeded = "OUTCOME_DEADLINE_EXCEEDED"
}

let rawValue: String

static let unrecognizedValueMessageCode =
AILog.MessageCode.codeExecutionResultUnrecognizedOutcome
}

let outcome: Outcome?
let output: String?

init(outcome: Outcome, output: String) {
self.outcome = outcome
self.output = output
}
}

@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
struct ErrorPart: Part, Error {
let error: Error
Expand Down
115 changes: 115 additions & 0 deletions FirebaseAI/Sources/Types/Public/Part.swift
Original file line number Diff line number Diff line change
Expand Up @@ -202,3 +202,118 @@ public struct FunctionResponsePart: Part {
self.thoughtSignature = thoughtSignature
}
}

/// A part containing code that was executed by the model.
@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
public struct ExecutableCodePart: Part {
/// The language of the code in an ``ExecutableCodePart``.
public struct Language: Sendable, Equatable, CustomStringConvertible {
let internalLanguage: ExecutableCode.Language

/// The Python programming language.
public static let python = ExecutableCodePart.Language(ExecutableCode.Language(kind: .python))

public var description: String { internalLanguage.rawValue }

init(_ language: ExecutableCode.Language) {
internalLanguage = language
}
}

let executableCode: ExecutableCode
let _isThought: Bool?
let thoughtSignature: String?

/// The language of the code.
public var language: ExecutableCodePart.Language {
ExecutableCodePart.Language(
// Fallback to "LANGUAGE_UNSPECIFIED" if the value is ever omitted by the backend; this should
// never happen.
AILog.safeUnwrap(
executableCode.language, fallback: ExecutableCode.Language(kind: .unspecified)
)
)
}

/// The code that was executed.
public var code: String {
// Fallback to empty string if `code` is ever omitted by the backend; this should never happen.
AILog.safeUnwrap(executableCode.code, fallback: "")
}

public var isThought: Bool { _isThought ?? false }

public init(language: ExecutableCodePart.Language, code: String) {
self.init(
ExecutableCode(language: language.internalLanguage, code: code),
isThought: nil,
thoughtSignature: nil
)
}

init(_ executableCode: ExecutableCode, isThought: Bool?, thoughtSignature: String?) {
self.executableCode = executableCode
_isThought = isThought
self.thoughtSignature = thoughtSignature
}
}

/// The result of executing code.
@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, tvOS 15.0, watchOS 8.0, *)
public struct CodeExecutionResultPart: Part {
/// The outcome of a code execution.
public struct Outcome: Sendable, Equatable, CustomStringConvertible {
let internalOutcome: CodeExecutionResult.Outcome

/// The code executed without errors.
public static let ok = CodeExecutionResultPart.Outcome(CodeExecutionResult.Outcome(kind: .ok))

/// The code failed to execute.
public static let failed =
CodeExecutionResultPart.Outcome(CodeExecutionResult.Outcome(kind: .failed))

/// The code took too long to execute.
public static let deadlineExceeded =
CodeExecutionResultPart.Outcome(CodeExecutionResult.Outcome(kind: .deadlineExceeded))

public var description: String { internalOutcome.rawValue }

init(_ outcome: CodeExecutionResult.Outcome) {
internalOutcome = outcome
}
}

let codeExecutionResult: CodeExecutionResult
let _isThought: Bool?
let thoughtSignature: String?

/// The outcome of the code execution.
public var outcome: CodeExecutionResultPart.Outcome {
CodeExecutionResultPart.Outcome(
// Fallback to "OUTCOME_UNSPECIFIED" if this value is ever omitted by the backend; this should
// never happen.
AILog.safeUnwrap(
codeExecutionResult.outcome, fallback: CodeExecutionResult.Outcome(kind: .unspecified)
)
)
}

/// The output of the code execution.
public var output: String? { codeExecutionResult.output }

public var isThought: Bool { _isThought ?? false }

public init(outcome: CodeExecutionResultPart.Outcome, output: String) {
self.init(
codeExecutionResult: CodeExecutionResult(outcome: outcome.internalOutcome, output: output),
isThought: nil,
thoughtSignature: nil
)
}

init(codeExecutionResult: CodeExecutionResult, isThought: Bool?, thoughtSignature: String?) {
self.codeExecutionResult = codeExecutionResult
_isThought = isThought
self.thoughtSignature = thoughtSignature
}
}
21 changes: 21 additions & 0 deletions FirebaseAI/Sources/Types/Public/Tools/CodeExecution.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/// A tool that allows the model to execute code.
///
/// This tool can be used to solve complex problems, for example, by generating and executing Python
/// code to solve a math problem.
public struct CodeExecution: Sendable, Encodable {
init() {}
}
1 change: 1 addition & 0 deletions FirebaseAI/Tests/TestApp/Sources/Constants.swift
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public enum ModelNames {
public static let gemini2FlashPreviewImageGeneration = "gemini-2.0-flash-preview-image-generation"
public static let gemini2_5_FlashImagePreview = "gemini-2.5-flash-image-preview"
public static let gemini2_5_Flash = "gemini-2.5-flash"
public static let gemini2_5_FlashLite = "gemini-2.5-flash-lite"
public static let gemini2_5_Pro = "gemini-2.5-pro"
public static let gemma3_4B = "gemma-3-4b-it"
}
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,35 @@ struct GenerateContentIntegrationTests {
}
}

@Test(arguments: InstanceConfig.allConfigs)
func generateContent_codeExecution_succeeds(_ config: InstanceConfig) async throws {
let model = FirebaseAI.componentInstance(config).generativeModel(
modelName: ModelNames.gemini2_5_FlashLite,
generationConfig: generationConfig,
tools: [.codeExecution()]
)
let prompt = """
What is the sum of the first 5 prime numbers? Generate and run code for the calculation.
"""

let response = try await model.generateContent(prompt)

let candidate = try #require(response.candidates.first)
let executableCodeParts = candidate.content.parts.compactMap { $0 as? ExecutableCodePart }
#expect(executableCodeParts.count == 1)
let executableCodePart = try #require(executableCodeParts.first)
#expect(executableCodePart.language == .python)
#expect(executableCodePart.code.contains("sum"))
let codeExecutionResults = candidate.content.parts.compactMap { $0 as? CodeExecutionResultPart }
#expect(codeExecutionResults.count == 1)
let codeExecutionResultPart = try #require(codeExecutionResults.first)
#expect(codeExecutionResultPart.outcome == .ok)
let output = try #require(codeExecutionResultPart.output)
#expect(output.contains("28")) // 2 + 3 + 5 + 7 + 11 = 28
let text = try #require(response.text)
#expect(text.contains("28"))
}

// MARK: Streaming Tests

@Test(arguments: [
Expand Down
Loading
Loading