From f54eb4f22f832ec5b8748d1dcdbc1b984f912eeb Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 10 Oct 2024 17:49:29 -0400 Subject: [PATCH 1/3] [Vertex AI] Refactor function calling APIs --- FirebaseVertexAI/CHANGELOG.md | 12 ++- .../ViewModels/FunctionCallingViewModel.swift | 2 +- .../Sources/FunctionCalling.swift | 98 ++++++++++--------- .../Tests/Integration/IntegrationTests.swift | 8 +- 4 files changed, 67 insertions(+), 53 deletions(-) diff --git a/FirebaseVertexAI/CHANGELOG.md b/FirebaseVertexAI/CHANGELOG.md index ad5e1b74f95..539591e3e16 100644 --- a/FirebaseVertexAI/CHANGELOG.md +++ b/FirebaseVertexAI/CHANGELOG.md @@ -15,10 +15,10 @@ renamed to `inlineData`; no functionality changes. (#13700) - [changed] **Breaking Change**: The property `citationSources` of `CitationMetadata` has been renamed to `citations`. (#13702) -- [changed] **Breaking Change**: The constructor for `Schema` is now internal; - use the new static methods `Schema.string(...)`, `Schema.object(...)`, etc., +- [changed] **Breaking Change**: The initializer for `Schema` is now internal; + use the new type methods `Schema.string(...)`, `Schema.object(...)`, etc., instead. (#13852) -- [changed] **Breaking Change**: The constructor for `FunctionDeclaration` now +- [changed] **Breaking Change**: The initializer for `FunctionDeclaration` now accepts an array of *optional* parameters instead of a list of *required* parameters; if a parameter is not listed as optional it is assumed to be required. (#13616) @@ -44,6 +44,12 @@ `FinishReason` are now structs instead of enums types and the `unknown` cases have been removed; in a `switch` statement, use the `default:` case to cover unknown or unhandled values. (#13728, #13854, #13860) +- [changed] **Breaking Change**: The `Tool` initializer is now internal; use the + new type method `functionDeclarations(_:)` to create a `Tool` for function + calling. (#TODO) +- [changed] **Breaking Change**: The `FunctionCallingConfig` initializer and + `Mode` enum are now internal; use one of the new type methods `auto()`, + `any(allowedFunctionNames:)`, or `none()` to create a config. (#TODO) - [changed] The default request timeout is now 180 seconds instead of the platform-default value of 60 seconds for a `URLRequest`; this timeout may still be customized in `RequestOptions`. (#13722) diff --git a/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift b/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift index f39540eb1a9..7a67175112f 100644 --- a/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift +++ b/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift @@ -40,7 +40,7 @@ class FunctionCallingViewModel: ObservableObject { init() { model = VertexAI.vertexAI().generativeModel( modelName: "gemini-1.5-flash", - tools: [Tool(functionDeclarations: [ + tools: [.functionDeclarations([ FunctionDeclaration( name: "get_exchange_rate", description: "Get the exchange rate for currencies between countries", diff --git a/FirebaseVertexAI/Sources/FunctionCalling.swift b/FirebaseVertexAI/Sources/FunctionCalling.swift index 4cda35aa4da..8ca676751a3 100644 --- a/FirebaseVertexAI/Sources/FunctionCalling.swift +++ b/FirebaseVertexAI/Sources/FunctionCalling.swift @@ -25,7 +25,7 @@ public struct FunctionDeclaration { /// A brief description of the function. let description: String - /// Describes the parameters to this function; must be of type ``DataType/object``. + /// Describes the parameters to this function; must be of type `"OBJECT"`. let parameters: Schema? /// Constructs a new `FunctionDeclaration`. @@ -47,55 +47,49 @@ public struct FunctionDeclaration { } } -/// Helper tools that the model may use to generate response. +/// A helper tool that the model may use when generating responses. /// -/// A `Tool` is a piece of code that enables the system to interact with external systems to -/// perform an action, or set of actions, outside of knowledge and scope of the model. +/// A `Tool` is a piece of code that enables the system to interact with external systems to perform +/// an action, or set of actions, outside of knowledge and scope of the model. public struct Tool { /// A list of `FunctionDeclarations` available to the model. let functionDeclarations: [FunctionDeclaration]? - /// Constructs a new `Tool`. + init(functionDeclarations: [FunctionDeclaration]?) { + self.functionDeclarations = functionDeclarations + } + + /// Creates a tool that allows the model to perform function calling. + /// + /// Function calling can be used to provide data to the model that was not known at the time it + /// was trained (for example, the current date or weather conditions) or to allow it to interact + /// with external systems (for example, making an API request or querying/updating a database). + /// For more details and use cases, see [Introduction to function + /// calling](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-calling). /// /// - Parameters: /// - functionDeclarations: A list of `FunctionDeclarations` available to the model that can be /// used for function calling. /// The model or system does not execute the function. Instead the defined function may be - /// returned as a ``FunctionCall`` in ``ModelContent/Part/functionCall(_:)`` with arguments to - /// the client side for execution. The model may decide to call a subset of these functions by - /// populating ``FunctionCall`` in the response. The next conversation turn may contain a - /// ``FunctionResponse`` in ``ModelContent/Part/functionResponse(_:)`` with the - /// ``ModelContent/role`` "function", providing generation context for the next model turn. - public init(functionDeclarations: [FunctionDeclaration]?) { - self.functionDeclarations = functionDeclarations + /// returned as a ``FunctionCallPart`` with arguments to the client side for execution. The + /// model may decide to call none, some or all of the declared functions; this behavior may be + /// configured by specifying a ``ToolConfig`` when instantiating the model. When a + /// ``FunctionCallPart`` is received, the next conversation turn may contain a + /// ``FunctionResponsePart`` in ``ModelContent/parts`` with a ``ModelContent/role`` of + /// `"function"`; this response contains the result of executing the function on the client, + /// providing generation context for the model's next turn. + public static func functionDeclarations(_ functionDeclarations: [FunctionDeclaration]) -> Tool { + return self.init(functionDeclarations: functionDeclarations) } } /// Configuration for specifying function calling behavior. public struct FunctionCallingConfig { /// Defines the execution behavior for function calling by defining the execution mode. - public struct Mode: EncodableProtoEnum { - enum Kind: String { - case auto = "AUTO" - case any = "ANY" - case none = "NONE" - } - - /// The default behavior for function calling. - /// - /// The model calls functions to answer queries at its discretion. - public static let auto = Mode(kind: .auto) - - /// The model always predicts a provided function call to answer every query. - public static let any = Mode(kind: .any) - - /// The model will never predict a function call to answer a query. - /// - /// > Note: This can also be achieved by not passing any ``FunctionDeclaration`` tools - /// > when instantiating the model. - public static let none = Mode(kind: .none) - - let rawValue: String + enum Mode: String { + case auto = "AUTO" + case any = "ANY" + case none = "NONE" } /// Specifies the mode in which function calling should execute. @@ -104,20 +98,34 @@ public struct FunctionCallingConfig { /// A set of function names that, when provided, limits the functions the model will call. let allowedFunctionNames: [String]? - /// Creates a new `FunctionCallingConfig`. - /// - /// - Parameters: - /// - mode: Specifies the mode in which function calling should execute; if unspecified, the - /// default behavior will be ``Mode/auto``. - /// - allowedFunctionNames: A set of function names that, when provided, limits the functions - /// the model will call. - /// Note: This should only be set when the ``Mode`` is ``Mode/any``. Function names should match - /// `[FunctionDeclaration.name]`. With mode set to ``Mode/any``, the model will predict a - /// function call from the set of function names provided. - public init(mode: FunctionCallingConfig.Mode? = nil, allowedFunctionNames: [String]? = nil) { + init(mode: FunctionCallingConfig.Mode? = nil, allowedFunctionNames: [String]? = nil) { self.mode = mode self.allowedFunctionNames = allowedFunctionNames } + + /// Creates a function calling config where the model calls functions at its discretion. + /// + /// > Note: This is the default behavior. + public static func auto() -> FunctionCallingConfig { + return FunctionCallingConfig(mode: .auto) + } + + /// Creates a function calling config where the model will always call a provided function. + /// + /// - Parameters: + /// - allowedFunctionNames: A set of function names that, when provided, limits the functions + /// that the model will call. + public static func any(allowedFunctionNames: [String]? = nil) -> FunctionCallingConfig { + return FunctionCallingConfig(mode: .any, allowedFunctionNames: allowedFunctionNames) + } + + /// Creates a function calling config where the model will never call a function. + /// + /// > Note: This can also be achieved by not passing any ``FunctionDeclaration`` tools when + /// > instantiating the model. + public static func none() -> FunctionCallingConfig { + return FunctionCallingConfig(mode: FunctionCallingConfig.Mode.none) + } } /// Tool configuration for any `Tool` specified in the request. diff --git a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift index d8b25cd8ec4..fee87108da7 100644 --- a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift +++ b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift @@ -59,7 +59,7 @@ final class IntegrationTests: XCTestCase { generationConfig: generationConfig, safetySettings: safetySettings, tools: [], - toolConfig: .init(functionCallingConfig: .init(mode: FunctionCallingConfig.Mode.none)), + toolConfig: .init(functionCallingConfig: .none()), systemInstruction: systemInstruction ) } @@ -95,7 +95,7 @@ final class IntegrationTests: XCTestCase { SafetySetting(harmCategory: .dangerousContent, threshold: .blockNone), SafetySetting(harmCategory: .civicIntegrity, threshold: .off), ], - toolConfig: .init(functionCallingConfig: .init(mode: .auto)), + toolConfig: .init(functionCallingConfig: .auto()), systemInstruction: systemInstruction ) @@ -137,8 +137,8 @@ final class IntegrationTests: XCTestCase { ) model = vertex.generativeModel( modelName: "gemini-1.5-flash", - tools: [Tool(functionDeclarations: [sumDeclaration])], - toolConfig: .init(functionCallingConfig: .init(mode: .any, allowedFunctionNames: ["sum"])) + tools: [.functionDeclarations([sumDeclaration])], + toolConfig: .init(functionCallingConfig: .any(allowedFunctionNames: ["sum"])) ) let prompt = "What is 10 + 32?" let sumCall = FunctionCallPart(name: "sum", args: ["x": .number(10), "y": .number(32)]) From 3076ccbb93d9e8d1a7dd08712fa5c454d952d57c Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 10 Oct 2024 18:01:09 -0400 Subject: [PATCH 2/3] Update PR# in changelog --- FirebaseVertexAI/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FirebaseVertexAI/CHANGELOG.md b/FirebaseVertexAI/CHANGELOG.md index 539591e3e16..f2facff2411 100644 --- a/FirebaseVertexAI/CHANGELOG.md +++ b/FirebaseVertexAI/CHANGELOG.md @@ -46,10 +46,10 @@ unknown or unhandled values. (#13728, #13854, #13860) - [changed] **Breaking Change**: The `Tool` initializer is now internal; use the new type method `functionDeclarations(_:)` to create a `Tool` for function - calling. (#TODO) + calling. (#13873) - [changed] **Breaking Change**: The `FunctionCallingConfig` initializer and `Mode` enum are now internal; use one of the new type methods `auto()`, - `any(allowedFunctionNames:)`, or `none()` to create a config. (#TODO) + `any(allowedFunctionNames:)`, or `none()` to create a config. (#13873) - [changed] The default request timeout is now 180 seconds instead of the platform-default value of 60 seconds for a `URLRequest`; this timeout may still be customized in `RequestOptions`. (#13722) From f445dcbeacad8ed0dea946281332d3084ea3e5d1 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Thu, 10 Oct 2024 18:33:36 -0400 Subject: [PATCH 3/3] Replace `"OBJECT"` with `DataType.object` in docs --- FirebaseVertexAI/Sources/FunctionCalling.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FirebaseVertexAI/Sources/FunctionCalling.swift b/FirebaseVertexAI/Sources/FunctionCalling.swift index 8ca676751a3..7bf366a7132 100644 --- a/FirebaseVertexAI/Sources/FunctionCalling.swift +++ b/FirebaseVertexAI/Sources/FunctionCalling.swift @@ -25,7 +25,7 @@ public struct FunctionDeclaration { /// A brief description of the function. let description: String - /// Describes the parameters to this function; must be of type `"OBJECT"`. + /// Describes the parameters to this function; must be of type `DataType.object`. let parameters: Schema? /// Constructs a new `FunctionDeclaration`.