diff --git a/.changeset/lucky-socks-roll.md b/.changeset/lucky-socks-roll.md new file mode 100644 index 00000000000..8b9d9a19e97 --- /dev/null +++ b/.changeset/lucky-socks-roll.md @@ -0,0 +1,6 @@ +--- +'@firebase/ai': minor +'firebase': minor +--- + +Added Code Execution feature. diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 43a10169cfa..c1b570a7e05 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -174,6 +174,39 @@ export interface CitationMetadata { citations: Citation[]; } +// @public +export interface CodeExecutionResult { + outcome?: Outcome; + output?: string; +} + +// @public +export interface CodeExecutionResultPart { + // (undocumented) + codeExecutionResult?: CodeExecutionResult; + // (undocumented) + executableCode?: never; + // (undocumented) + fileData: never; + // (undocumented) + functionCall?: never; + // (undocumented) + functionResponse?: never; + // (undocumented) + inlineData?: never; + // (undocumented) + text?: never; + // (undocumented) + thought?: never; + // @internal (undocumented) + thoughtSignature?: never; +} + +// @public +export interface CodeExecutionTool { + codeExecution: {}; +} + // @public export interface Content { // (undocumented) @@ -237,6 +270,34 @@ export interface ErrorDetails { reason?: string; } +// @public +export interface ExecutableCode { + code?: string; + language?: Language; +} + +// @public +export interface ExecutableCodePart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: ExecutableCode; + // (undocumented) + fileData: never; + // (undocumented) + functionCall?: never; + // (undocumented) + functionResponse?: never; + // (undocumented) + inlineData?: never; + // (undocumented) + text?: never; + // (undocumented) + thought?: never; + // @internal (undocumented) + thoughtSignature?: never; +} + // @public export interface FileData { // (undocumented) @@ -247,6 +308,10 @@ export interface FileData { // @public export interface FileDataPart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) fileData: FileData; // (undocumented) @@ -308,6 +373,10 @@ export type FunctionCallingMode = (typeof FunctionCallingMode)[keyof typeof Func // @public export interface FunctionCallPart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) functionCall: FunctionCall; // (undocumented) @@ -345,6 +414,10 @@ export interface FunctionResponse { // @public export interface FunctionResponsePart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) functionCall?: never; // (undocumented) @@ -735,6 +808,10 @@ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; // @public export interface InlineDataPart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) functionCall?: never; // (undocumented) @@ -755,6 +832,15 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } +// @public +export const Language: { + UNSPECIFIED: string; + PYTHON: string; +}; + +// @public +export type Language = (typeof Language)[keyof typeof Language]; + // @public export interface LanguageModelCreateCoreOptions { // (undocumented) @@ -969,7 +1055,18 @@ export interface OnDeviceParams { } // @public -export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; +export const Outcome: { + UNSPECIFIED: string; + OK: string; + FAILED: string; + DEADLINE_EXCEEDED: string; +}; + +// @public +export type Outcome = (typeof Outcome)[keyof typeof Outcome]; + +// @public +export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart | ExecutableCodePart | CodeExecutionResultPart; // @public export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]; @@ -1179,6 +1276,10 @@ export class StringSchema extends Schema { // @public export interface TextPart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) functionCall?: never; // (undocumented) @@ -1200,7 +1301,7 @@ export interface ThinkingConfig { } // @public -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool; // @public export interface ToolConfig { diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 9161f501aa3..8bec927e163 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -32,6 +32,12 @@ toc: path: /docs/reference/js/ai.citation.md - title: CitationMetadata path: /docs/reference/js/ai.citationmetadata.md + - title: CodeExecutionResult + path: /docs/reference/js/ai.codeexecutionresult.md + - title: CodeExecutionResultPart + path: /docs/reference/js/ai.codeexecutionresultpart.md + - title: CodeExecutionTool + path: /docs/reference/js/ai.codeexecutiontool.md - title: Content path: /docs/reference/js/ai.content.md - title: CountTokensRequest @@ -46,6 +52,10 @@ toc: path: /docs/reference/js/ai.enhancedgeneratecontentresponse.md - title: ErrorDetails path: /docs/reference/js/ai.errordetails.md + - title: ExecutableCode + path: /docs/reference/js/ai.executablecode.md + - title: ExecutableCodePart + path: /docs/reference/js/ai.executablecodepart.md - title: FileData path: /docs/reference/js/ai.filedata.md - title: FileDataPart diff --git a/docs-devsite/ai.codeexecutionresult.md b/docs-devsite/ai.codeexecutionresult.md new file mode 100644 index 00000000000..d0be155c4e6 --- /dev/null +++ b/docs-devsite/ai.codeexecutionresult.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# CodeExecutionResult interface +The results of code execution run by the model. + +Signature: + +```typescript +export interface CodeExecutionResult +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [outcome](./ai.codeexecutionresult.md#codeexecutionresultoutcome) | [Outcome](./ai.md#outcome) | The result of the code execution. | +| [output](./ai.codeexecutionresult.md#codeexecutionresultoutput) | string | The output from the code execution, or an error message if it failed. | + +## CodeExecutionResult.outcome + +The result of the code execution. + +Signature: + +```typescript +outcome?: Outcome; +``` + +## CodeExecutionResult.output + +The output from the code execution, or an error message if it failed. + +Signature: + +```typescript +output?: string; +``` diff --git a/docs-devsite/ai.codeexecutionresultpart.md b/docs-devsite/ai.codeexecutionresultpart.md new file mode 100644 index 00000000000..6142a3ec79f --- /dev/null +++ b/docs-devsite/ai.codeexecutionresultpart.md @@ -0,0 +1,96 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# CodeExecutionResultPart interface +Represents the code execution result from the model. + +Signature: + +```typescript +export interface CodeExecutionResultPart +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [codeExecutionResult](./ai.codeexecutionresultpart.md#codeexecutionresultpartcodeexecutionresult) | [CodeExecutionResult](./ai.codeexecutionresult.md#codeexecutionresult_interface) | | +| [executableCode](./ai.codeexecutionresultpart.md#codeexecutionresultpartexecutablecode) | never | | +| [fileData](./ai.codeexecutionresultpart.md#codeexecutionresultpartfiledata) | never | | +| [functionCall](./ai.codeexecutionresultpart.md#codeexecutionresultpartfunctioncall) | never | | +| [functionResponse](./ai.codeexecutionresultpart.md#codeexecutionresultpartfunctionresponse) | never | | +| [inlineData](./ai.codeexecutionresultpart.md#codeexecutionresultpartinlinedata) | never | | +| [text](./ai.codeexecutionresultpart.md#codeexecutionresultparttext) | never | | +| [thought](./ai.codeexecutionresultpart.md#codeexecutionresultpartthought) | never | | + +## CodeExecutionResultPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: CodeExecutionResult; +``` + +## CodeExecutionResultPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + +## CodeExecutionResultPart.fileData + +Signature: + +```typescript +fileData: never; +``` + +## CodeExecutionResultPart.functionCall + +Signature: + +```typescript +functionCall?: never; +``` + +## CodeExecutionResultPart.functionResponse + +Signature: + +```typescript +functionResponse?: never; +``` + +## CodeExecutionResultPart.inlineData + +Signature: + +```typescript +inlineData?: never; +``` + +## CodeExecutionResultPart.text + +Signature: + +```typescript +text?: never; +``` + +## CodeExecutionResultPart.thought + +Signature: + +```typescript +thought?: never; +``` diff --git a/docs-devsite/ai.codeexecutiontool.md b/docs-devsite/ai.codeexecutiontool.md new file mode 100644 index 00000000000..34a4715ac63 --- /dev/null +++ b/docs-devsite/ai.codeexecutiontool.md @@ -0,0 +1,35 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# CodeExecutionTool interface +A tool that enables the model to use code execution. + +Signature: + +```typescript +export interface CodeExecutionTool +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [codeExecution](./ai.codeexecutiontool.md#codeexecutiontoolcodeexecution) | {} | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. | + +## CodeExecutionTool.codeExecution + +Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. + +Signature: + +```typescript +codeExecution: {}; +``` diff --git a/docs-devsite/ai.executablecode.md b/docs-devsite/ai.executablecode.md new file mode 100644 index 00000000000..bab79bacc50 --- /dev/null +++ b/docs-devsite/ai.executablecode.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ExecutableCode interface +An interface for executable code returned by the model. + +Signature: + +```typescript +export interface ExecutableCode +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [code](./ai.executablecode.md#executablecodecode) | string | The source code to be executed. | +| [language](./ai.executablecode.md#executablecodelanguage) | [Language](./ai.md#language) | The programming language of the code. | + +## ExecutableCode.code + +The source code to be executed. + +Signature: + +```typescript +code?: string; +``` + +## ExecutableCode.language + +The programming language of the code. + +Signature: + +```typescript +language?: Language; +``` diff --git a/docs-devsite/ai.executablecodepart.md b/docs-devsite/ai.executablecodepart.md new file mode 100644 index 00000000000..99c6bf05987 --- /dev/null +++ b/docs-devsite/ai.executablecodepart.md @@ -0,0 +1,96 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ExecutableCodePart interface +Represents the code that is executed by the model. + +Signature: + +```typescript +export interface ExecutableCodePart +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [codeExecutionResult](./ai.executablecodepart.md#executablecodepartcodeexecutionresult) | never | | +| [executableCode](./ai.executablecodepart.md#executablecodepartexecutablecode) | [ExecutableCode](./ai.executablecode.md#executablecode_interface) | | +| [fileData](./ai.executablecodepart.md#executablecodepartfiledata) | never | | +| [functionCall](./ai.executablecodepart.md#executablecodepartfunctioncall) | never | | +| [functionResponse](./ai.executablecodepart.md#executablecodepartfunctionresponse) | never | | +| [inlineData](./ai.executablecodepart.md#executablecodepartinlinedata) | never | | +| [text](./ai.executablecodepart.md#executablecodeparttext) | never | | +| [thought](./ai.executablecodepart.md#executablecodepartthought) | never | | + +## ExecutableCodePart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## ExecutableCodePart.executableCode + +Signature: + +```typescript +executableCode?: ExecutableCode; +``` + +## ExecutableCodePart.fileData + +Signature: + +```typescript +fileData: never; +``` + +## ExecutableCodePart.functionCall + +Signature: + +```typescript +functionCall?: never; +``` + +## ExecutableCodePart.functionResponse + +Signature: + +```typescript +functionResponse?: never; +``` + +## ExecutableCodePart.inlineData + +Signature: + +```typescript +inlineData?: never; +``` + +## ExecutableCodePart.text + +Signature: + +```typescript +text?: never; +``` + +## ExecutableCodePart.thought + +Signature: + +```typescript +thought?: never; +``` diff --git a/docs-devsite/ai.filedatapart.md b/docs-devsite/ai.filedatapart.md index 2b5179319f7..f031988a993 100644 --- a/docs-devsite/ai.filedatapart.md +++ b/docs-devsite/ai.filedatapart.md @@ -22,6 +22,8 @@ export interface FileDataPart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.filedatapart.md#filedatapartcodeexecutionresult) | never | | +| [executableCode](./ai.filedatapart.md#filedatapartexecutablecode) | never | | | [fileData](./ai.filedatapart.md#filedatapartfiledata) | [FileData](./ai.filedata.md#filedata_interface) | | | [functionCall](./ai.filedatapart.md#filedatapartfunctioncall) | never | | | [functionResponse](./ai.filedatapart.md#filedatapartfunctionresponse) | never | | @@ -29,6 +31,22 @@ export interface FileDataPart | [text](./ai.filedatapart.md#filedataparttext) | never | | | [thought](./ai.filedatapart.md#filedatapartthought) | boolean | | +## FileDataPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## FileDataPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## FileDataPart.fileData Signature: diff --git a/docs-devsite/ai.functioncallpart.md b/docs-devsite/ai.functioncallpart.md index 3f07c5d0d74..cb9d8b89cf2 100644 --- a/docs-devsite/ai.functioncallpart.md +++ b/docs-devsite/ai.functioncallpart.md @@ -22,12 +22,30 @@ export interface FunctionCallPart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.functioncallpart.md#functioncallpartcodeexecutionresult) | never | | +| [executableCode](./ai.functioncallpart.md#functioncallpartexecutablecode) | never | | | [functionCall](./ai.functioncallpart.md#functioncallpartfunctioncall) | [FunctionCall](./ai.functioncall.md#functioncall_interface) | | | [functionResponse](./ai.functioncallpart.md#functioncallpartfunctionresponse) | never | | | [inlineData](./ai.functioncallpart.md#functioncallpartinlinedata) | never | | | [text](./ai.functioncallpart.md#functioncallparttext) | never | | | [thought](./ai.functioncallpart.md#functioncallpartthought) | boolean | | +## FunctionCallPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## FunctionCallPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## FunctionCallPart.functionCall Signature: diff --git a/docs-devsite/ai.functionresponsepart.md b/docs-devsite/ai.functionresponsepart.md index 4e8c9ea5724..2d80e1706a3 100644 --- a/docs-devsite/ai.functionresponsepart.md +++ b/docs-devsite/ai.functionresponsepart.md @@ -22,12 +22,30 @@ export interface FunctionResponsePart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.functionresponsepart.md#functionresponsepartcodeexecutionresult) | never | | +| [executableCode](./ai.functionresponsepart.md#functionresponsepartexecutablecode) | never | | | [functionCall](./ai.functionresponsepart.md#functionresponsepartfunctioncall) | never | | | [functionResponse](./ai.functionresponsepart.md#functionresponsepartfunctionresponse) | [FunctionResponse](./ai.functionresponse.md#functionresponse_interface) | | | [inlineData](./ai.functionresponsepart.md#functionresponsepartinlinedata) | never | | | [text](./ai.functionresponsepart.md#functionresponseparttext) | never | | | [thought](./ai.functionresponsepart.md#functionresponsepartthought) | boolean | | +## FunctionResponsePart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## FunctionResponsePart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## FunctionResponsePart.functionCall Signature: diff --git a/docs-devsite/ai.googlesearchtool.md b/docs-devsite/ai.googlesearchtool.md index 4e446b0cdea..ef29ccbdaa0 100644 --- a/docs-devsite/ai.googlesearchtool.md +++ b/docs-devsite/ai.googlesearchtool.md @@ -24,11 +24,11 @@ export interface GoogleSearchTool | Property | Type | Description | | --- | --- | --- | -| [googleSearch](./ai.googlesearchtool.md#googlesearchtoolgooglesearch) | [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options.When using this feature, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | +| [googleSearch](./ai.googlesearchtool.md#googlesearchtoolgooglesearch) | [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options.When using this feature, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | ## GoogleSearchTool.googleSearch -Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. +Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. When using this feature, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). diff --git a/docs-devsite/ai.inlinedatapart.md b/docs-devsite/ai.inlinedatapart.md index c9ead9d061d..a3d581861e7 100644 --- a/docs-devsite/ai.inlinedatapart.md +++ b/docs-devsite/ai.inlinedatapart.md @@ -22,6 +22,8 @@ export interface InlineDataPart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.inlinedatapart.md#inlinedatapartcodeexecutionresult) | never | | +| [executableCode](./ai.inlinedatapart.md#inlinedatapartexecutablecode) | never | | | [functionCall](./ai.inlinedatapart.md#inlinedatapartfunctioncall) | never | | | [functionResponse](./ai.inlinedatapart.md#inlinedatapartfunctionresponse) | never | | | [inlineData](./ai.inlinedatapart.md#inlinedatapartinlinedata) | [GenerativeContentBlob](./ai.generativecontentblob.md#generativecontentblob_interface) | | @@ -29,6 +31,22 @@ export interface InlineDataPart | [thought](./ai.inlinedatapart.md#inlinedatapartthought) | boolean | | | [videoMetadata](./ai.inlinedatapart.md#inlinedatapartvideometadata) | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Applicable if inlineData is a video. | +## InlineDataPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## InlineDataPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## InlineDataPart.functionCall Signature: diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index d94c31155de..e4e382256b3 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -60,6 +60,9 @@ The Firebase AI Web SDK. | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | (EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device inference is possible.These methods should not be called directly by the user. | | [Citation](./ai.citation.md#citation_interface) | A single citation. | | [CitationMetadata](./ai.citationmetadata.md#citationmetadata_interface) | Citation metadata that may be found on a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface). | +| [CodeExecutionResult](./ai.codeexecutionresult.md#codeexecutionresult_interface) | The results of code execution run by the model. | +| [CodeExecutionResultPart](./ai.codeexecutionresultpart.md#codeexecutionresultpart_interface) | Represents the code execution result from the model. | +| [CodeExecutionTool](./ai.codeexecutiontool.md#codeexecutiontool_interface) | A tool that enables the model to use code execution. | | [Content](./ai.content.md#content_interface) | Content type for both prompts and response candidates. | | [CountTokensRequest](./ai.counttokensrequest.md#counttokensrequest_interface) | Params for calling [GenerativeModel.countTokens()](./ai.generativemodel.md#generativemodelcounttokens) | | [CountTokensResponse](./ai.counttokensresponse.md#counttokensresponse_interface) | Response from calling [GenerativeModel.countTokens()](./ai.generativemodel.md#generativemodelcounttokens). | @@ -67,6 +70,8 @@ The Firebase AI Web SDK. | [Date\_2](./ai.date_2.md#date_2_interface) | Protobuf google.type.Date | | [EnhancedGenerateContentResponse](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponse_interface) | Response object wrapped with helper methods. | | [ErrorDetails](./ai.errordetails.md#errordetails_interface) | Details object that may be included in an error response. | +| [ExecutableCode](./ai.executablecode.md#executablecode_interface) | An interface for executable code returned by the model. | +| [ExecutableCodePart](./ai.executablecodepart.md#executablecodepart_interface) | Represents the code that is executed by the model. | | [FileData](./ai.filedata.md#filedata_interface) | Data pointing to a file uploaded on Google Cloud Storage. | | [FileDataPart](./ai.filedatapart.md#filedatapart_interface) | Content part interface if the part represents [FileData](./ai.filedata.md#filedata_interface) | | [FunctionCall](./ai.functioncall.md#functioncall_interface) | A predicted [FunctionCall](./ai.functioncall.md#functioncall_interface) returned from the model that contains a string representing the [FunctionDeclaration.name](./ai.functiondeclaration.md#functiondeclarationname) and a structured JSON object containing the parameters and their values. | @@ -153,8 +158,10 @@ The Firebase AI Web SDK. | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | | [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | +| [Language](./ai.md#language) | The programming language of the code. | | [LiveResponseType](./ai.md#liveresponsetype) | (Public Preview) The types of responses that can be returned by [LiveSession.receive()](./ai.livesession.md#livesessionreceive). | | [Modality](./ai.md#modality) | Content part modality. | +| [Outcome](./ai.md#outcome) | Represents the result of the code execution. | | [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | @@ -177,11 +184,13 @@ The Firebase AI Web SDK. | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | | [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | +| [Language](./ai.md#language) | The programming language of the code. | | [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | (EXPERIMENTAL) Content formats that can be provided as on-device message content. | | [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | (EXPERIMENTAL) Allowable roles for on-device language model usage. | | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | (EXPERIMENTAL) Allowable types for on-device language model messages. | | [LiveResponseType](./ai.md#liveresponsetype) | (Public Preview) The types of responses that can be returned by [LiveSession.receive()](./ai.livesession.md#livesessionreceive). This is a property on all messages that can be used for type narrowing. This property is not returned by the server, it is assigned to a server message object once it's parsed. | | [Modality](./ai.md#modality) | Content part modality. | +| [Outcome](./ai.md#outcome) | Represents the result of the code execution. | | [Part](./ai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [Role](./ai.md#role) | Role is the producer of the content. | @@ -637,6 +646,19 @@ InferenceMode: { } ``` +## Language + +The programming language of the code. + +Signature: + +```typescript +Language: { + UNSPECIFIED: string; + PYTHON: string; +} +``` + ## LiveResponseType > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. @@ -671,6 +693,21 @@ Modality: { } ``` +## Outcome + +Represents the result of the code execution. + +Signature: + +```typescript +Outcome: { + UNSPECIFIED: string; + OK: string; + FAILED: string; + DEADLINE_EXCEEDED: string; +} +``` + ## POSSIBLE\_ROLES Possible roles. @@ -871,6 +908,16 @@ export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; ``` +## Language + +The programming language of the code. + +Signature: + +```typescript +export type Language = (typeof Language)[keyof typeof Language]; +``` + ## LanguageModelMessageContentValue (EXPERIMENTAL) Content formats that can be provided as on-device message content. @@ -924,6 +971,16 @@ Content part modality. export type Modality = (typeof Modality)[keyof typeof Modality]; ``` +## Outcome + +Represents the result of the code execution. + +Signature: + +```typescript +export type Outcome = (typeof Outcome)[keyof typeof Outcome]; +``` + ## Part Content part - includes text, image/video, or function call/response part types. @@ -931,7 +988,7 @@ Content part - includes text, image/video, or function call/response part types. Signature: ```typescript -export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; +export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart | ExecutableCodePart | CodeExecutionResultPart; ``` ## ResponseModality @@ -974,7 +1031,7 @@ Defines a tool that model can call to access external knowledge. Signature: ```typescript -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool; ``` ## TypedSchema diff --git a/docs-devsite/ai.textpart.md b/docs-devsite/ai.textpart.md index 2466f9cca8f..22236b37d7d 100644 --- a/docs-devsite/ai.textpart.md +++ b/docs-devsite/ai.textpart.md @@ -22,12 +22,30 @@ export interface TextPart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.textpart.md#textpartcodeexecutionresult) | never | | +| [executableCode](./ai.textpart.md#textpartexecutablecode) | never | | | [functionCall](./ai.textpart.md#textpartfunctioncall) | never | | | [functionResponse](./ai.textpart.md#textpartfunctionresponse) | never | | | [inlineData](./ai.textpart.md#textpartinlinedata) | never | | | [text](./ai.textpart.md#textparttext) | string | | | [thought](./ai.textpart.md#textpartthought) | boolean | | +## TextPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## TextPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## TextPart.functionCall Signature: diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 0b83df38ecb..a827a447d90 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -21,7 +21,9 @@ import { GenerationConfig, HarmBlockThreshold, HarmCategory, + Language, Modality, + Outcome, SafetySetting, getGenerativeModel } from '../src'; @@ -188,6 +190,32 @@ describe('Generate Content', () => { }); }); + it('generateContent: code execution', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + tools: [{ codeExecution: {} }] + }); + const prompt = + 'What is the sum of the first 50 prime numbers? ' + + 'Generate and run code for the calculation, and make sure you get all 50.'; + + const result = await model.generateContent(prompt); + const parts = result.response.candidates?.[0].content.parts; + expect( + parts?.some(part => part.executableCode?.language === Language.PYTHON) + ).to.be.true; + expect( + parts?.some(part => part.codeExecutionResult?.outcome === Outcome.OK) + ).to.be.true; + // Expect these to be truthy (!= null) + expect(parts?.some(part => part.executableCode?.code != null)).to.be + .true; + expect(parts?.some(part => part.codeExecutionResult?.output != null)).to + .be.true; + }); + it('generateContentStream: text input, text output', async () => { const model = getGenerativeModel(testConfig.ai, { model: testConfig.model, diff --git a/packages/ai/src/methods/chat-session-helpers.ts b/packages/ai/src/methods/chat-session-helpers.ts index 709f616f3c0..3c0c58b7bf5 100644 --- a/packages/ai/src/methods/chat-session-helpers.ts +++ b/packages/ai/src/methods/chat-session-helpers.ts @@ -84,7 +84,9 @@ export function validateChatHistory(history: Content[]): void { functionCall: 0, functionResponse: 0, thought: 0, - thoughtSignature: 0 + thoughtSignature: 0, + executableCode: 0, + codeExecutionResult: 0 }; for (const part of parts) { diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index 19c0761949a..3bb396ac6d8 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -28,7 +28,9 @@ import { HarmBlockMethod, HarmBlockThreshold, HarmCategory, - InferenceMode + InferenceMode, + Language, + Outcome } from '../types'; import { ApiSettings } from '../types/internal'; import { Task } from '../requests/request'; @@ -248,6 +250,25 @@ describe('generateContent()', () => { match.any ); }); + it('codeExecution', async () => { + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-code-execution.json' + ); + stub(request, 'makeRequest').resolves(mockResponse as Response); + const result = await generateContent( + fakeApiSettings, + 'model', + fakeRequestParams + ); + const parts = result.response.candidates?.[0].content.parts; + expect( + parts?.some(part => part.codeExecutionResult?.outcome === Outcome.OK) + ).to.be.true; + expect( + parts?.some(part => part.executableCode?.language === Language.PYTHON) + ).to.be.true; + }); it('blocked prompt', async () => { const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/models/generative-model.test.ts b/packages/ai/src/models/generative-model.test.ts index 9c5074b8eb3..bcd78d746d4 100644 --- a/packages/ai/src/models/generative-model.test.ts +++ b/packages/ai/src/models/generative-model.test.ts @@ -71,7 +71,9 @@ describe('GenerativeModel', () => { description: 'mydesc' } ] - } + }, + { googleSearch: {} }, + { codeExecution: {} } ], toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } @@ -81,7 +83,7 @@ describe('GenerativeModel', () => { {}, fakeChromeAdapter ); - expect(genModel.tools?.length).to.equal(1); + expect(genModel.tools?.length).to.equal(3); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE ); @@ -102,6 +104,8 @@ describe('GenerativeModel', () => { match((value: string) => { return ( value.includes('myfunc') && + value.includes('googleSearch') && + value.includes('codeExecution') && value.includes(FunctionCallingMode.NONE) && value.includes('be friendly') ); @@ -183,7 +187,9 @@ describe('GenerativeModel', () => { functionDeclarations: [ { name: 'otherfunc', description: 'otherdesc' } ] - } + }, + { googleSearch: {} }, + { codeExecution: {} } ], toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.AUTO } }, systemInstruction: { role: 'system', parts: [{ text: 'be formal' }] } @@ -196,6 +202,8 @@ describe('GenerativeModel', () => { match((value: string) => { return ( value.includes('otherfunc') && + value.includes('googleSearch') && + value.includes('codeExecution') && value.includes(FunctionCallingMode.AUTO) && value.includes('be formal') ); @@ -249,7 +257,9 @@ describe('GenerativeModel', () => { { model: 'my-model', tools: [ - { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } + { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] }, + { googleSearch: {} }, + { codeExecution: {} } ], toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } @@ -262,7 +272,7 @@ describe('GenerativeModel', () => { {}, fakeChromeAdapter ); - expect(genModel.tools?.length).to.equal(1); + expect(genModel.tools?.length).to.equal(3); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE ); @@ -283,6 +293,8 @@ describe('GenerativeModel', () => { match((value: string) => { return ( value.includes('myfunc') && + value.includes('googleSearch') && + value.includes('codeExecution') && value.includes(FunctionCallingMode.NONE) && value.includes('be friendly') && value.includes('topK') @@ -361,7 +373,9 @@ describe('GenerativeModel', () => { functionDeclarations: [ { name: 'otherfunc', description: 'otherdesc' } ] - } + }, + { googleSearch: {} }, + { codeExecution: {} } ], toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.AUTO } @@ -380,6 +394,8 @@ describe('GenerativeModel', () => { match((value: string) => { return ( value.includes('otherfunc') && + value.includes('googleSearch') && + value.includes('codeExecution') && value.includes(FunctionCallingMode.AUTO) && value.includes('be formal') && value.includes('image/png') && diff --git a/packages/ai/src/types/content.ts b/packages/ai/src/types/content.ts index bd8ccf1ef10..b51dcd9c69f 100644 --- a/packages/ai/src/types/content.ts +++ b/packages/ai/src/types/content.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { Role } from './enums'; +import { Language, Outcome, Role } from './enums'; /** * Content type for both prompts and response candidates. @@ -36,7 +36,9 @@ export type Part = | InlineDataPart | FunctionCallPart | FunctionResponsePart - | FileDataPart; + | FileDataPart + | ExecutableCodePart + | CodeExecutionResultPart; /** * Content part interface if the part represents a text string. @@ -52,6 +54,8 @@ export interface TextPart { * @internal */ thoughtSignature?: string; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -72,6 +76,8 @@ export interface InlineDataPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -105,6 +111,8 @@ export interface FunctionCallPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -121,6 +129,8 @@ export interface FunctionResponsePart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -138,6 +148,81 @@ export interface FileDataPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; +} + +/** + * Represents the code that is executed by the model. + * + * @public + */ +export interface ExecutableCodePart { + text?: never; + inlineData?: never; + functionCall?: never; + functionResponse?: never; + fileData: never; + thought?: never; + /** + * @internal + */ + thoughtSignature?: never; + executableCode?: ExecutableCode; + codeExecutionResult?: never; +} + +/** + * Represents the code execution result from the model. + * + * @public + */ +export interface CodeExecutionResultPart { + text?: never; + inlineData?: never; + functionCall?: never; + functionResponse?: never; + fileData: never; + thought?: never; + /** + * @internal + */ + thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: CodeExecutionResult; +} + +/** + * An interface for executable code returned by the model. + * + * @public + */ +export interface ExecutableCode { + /** + * The programming language of the code. + */ + language?: Language; + /** + * The source code to be executed. + */ + code?: string; +} + +/** + * The results of code execution run by the model. + * + * @public + */ +export interface CodeExecutionResult { + /** + * The result of the code execution. + */ + outcome?: Outcome; + /** + * The output from the code execution, or an error message + * if it failed. + */ + output?: string; } /** diff --git a/packages/ai/src/types/enums.ts b/packages/ai/src/types/enums.ts index de70d325157..177dde296b7 100644 --- a/packages/ai/src/types/enums.ts +++ b/packages/ai/src/types/enums.ts @@ -380,3 +380,39 @@ export const InferenceMode = { * @public */ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; + +/** + * Represents the result of the code execution. + * + * @public + */ +export const Outcome = { + UNSPECIFIED: 'OUTCOME_UNSPECIFIED', + OK: 'OUTCOME_OK', + FAILED: 'OUTCOME_FAILED', + DEADLINE_EXCEEDED: 'OUTCOME_DEADLINE_EXCEEDED' +}; + +/** + * Represents the result of the code execution. + * + * @public + */ +export type Outcome = (typeof Outcome)[keyof typeof Outcome]; + +/** + * The programming language of the code. + * + * @public + */ +export const Language = { + UNSPECIFIED: 'LANGUAGE_UNSPECIFIED', + PYTHON: 'PYTHON' +}; + +/** + * The programming language of the code. + * + * @public + */ +export type Language = (typeof Language)[keyof typeof Language]; diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 21ccce8bd11..0ce87d0c8da 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -239,7 +239,10 @@ export interface RequestOptions { * Defines a tool that model can call to access external knowledge. * @public */ -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = + | FunctionDeclarationsTool + | GoogleSearchTool + | CodeExecutionTool; /** * Structured representation of a function declaration as defined by the @@ -285,8 +288,6 @@ export interface GoogleSearchTool { /** * Specifies the Google Search configuration. * Currently, this is an empty object, but it's reserved for future configuration options. - * Specifies the Google Search configuration. Currently, this is an empty object, but it's - * reserved for future configuration options. * * When using this feature, you are required to comply with the "Grounding with Google Search" * usage requirements for your chosen API provider: {@link https://ai.google.dev/gemini-api/terms#grounding-with-google-search | Gemini Developer API} @@ -296,6 +297,19 @@ export interface GoogleSearchTool { googleSearch: GoogleSearch; } +/** + * A tool that enables the model to use code execution. + * + * @public + */ +export interface CodeExecutionTool { + /** + * Specifies the Google Search configuration. + * Currently, this is an empty object, but it's reserved for future configuration options. + */ + codeExecution: {}; +} + /** * Specifies the Google Search configuration. *