From 92fd93690d50415c47725f2c74b04962a1cb0e15 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Thu, 11 Sep 2025 12:57:58 -0700 Subject: [PATCH 1/3] Add code execution. --- .changeset/lucky-socks-roll.md | 6 + common/api-review/ai.api.md | 105 ++++++++++++- docs-devsite/_toc.yaml | 15 ++ docs-devsite/ai.codeexecutionresult.md | 46 ++++++ docs-devsite/ai.codeexecutionresultpart.md | 96 ++++++++++++ docs-devsite/ai.codeexecutiontool.md | 35 +++++ docs-devsite/ai.executablecode.md | 46 ++++++ docs-devsite/ai.executablecodepart.md | 96 ++++++++++++ docs-devsite/ai.filedatapart.md | 18 +++ docs-devsite/ai.functioncallpart.md | 18 +++ docs-devsite/ai.functionresponsepart.md | 18 +++ docs-devsite/ai.googlesearchtool.md | 4 +- docs-devsite/ai.inlinedatapart.md | 18 +++ docs-devsite/ai.md | 61 +++++++- docs-devsite/ai.textpart.md | 18 +++ docs-devsite/index.md | 1 + docs-devsite/telemetry.instrumentation.md | 31 ++++ docs-devsite/telemetry.md | 138 ++++++++++++++++++ docs-devsite/telemetry.telemetry.md | 48 ++++++ .../ai/src/methods/chat-session-helpers.ts | 4 +- .../ai/src/methods/generate-content.test.ts | 24 ++- .../ai/src/models/generative-model.test.ts | 28 +++- packages/ai/src/types/content.ts | 89 ++++++++++- packages/ai/src/types/enums.ts | 36 +++++ packages/ai/src/types/requests.ts | 20 ++- 25 files changed, 1000 insertions(+), 19 deletions(-) create mode 100644 .changeset/lucky-socks-roll.md create mode 100644 docs-devsite/ai.codeexecutionresult.md create mode 100644 docs-devsite/ai.codeexecutionresultpart.md create mode 100644 docs-devsite/ai.codeexecutiontool.md create mode 100644 docs-devsite/ai.executablecode.md create mode 100644 docs-devsite/ai.executablecodepart.md create mode 100644 docs-devsite/telemetry.instrumentation.md create mode 100644 docs-devsite/telemetry.md create mode 100644 docs-devsite/telemetry.telemetry.md diff --git a/.changeset/lucky-socks-roll.md b/.changeset/lucky-socks-roll.md new file mode 100644 index 00000000000..2e89ebf0769 --- /dev/null +++ b/.changeset/lucky-socks-roll.md @@ -0,0 +1,6 @@ +--- +'@firebase/ai': minor +'firebase': minor +--- + +Add Code Execution feature. diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 43a10169cfa..c1b570a7e05 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -174,6 +174,39 @@ export interface CitationMetadata { citations: Citation[]; } +// @public +export interface CodeExecutionResult { + outcome?: Outcome; + output?: string; +} + +// @public +export interface CodeExecutionResultPart { + // (undocumented) + codeExecutionResult?: CodeExecutionResult; + // (undocumented) + executableCode?: never; + // (undocumented) + fileData: never; + // (undocumented) + functionCall?: never; + // (undocumented) + functionResponse?: never; + // (undocumented) + inlineData?: never; + // (undocumented) + text?: never; + // (undocumented) + thought?: never; + // @internal (undocumented) + thoughtSignature?: never; +} + +// @public +export interface CodeExecutionTool { + codeExecution: {}; +} + // @public export interface Content { // (undocumented) @@ -237,6 +270,34 @@ export interface ErrorDetails { reason?: string; } +// @public +export interface ExecutableCode { + code?: string; + language?: Language; +} + +// @public +export interface ExecutableCodePart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: ExecutableCode; + // (undocumented) + fileData: never; + // (undocumented) + functionCall?: never; + // (undocumented) + functionResponse?: never; + // (undocumented) + inlineData?: never; + // (undocumented) + text?: never; + // (undocumented) + thought?: never; + // @internal (undocumented) + thoughtSignature?: never; +} + // @public export interface FileData { // (undocumented) @@ -247,6 +308,10 @@ export interface FileData { // @public export interface FileDataPart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) fileData: FileData; // (undocumented) @@ -308,6 +373,10 @@ export type FunctionCallingMode = (typeof FunctionCallingMode)[keyof typeof Func // @public export interface FunctionCallPart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) functionCall: FunctionCall; // (undocumented) @@ -345,6 +414,10 @@ export interface FunctionResponse { // @public export interface FunctionResponsePart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) functionCall?: never; // (undocumented) @@ -735,6 +808,10 @@ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; // @public export interface InlineDataPart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) functionCall?: never; // (undocumented) @@ -755,6 +832,15 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } +// @public +export const Language: { + UNSPECIFIED: string; + PYTHON: string; +}; + +// @public +export type Language = (typeof Language)[keyof typeof Language]; + // @public export interface LanguageModelCreateCoreOptions { // (undocumented) @@ -969,7 +1055,18 @@ export interface OnDeviceParams { } // @public -export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; +export const Outcome: { + UNSPECIFIED: string; + OK: string; + FAILED: string; + DEADLINE_EXCEEDED: string; +}; + +// @public +export type Outcome = (typeof Outcome)[keyof typeof Outcome]; + +// @public +export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart | ExecutableCodePart | CodeExecutionResultPart; // @public export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"]; @@ -1179,6 +1276,10 @@ export class StringSchema extends Schema { // @public export interface TextPart { + // (undocumented) + codeExecutionResult?: never; + // (undocumented) + executableCode?: never; // (undocumented) functionCall?: never; // (undocumented) @@ -1200,7 +1301,7 @@ export interface ThinkingConfig { } // @public -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool; // @public export interface ToolConfig { diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 9161f501aa3..d86a97079de 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -32,6 +32,12 @@ toc: path: /docs/reference/js/ai.citation.md - title: CitationMetadata path: /docs/reference/js/ai.citationmetadata.md + - title: CodeExecutionResult + path: /docs/reference/js/ai.codeexecutionresult.md + - title: CodeExecutionResultPart + path: /docs/reference/js/ai.codeexecutionresultpart.md + - title: CodeExecutionTool + path: /docs/reference/js/ai.codeexecutiontool.md - title: Content path: /docs/reference/js/ai.content.md - title: CountTokensRequest @@ -46,6 +52,10 @@ toc: path: /docs/reference/js/ai.enhancedgeneratecontentresponse.md - title: ErrorDetails path: /docs/reference/js/ai.errordetails.md + - title: ExecutableCode + path: /docs/reference/js/ai.executablecode.md + - title: ExecutableCodePart + path: /docs/reference/js/ai.executablecodepart.md - title: FileData path: /docs/reference/js/ai.filedata.md - title: FileDataPart @@ -672,3 +682,8 @@ toc: path: /docs/reference/js/storage.uploadtask.md - title: UploadTaskSnapshot path: /docs/reference/js/storage.uploadtasksnapshot.md +- title: telemetry + path: /docs/reference/js/telemetry.md + section: + - title: Telemetry + path: /docs/reference/js/telemetry.telemetry.md diff --git a/docs-devsite/ai.codeexecutionresult.md b/docs-devsite/ai.codeexecutionresult.md new file mode 100644 index 00000000000..31587a4d256 --- /dev/null +++ b/docs-devsite/ai.codeexecutionresult.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# CodeExecutionResult interface +The results of code execution run by the model. + +Signature: + +```typescript +export interface CodeExecutionResult +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [outcome](./ai.codeexecutionresult.md#codeexecutionresultoutcome) | [Outcome](./ai.md#outcome) | The result of the execution. | +| [output](./ai.codeexecutionresult.md#codeexecutionresultoutput) | string | The output from the code execution, or an error message if it failed. | + +## CodeExecutionResult.outcome + +The result of the execution. + +Signature: + +```typescript +outcome?: Outcome; +``` + +## CodeExecutionResult.output + +The output from the code execution, or an error message if it failed. + +Signature: + +```typescript +output?: string; +``` diff --git a/docs-devsite/ai.codeexecutionresultpart.md b/docs-devsite/ai.codeexecutionresultpart.md new file mode 100644 index 00000000000..6142a3ec79f --- /dev/null +++ b/docs-devsite/ai.codeexecutionresultpart.md @@ -0,0 +1,96 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# CodeExecutionResultPart interface +Represents the code execution result from the model. + +Signature: + +```typescript +export interface CodeExecutionResultPart +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [codeExecutionResult](./ai.codeexecutionresultpart.md#codeexecutionresultpartcodeexecutionresult) | [CodeExecutionResult](./ai.codeexecutionresult.md#codeexecutionresult_interface) | | +| [executableCode](./ai.codeexecutionresultpart.md#codeexecutionresultpartexecutablecode) | never | | +| [fileData](./ai.codeexecutionresultpart.md#codeexecutionresultpartfiledata) | never | | +| [functionCall](./ai.codeexecutionresultpart.md#codeexecutionresultpartfunctioncall) | never | | +| [functionResponse](./ai.codeexecutionresultpart.md#codeexecutionresultpartfunctionresponse) | never | | +| [inlineData](./ai.codeexecutionresultpart.md#codeexecutionresultpartinlinedata) | never | | +| [text](./ai.codeexecutionresultpart.md#codeexecutionresultparttext) | never | | +| [thought](./ai.codeexecutionresultpart.md#codeexecutionresultpartthought) | never | | + +## CodeExecutionResultPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: CodeExecutionResult; +``` + +## CodeExecutionResultPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + +## CodeExecutionResultPart.fileData + +Signature: + +```typescript +fileData: never; +``` + +## CodeExecutionResultPart.functionCall + +Signature: + +```typescript +functionCall?: never; +``` + +## CodeExecutionResultPart.functionResponse + +Signature: + +```typescript +functionResponse?: never; +``` + +## CodeExecutionResultPart.inlineData + +Signature: + +```typescript +inlineData?: never; +``` + +## CodeExecutionResultPart.text + +Signature: + +```typescript +text?: never; +``` + +## CodeExecutionResultPart.thought + +Signature: + +```typescript +thought?: never; +``` diff --git a/docs-devsite/ai.codeexecutiontool.md b/docs-devsite/ai.codeexecutiontool.md new file mode 100644 index 00000000000..34a4715ac63 --- /dev/null +++ b/docs-devsite/ai.codeexecutiontool.md @@ -0,0 +1,35 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# CodeExecutionTool interface +A tool that enables the model to use code execution. + +Signature: + +```typescript +export interface CodeExecutionTool +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [codeExecution](./ai.codeexecutiontool.md#codeexecutiontoolcodeexecution) | {} | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. | + +## CodeExecutionTool.codeExecution + +Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. + +Signature: + +```typescript +codeExecution: {}; +``` diff --git a/docs-devsite/ai.executablecode.md b/docs-devsite/ai.executablecode.md new file mode 100644 index 00000000000..bab79bacc50 --- /dev/null +++ b/docs-devsite/ai.executablecode.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ExecutableCode interface +An interface for executable code returned by the model. + +Signature: + +```typescript +export interface ExecutableCode +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [code](./ai.executablecode.md#executablecodecode) | string | The source code to be executed. | +| [language](./ai.executablecode.md#executablecodelanguage) | [Language](./ai.md#language) | The programming language of the code. | + +## ExecutableCode.code + +The source code to be executed. + +Signature: + +```typescript +code?: string; +``` + +## ExecutableCode.language + +The programming language of the code. + +Signature: + +```typescript +language?: Language; +``` diff --git a/docs-devsite/ai.executablecodepart.md b/docs-devsite/ai.executablecodepart.md new file mode 100644 index 00000000000..99c6bf05987 --- /dev/null +++ b/docs-devsite/ai.executablecodepart.md @@ -0,0 +1,96 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ExecutableCodePart interface +Represents the code that is executed by the model. + +Signature: + +```typescript +export interface ExecutableCodePart +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [codeExecutionResult](./ai.executablecodepart.md#executablecodepartcodeexecutionresult) | never | | +| [executableCode](./ai.executablecodepart.md#executablecodepartexecutablecode) | [ExecutableCode](./ai.executablecode.md#executablecode_interface) | | +| [fileData](./ai.executablecodepart.md#executablecodepartfiledata) | never | | +| [functionCall](./ai.executablecodepart.md#executablecodepartfunctioncall) | never | | +| [functionResponse](./ai.executablecodepart.md#executablecodepartfunctionresponse) | never | | +| [inlineData](./ai.executablecodepart.md#executablecodepartinlinedata) | never | | +| [text](./ai.executablecodepart.md#executablecodeparttext) | never | | +| [thought](./ai.executablecodepart.md#executablecodepartthought) | never | | + +## ExecutableCodePart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## ExecutableCodePart.executableCode + +Signature: + +```typescript +executableCode?: ExecutableCode; +``` + +## ExecutableCodePart.fileData + +Signature: + +```typescript +fileData: never; +``` + +## ExecutableCodePart.functionCall + +Signature: + +```typescript +functionCall?: never; +``` + +## ExecutableCodePart.functionResponse + +Signature: + +```typescript +functionResponse?: never; +``` + +## ExecutableCodePart.inlineData + +Signature: + +```typescript +inlineData?: never; +``` + +## ExecutableCodePart.text + +Signature: + +```typescript +text?: never; +``` + +## ExecutableCodePart.thought + +Signature: + +```typescript +thought?: never; +``` diff --git a/docs-devsite/ai.filedatapart.md b/docs-devsite/ai.filedatapart.md index 2b5179319f7..f031988a993 100644 --- a/docs-devsite/ai.filedatapart.md +++ b/docs-devsite/ai.filedatapart.md @@ -22,6 +22,8 @@ export interface FileDataPart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.filedatapart.md#filedatapartcodeexecutionresult) | never | | +| [executableCode](./ai.filedatapart.md#filedatapartexecutablecode) | never | | | [fileData](./ai.filedatapart.md#filedatapartfiledata) | [FileData](./ai.filedata.md#filedata_interface) | | | [functionCall](./ai.filedatapart.md#filedatapartfunctioncall) | never | | | [functionResponse](./ai.filedatapart.md#filedatapartfunctionresponse) | never | | @@ -29,6 +31,22 @@ export interface FileDataPart | [text](./ai.filedatapart.md#filedataparttext) | never | | | [thought](./ai.filedatapart.md#filedatapartthought) | boolean | | +## FileDataPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## FileDataPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## FileDataPart.fileData Signature: diff --git a/docs-devsite/ai.functioncallpart.md b/docs-devsite/ai.functioncallpart.md index 3f07c5d0d74..cb9d8b89cf2 100644 --- a/docs-devsite/ai.functioncallpart.md +++ b/docs-devsite/ai.functioncallpart.md @@ -22,12 +22,30 @@ export interface FunctionCallPart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.functioncallpart.md#functioncallpartcodeexecutionresult) | never | | +| [executableCode](./ai.functioncallpart.md#functioncallpartexecutablecode) | never | | | [functionCall](./ai.functioncallpart.md#functioncallpartfunctioncall) | [FunctionCall](./ai.functioncall.md#functioncall_interface) | | | [functionResponse](./ai.functioncallpart.md#functioncallpartfunctionresponse) | never | | | [inlineData](./ai.functioncallpart.md#functioncallpartinlinedata) | never | | | [text](./ai.functioncallpart.md#functioncallparttext) | never | | | [thought](./ai.functioncallpart.md#functioncallpartthought) | boolean | | +## FunctionCallPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## FunctionCallPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## FunctionCallPart.functionCall Signature: diff --git a/docs-devsite/ai.functionresponsepart.md b/docs-devsite/ai.functionresponsepart.md index 4e8c9ea5724..2d80e1706a3 100644 --- a/docs-devsite/ai.functionresponsepart.md +++ b/docs-devsite/ai.functionresponsepart.md @@ -22,12 +22,30 @@ export interface FunctionResponsePart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.functionresponsepart.md#functionresponsepartcodeexecutionresult) | never | | +| [executableCode](./ai.functionresponsepart.md#functionresponsepartexecutablecode) | never | | | [functionCall](./ai.functionresponsepart.md#functionresponsepartfunctioncall) | never | | | [functionResponse](./ai.functionresponsepart.md#functionresponsepartfunctionresponse) | [FunctionResponse](./ai.functionresponse.md#functionresponse_interface) | | | [inlineData](./ai.functionresponsepart.md#functionresponsepartinlinedata) | never | | | [text](./ai.functionresponsepart.md#functionresponseparttext) | never | | | [thought](./ai.functionresponsepart.md#functionresponsepartthought) | boolean | | +## FunctionResponsePart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## FunctionResponsePart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## FunctionResponsePart.functionCall Signature: diff --git a/docs-devsite/ai.googlesearchtool.md b/docs-devsite/ai.googlesearchtool.md index 4e446b0cdea..ef29ccbdaa0 100644 --- a/docs-devsite/ai.googlesearchtool.md +++ b/docs-devsite/ai.googlesearchtool.md @@ -24,11 +24,11 @@ export interface GoogleSearchTool | Property | Type | Description | | --- | --- | --- | -| [googleSearch](./ai.googlesearchtool.md#googlesearchtoolgooglesearch) | [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options.When using this feature, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | +| [googleSearch](./ai.googlesearchtool.md#googlesearchtoolgooglesearch) | [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options.When using this feature, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | ## GoogleSearchTool.googleSearch -Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. +Specifies the Google Search configuration. Currently, this is an empty object, but it's reserved for future configuration options. When using this feature, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). diff --git a/docs-devsite/ai.inlinedatapart.md b/docs-devsite/ai.inlinedatapart.md index c9ead9d061d..a3d581861e7 100644 --- a/docs-devsite/ai.inlinedatapart.md +++ b/docs-devsite/ai.inlinedatapart.md @@ -22,6 +22,8 @@ export interface InlineDataPart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.inlinedatapart.md#inlinedatapartcodeexecutionresult) | never | | +| [executableCode](./ai.inlinedatapart.md#inlinedatapartexecutablecode) | never | | | [functionCall](./ai.inlinedatapart.md#inlinedatapartfunctioncall) | never | | | [functionResponse](./ai.inlinedatapart.md#inlinedatapartfunctionresponse) | never | | | [inlineData](./ai.inlinedatapart.md#inlinedatapartinlinedata) | [GenerativeContentBlob](./ai.generativecontentblob.md#generativecontentblob_interface) | | @@ -29,6 +31,22 @@ export interface InlineDataPart | [thought](./ai.inlinedatapart.md#inlinedatapartthought) | boolean | | | [videoMetadata](./ai.inlinedatapart.md#inlinedatapartvideometadata) | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Applicable if inlineData is a video. | +## InlineDataPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## InlineDataPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## InlineDataPart.functionCall Signature: diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index d94c31155de..e4e382256b3 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -60,6 +60,9 @@ The Firebase AI Web SDK. | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | (EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device inference is possible.These methods should not be called directly by the user. | | [Citation](./ai.citation.md#citation_interface) | A single citation. | | [CitationMetadata](./ai.citationmetadata.md#citationmetadata_interface) | Citation metadata that may be found on a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface). | +| [CodeExecutionResult](./ai.codeexecutionresult.md#codeexecutionresult_interface) | The results of code execution run by the model. | +| [CodeExecutionResultPart](./ai.codeexecutionresultpart.md#codeexecutionresultpart_interface) | Represents the code execution result from the model. | +| [CodeExecutionTool](./ai.codeexecutiontool.md#codeexecutiontool_interface) | A tool that enables the model to use code execution. | | [Content](./ai.content.md#content_interface) | Content type for both prompts and response candidates. | | [CountTokensRequest](./ai.counttokensrequest.md#counttokensrequest_interface) | Params for calling [GenerativeModel.countTokens()](./ai.generativemodel.md#generativemodelcounttokens) | | [CountTokensResponse](./ai.counttokensresponse.md#counttokensresponse_interface) | Response from calling [GenerativeModel.countTokens()](./ai.generativemodel.md#generativemodelcounttokens). | @@ -67,6 +70,8 @@ The Firebase AI Web SDK. | [Date\_2](./ai.date_2.md#date_2_interface) | Protobuf google.type.Date | | [EnhancedGenerateContentResponse](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponse_interface) | Response object wrapped with helper methods. | | [ErrorDetails](./ai.errordetails.md#errordetails_interface) | Details object that may be included in an error response. | +| [ExecutableCode](./ai.executablecode.md#executablecode_interface) | An interface for executable code returned by the model. | +| [ExecutableCodePart](./ai.executablecodepart.md#executablecodepart_interface) | Represents the code that is executed by the model. | | [FileData](./ai.filedata.md#filedata_interface) | Data pointing to a file uploaded on Google Cloud Storage. | | [FileDataPart](./ai.filedatapart.md#filedatapart_interface) | Content part interface if the part represents [FileData](./ai.filedata.md#filedata_interface) | | [FunctionCall](./ai.functioncall.md#functioncall_interface) | A predicted [FunctionCall](./ai.functioncall.md#functioncall_interface) returned from the model that contains a string representing the [FunctionDeclaration.name](./ai.functiondeclaration.md#functiondeclarationname) and a structured JSON object containing the parameters and their values. | @@ -153,8 +158,10 @@ The Firebase AI Web SDK. | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | | [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | +| [Language](./ai.md#language) | The programming language of the code. | | [LiveResponseType](./ai.md#liveresponsetype) | (Public Preview) The types of responses that can be returned by [LiveSession.receive()](./ai.livesession.md#livesessionreceive). | | [Modality](./ai.md#modality) | Content part modality. | +| [Outcome](./ai.md#outcome) | Represents the result of the code execution. | | [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | @@ -177,11 +184,13 @@ The Firebase AI Web SDK. | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | | [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | +| [Language](./ai.md#language) | The programming language of the code. | | [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | (EXPERIMENTAL) Content formats that can be provided as on-device message content. | | [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | (EXPERIMENTAL) Allowable roles for on-device language model usage. | | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | (EXPERIMENTAL) Allowable types for on-device language model messages. | | [LiveResponseType](./ai.md#liveresponsetype) | (Public Preview) The types of responses that can be returned by [LiveSession.receive()](./ai.livesession.md#livesessionreceive). This is a property on all messages that can be used for type narrowing. This property is not returned by the server, it is assigned to a server message object once it's parsed. | | [Modality](./ai.md#modality) | Content part modality. | +| [Outcome](./ai.md#outcome) | Represents the result of the code execution. | | [Part](./ai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [Role](./ai.md#role) | Role is the producer of the content. | @@ -637,6 +646,19 @@ InferenceMode: { } ``` +## Language + +The programming language of the code. + +Signature: + +```typescript +Language: { + UNSPECIFIED: string; + PYTHON: string; +} +``` + ## LiveResponseType > This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. @@ -671,6 +693,21 @@ Modality: { } ``` +## Outcome + +Represents the result of the code execution. + +Signature: + +```typescript +Outcome: { + UNSPECIFIED: string; + OK: string; + FAILED: string; + DEADLINE_EXCEEDED: string; +} +``` + ## POSSIBLE\_ROLES Possible roles. @@ -871,6 +908,16 @@ export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; ``` +## Language + +The programming language of the code. + +Signature: + +```typescript +export type Language = (typeof Language)[keyof typeof Language]; +``` + ## LanguageModelMessageContentValue (EXPERIMENTAL) Content formats that can be provided as on-device message content. @@ -924,6 +971,16 @@ Content part modality. export type Modality = (typeof Modality)[keyof typeof Modality]; ``` +## Outcome + +Represents the result of the code execution. + +Signature: + +```typescript +export type Outcome = (typeof Outcome)[keyof typeof Outcome]; +``` + ## Part Content part - includes text, image/video, or function call/response part types. @@ -931,7 +988,7 @@ Content part - includes text, image/video, or function call/response part types. Signature: ```typescript -export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; +export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart | ExecutableCodePart | CodeExecutionResultPart; ``` ## ResponseModality @@ -974,7 +1031,7 @@ Defines a tool that model can call to access external knowledge. Signature: ```typescript -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool; ``` ## TypedSchema diff --git a/docs-devsite/ai.textpart.md b/docs-devsite/ai.textpart.md index 2466f9cca8f..22236b37d7d 100644 --- a/docs-devsite/ai.textpart.md +++ b/docs-devsite/ai.textpart.md @@ -22,12 +22,30 @@ export interface TextPart | Property | Type | Description | | --- | --- | --- | +| [codeExecutionResult](./ai.textpart.md#textpartcodeexecutionresult) | never | | +| [executableCode](./ai.textpart.md#textpartexecutablecode) | never | | | [functionCall](./ai.textpart.md#textpartfunctioncall) | never | | | [functionResponse](./ai.textpart.md#textpartfunctionresponse) | never | | | [inlineData](./ai.textpart.md#textpartinlinedata) | never | | | [text](./ai.textpart.md#textparttext) | string | | | [thought](./ai.textpart.md#textpartthought) | boolean | | +## TextPart.codeExecutionResult + +Signature: + +```typescript +codeExecutionResult?: never; +``` + +## TextPart.executableCode + +Signature: + +```typescript +executableCode?: never; +``` + ## TextPart.functionCall Signature: diff --git a/docs-devsite/index.md b/docs-devsite/index.md index 47ec0be16ba..a52937d582c 100644 --- a/docs-devsite/index.md +++ b/docs-devsite/index.md @@ -28,4 +28,5 @@ https://github.com/firebase/firebase-js-sdk | [@firebase/performance](./performance.md#performance_package) | The Firebase Performance Monitoring Web SDK. This SDK does not work in a Node.js environment. | | [@firebase/remote-config](./remote-config.md#remote-config_package) | The Firebase Remote Config Web SDK. This SDK does not work in a Node.js environment. | | [@firebase/storage](./storage.md#storage_package) | Cloud Storage for Firebase | +| [@firebase/telemetry](./telemetry.md#telemetry_package) | | diff --git a/docs-devsite/telemetry.instrumentation.md b/docs-devsite/telemetry.instrumentation.md new file mode 100644 index 00000000000..2e2be64711e --- /dev/null +++ b/docs-devsite/telemetry.instrumentation.md @@ -0,0 +1,31 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# Instrumentation namespace +Signature: + +```typescript +export declare namespace Instrumentation +``` + +## Type Aliases + +| Type Alias | Description | +| --- | --- | +| [onRequestError](./telemetry.instrumentation.md#instrumentationonrequesterror) | | + +## Instrumentation.onRequestError + +Signature: + +```typescript +type onRequestError = InstrumentationOnRequestError; +``` diff --git a/docs-devsite/telemetry.md b/docs-devsite/telemetry.md new file mode 100644 index 00000000000..fa62b62e7b8 --- /dev/null +++ b/docs-devsite/telemetry.md @@ -0,0 +1,138 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# telemetry package + +## Functions + +| Function | Description | +| --- | --- | +| function(app, ...) | +| [getTelemetry(app)](./telemetry.md#gettelemetry_cf608e1) | Returns the default [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance that is associated with the provided [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface). If no instance exists, initializes a new instance with the default settings. | +| function(telemetry, ...) | +| [captureError(telemetry, error, attributes)](./telemetry.md#captureerror_862e6b3) | Enqueues an error to be uploaded to the Firebase Telemetry API. | +| [flush(telemetry)](./telemetry.md#flush_8975134) | Flushes all enqueued telemetry data immediately, instead of waiting for default batching. | + +## Interfaces + +| Interface | Description | +| --- | --- | +| [Telemetry](./telemetry.telemetry.md#telemetry_interface) | An instance of the Firebase Telemetry SDK.Do not create this instance directly. Instead, use [getTelemetry()](./telemetry.md#gettelemetry_cf608e1). | + +## Namespaces + +| Namespace | Description | +| --- | --- | +| [Instrumentation](./telemetry.instrumentation.md#instrumentation_namespace) | | + +## Variables + +| Variable | Description | +| --- | --- | +| [nextOnRequestError](./telemetry.md#nextonrequesterror) | Automatically report uncaught errors from server routes to Firebase Telemetry. | + +## function(app, ...) + +### getTelemetry(app) {:#gettelemetry_cf608e1} + +Returns the default [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance that is associated with the provided [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface). If no instance exists, initializes a new instance with the default settings. + +Signature: + +```typescript +export declare function getTelemetry(app?: FirebaseApp): Telemetry; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| app | [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) | The [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) to use. | + +Returns: + +[Telemetry](./telemetry.telemetry.md#telemetry_interface) + +The default [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance for the given [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface). + +### Example + + +```javascript +const telemetry = getTelemetry(app); + +``` + +## function(telemetry, ...) + +### captureError(telemetry, error, attributes) {:#captureerror_862e6b3} + +Enqueues an error to be uploaded to the Firebase Telemetry API. + +Signature: + +```typescript +export declare function captureError(telemetry: Telemetry, error: unknown, attributes?: AnyValueMap): void; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| telemetry | [Telemetry](./telemetry.telemetry.md#telemetry_interface) | The [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance. | +| error | unknown | The caught exception, typically an | +| attributes | AnyValueMap | = Optional, arbitrary attributes to attach to the error log | + +Returns: + +void + +### flush(telemetry) {:#flush_8975134} + +Flushes all enqueued telemetry data immediately, instead of waiting for default batching. + +Signature: + +```typescript +export declare function flush(telemetry: Telemetry): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| telemetry | [Telemetry](./telemetry.telemetry.md#telemetry_interface) | The [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance. | + +Returns: + +Promise<void> + +a promise which is resolved when all flushes are complete + +## nextOnRequestError + +Automatically report uncaught errors from server routes to Firebase Telemetry. + +Signature: + +```typescript +nextOnRequestError: Instrumentation.onRequestError +``` + +### Example + + +```javascript +// In instrumentation.ts (https://nextjs.org/docs/app/guides/instrumentation): +export { nextOnRequestError as onRequestError } from 'firebase/telemetry' + +``` + diff --git a/docs-devsite/telemetry.telemetry.md b/docs-devsite/telemetry.telemetry.md new file mode 100644 index 00000000000..3e7396f79d6 --- /dev/null +++ b/docs-devsite/telemetry.telemetry.md @@ -0,0 +1,48 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# Telemetry interface +An instance of the Firebase Telemetry SDK. + +Do not create this instance directly. Instead, use [getTelemetry()](./telemetry.md#gettelemetry_cf608e1). + +Signature: + +```typescript +export interface Telemetry +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [app](./telemetry.telemetry.md#telemetryapp) | [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) | The [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) this [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance is associated with. | +| [loggerProvider](./telemetry.telemetry.md#telemetryloggerprovider) | LoggerProvider | The this [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance uses. | + +## Telemetry.app + +The [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) this [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance is associated with. + +Signature: + +```typescript +app: FirebaseApp; +``` + +## Telemetry.loggerProvider + +The this [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance uses. + +Signature: + +```typescript +loggerProvider: LoggerProvider; +``` diff --git a/packages/ai/src/methods/chat-session-helpers.ts b/packages/ai/src/methods/chat-session-helpers.ts index 709f616f3c0..3c0c58b7bf5 100644 --- a/packages/ai/src/methods/chat-session-helpers.ts +++ b/packages/ai/src/methods/chat-session-helpers.ts @@ -84,7 +84,9 @@ export function validateChatHistory(history: Content[]): void { functionCall: 0, functionResponse: 0, thought: 0, - thoughtSignature: 0 + thoughtSignature: 0, + executableCode: 0, + codeExecutionResult: 0 }; for (const part of parts) { diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index 19c0761949a..e9298c75349 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -28,7 +28,9 @@ import { HarmBlockMethod, HarmBlockThreshold, HarmCategory, - InferenceMode + InferenceMode, + Language, + Outcome } from '../types'; import { ApiSettings } from '../types/internal'; import { Task } from '../requests/request'; @@ -248,6 +250,26 @@ describe('generateContent()', () => { match.any ); }); + it('codeExecution', async () => { + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-code-execution.json' + ); + stub(request, 'makeRequest').resolves(mockResponse as Response); + const result = await generateContent( + fakeApiSettings, + 'model', + fakeRequestParams + ); + const parts = result.response.candidates?.[0].content.parts; + parts?.forEach(part => console.log(Object.keys(part))); + expect( + parts?.some(part => part.codeExecutionResult?.outcome === Outcome.OK) + ).to.be.true; + expect( + parts?.some(part => part.executableCode?.language === Language.PYTHON) + ).to.be.true; + }); it('blocked prompt', async () => { const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/models/generative-model.test.ts b/packages/ai/src/models/generative-model.test.ts index 9c5074b8eb3..bcd78d746d4 100644 --- a/packages/ai/src/models/generative-model.test.ts +++ b/packages/ai/src/models/generative-model.test.ts @@ -71,7 +71,9 @@ describe('GenerativeModel', () => { description: 'mydesc' } ] - } + }, + { googleSearch: {} }, + { codeExecution: {} } ], toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } @@ -81,7 +83,7 @@ describe('GenerativeModel', () => { {}, fakeChromeAdapter ); - expect(genModel.tools?.length).to.equal(1); + expect(genModel.tools?.length).to.equal(3); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE ); @@ -102,6 +104,8 @@ describe('GenerativeModel', () => { match((value: string) => { return ( value.includes('myfunc') && + value.includes('googleSearch') && + value.includes('codeExecution') && value.includes(FunctionCallingMode.NONE) && value.includes('be friendly') ); @@ -183,7 +187,9 @@ describe('GenerativeModel', () => { functionDeclarations: [ { name: 'otherfunc', description: 'otherdesc' } ] - } + }, + { googleSearch: {} }, + { codeExecution: {} } ], toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.AUTO } }, systemInstruction: { role: 'system', parts: [{ text: 'be formal' }] } @@ -196,6 +202,8 @@ describe('GenerativeModel', () => { match((value: string) => { return ( value.includes('otherfunc') && + value.includes('googleSearch') && + value.includes('codeExecution') && value.includes(FunctionCallingMode.AUTO) && value.includes('be formal') ); @@ -249,7 +257,9 @@ describe('GenerativeModel', () => { { model: 'my-model', tools: [ - { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } + { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] }, + { googleSearch: {} }, + { codeExecution: {} } ], toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } @@ -262,7 +272,7 @@ describe('GenerativeModel', () => { {}, fakeChromeAdapter ); - expect(genModel.tools?.length).to.equal(1); + expect(genModel.tools?.length).to.equal(3); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE ); @@ -283,6 +293,8 @@ describe('GenerativeModel', () => { match((value: string) => { return ( value.includes('myfunc') && + value.includes('googleSearch') && + value.includes('codeExecution') && value.includes(FunctionCallingMode.NONE) && value.includes('be friendly') && value.includes('topK') @@ -361,7 +373,9 @@ describe('GenerativeModel', () => { functionDeclarations: [ { name: 'otherfunc', description: 'otherdesc' } ] - } + }, + { googleSearch: {} }, + { codeExecution: {} } ], toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.AUTO } @@ -380,6 +394,8 @@ describe('GenerativeModel', () => { match((value: string) => { return ( value.includes('otherfunc') && + value.includes('googleSearch') && + value.includes('codeExecution') && value.includes(FunctionCallingMode.AUTO) && value.includes('be formal') && value.includes('image/png') && diff --git a/packages/ai/src/types/content.ts b/packages/ai/src/types/content.ts index bd8ccf1ef10..56a159e30ce 100644 --- a/packages/ai/src/types/content.ts +++ b/packages/ai/src/types/content.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { Role } from './enums'; +import { Language, Outcome, Role } from './enums'; /** * Content type for both prompts and response candidates. @@ -36,7 +36,9 @@ export type Part = | InlineDataPart | FunctionCallPart | FunctionResponsePart - | FileDataPart; + | FileDataPart + | ExecutableCodePart + | CodeExecutionResultPart; /** * Content part interface if the part represents a text string. @@ -52,6 +54,8 @@ export interface TextPart { * @internal */ thoughtSignature?: string; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -72,6 +76,8 @@ export interface InlineDataPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -105,6 +111,8 @@ export interface FunctionCallPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -121,6 +129,8 @@ export interface FunctionResponsePart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; } /** @@ -138,6 +148,81 @@ export interface FileDataPart { * @internal */ thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: never; +} + +/** + * Represents the code that is executed by the model. + * + * @public + */ +export interface ExecutableCodePart { + text?: never; + inlineData?: never; + functionCall?: never; + functionResponse?: never; + fileData: never; + thought?: never; + /** + * @internal + */ + thoughtSignature?: never; + executableCode?: ExecutableCode; + codeExecutionResult?: never; +} + +/** + * Represents the code execution result from the model. + * + * @public + */ +export interface CodeExecutionResultPart { + text?: never; + inlineData?: never; + functionCall?: never; + functionResponse?: never; + fileData: never; + thought?: never; + /** + * @internal + */ + thoughtSignature?: never; + executableCode?: never; + codeExecutionResult?: CodeExecutionResult; +} + +/** + * An interface for executable code returned by the model. + * + * @public + */ +export interface ExecutableCode { + /** + * The programming language of the code. + */ + language?: Language; + /** + * The source code to be executed. + */ + code?: string; +} + +/** + * The results of code execution run by the model. + * + * @public + */ +export interface CodeExecutionResult { + /** + * The result of the execution. + */ + outcome?: Outcome; + /** + * The output from the code execution, or an error message + * if it failed. + */ + output?: string; } /** diff --git a/packages/ai/src/types/enums.ts b/packages/ai/src/types/enums.ts index de70d325157..177dde296b7 100644 --- a/packages/ai/src/types/enums.ts +++ b/packages/ai/src/types/enums.ts @@ -380,3 +380,39 @@ export const InferenceMode = { * @public */ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; + +/** + * Represents the result of the code execution. + * + * @public + */ +export const Outcome = { + UNSPECIFIED: 'OUTCOME_UNSPECIFIED', + OK: 'OUTCOME_OK', + FAILED: 'OUTCOME_FAILED', + DEADLINE_EXCEEDED: 'OUTCOME_DEADLINE_EXCEEDED' +}; + +/** + * Represents the result of the code execution. + * + * @public + */ +export type Outcome = (typeof Outcome)[keyof typeof Outcome]; + +/** + * The programming language of the code. + * + * @public + */ +export const Language = { + UNSPECIFIED: 'LANGUAGE_UNSPECIFIED', + PYTHON: 'PYTHON' +}; + +/** + * The programming language of the code. + * + * @public + */ +export type Language = (typeof Language)[keyof typeof Language]; diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 21ccce8bd11..0ce87d0c8da 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -239,7 +239,10 @@ export interface RequestOptions { * Defines a tool that model can call to access external knowledge. * @public */ -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = + | FunctionDeclarationsTool + | GoogleSearchTool + | CodeExecutionTool; /** * Structured representation of a function declaration as defined by the @@ -285,8 +288,6 @@ export interface GoogleSearchTool { /** * Specifies the Google Search configuration. * Currently, this is an empty object, but it's reserved for future configuration options. - * Specifies the Google Search configuration. Currently, this is an empty object, but it's - * reserved for future configuration options. * * When using this feature, you are required to comply with the "Grounding with Google Search" * usage requirements for your chosen API provider: {@link https://ai.google.dev/gemini-api/terms#grounding-with-google-search | Gemini Developer API} @@ -296,6 +297,19 @@ export interface GoogleSearchTool { googleSearch: GoogleSearch; } +/** + * A tool that enables the model to use code execution. + * + * @public + */ +export interface CodeExecutionTool { + /** + * Specifies the Google Search configuration. + * Currently, this is an empty object, but it's reserved for future configuration options. + */ + codeExecution: {}; +} + /** * Specifies the Google Search configuration. * From 2fddba2ff558a24a07bcdd39e2cda4ce6137adc9 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Fri, 12 Sep 2025 08:29:30 -0700 Subject: [PATCH 2/3] rm telemetry files --- .changeset/lucky-socks-roll.md | 2 +- docs-devsite/_toc.yaml | 5 - docs-devsite/ai.codeexecutionresult.md | 4 +- docs-devsite/index.md | 1 - docs-devsite/telemetry.instrumentation.md | 31 ----- docs-devsite/telemetry.md | 138 ---------------------- docs-devsite/telemetry.telemetry.md | 48 -------- packages/ai/src/types/content.ts | 2 +- 8 files changed, 4 insertions(+), 227 deletions(-) delete mode 100644 docs-devsite/telemetry.instrumentation.md delete mode 100644 docs-devsite/telemetry.md delete mode 100644 docs-devsite/telemetry.telemetry.md diff --git a/.changeset/lucky-socks-roll.md b/.changeset/lucky-socks-roll.md index 2e89ebf0769..8b9d9a19e97 100644 --- a/.changeset/lucky-socks-roll.md +++ b/.changeset/lucky-socks-roll.md @@ -3,4 +3,4 @@ 'firebase': minor --- -Add Code Execution feature. +Added Code Execution feature. diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index d86a97079de..8bec927e163 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -682,8 +682,3 @@ toc: path: /docs/reference/js/storage.uploadtask.md - title: UploadTaskSnapshot path: /docs/reference/js/storage.uploadtasksnapshot.md -- title: telemetry - path: /docs/reference/js/telemetry.md - section: - - title: Telemetry - path: /docs/reference/js/telemetry.telemetry.md diff --git a/docs-devsite/ai.codeexecutionresult.md b/docs-devsite/ai.codeexecutionresult.md index 31587a4d256..d0be155c4e6 100644 --- a/docs-devsite/ai.codeexecutionresult.md +++ b/docs-devsite/ai.codeexecutionresult.md @@ -22,12 +22,12 @@ export interface CodeExecutionResult | Property | Type | Description | | --- | --- | --- | -| [outcome](./ai.codeexecutionresult.md#codeexecutionresultoutcome) | [Outcome](./ai.md#outcome) | The result of the execution. | +| [outcome](./ai.codeexecutionresult.md#codeexecutionresultoutcome) | [Outcome](./ai.md#outcome) | The result of the code execution. | | [output](./ai.codeexecutionresult.md#codeexecutionresultoutput) | string | The output from the code execution, or an error message if it failed. | ## CodeExecutionResult.outcome -The result of the execution. +The result of the code execution. Signature: diff --git a/docs-devsite/index.md b/docs-devsite/index.md index a52937d582c..47ec0be16ba 100644 --- a/docs-devsite/index.md +++ b/docs-devsite/index.md @@ -28,5 +28,4 @@ https://github.com/firebase/firebase-js-sdk | [@firebase/performance](./performance.md#performance_package) | The Firebase Performance Monitoring Web SDK. This SDK does not work in a Node.js environment. | | [@firebase/remote-config](./remote-config.md#remote-config_package) | The Firebase Remote Config Web SDK. This SDK does not work in a Node.js environment. | | [@firebase/storage](./storage.md#storage_package) | Cloud Storage for Firebase | -| [@firebase/telemetry](./telemetry.md#telemetry_package) | | diff --git a/docs-devsite/telemetry.instrumentation.md b/docs-devsite/telemetry.instrumentation.md deleted file mode 100644 index 2e2be64711e..00000000000 --- a/docs-devsite/telemetry.instrumentation.md +++ /dev/null @@ -1,31 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# Instrumentation namespace -Signature: - -```typescript -export declare namespace Instrumentation -``` - -## Type Aliases - -| Type Alias | Description | -| --- | --- | -| [onRequestError](./telemetry.instrumentation.md#instrumentationonrequesterror) | | - -## Instrumentation.onRequestError - -Signature: - -```typescript -type onRequestError = InstrumentationOnRequestError; -``` diff --git a/docs-devsite/telemetry.md b/docs-devsite/telemetry.md deleted file mode 100644 index fa62b62e7b8..00000000000 --- a/docs-devsite/telemetry.md +++ /dev/null @@ -1,138 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# telemetry package - -## Functions - -| Function | Description | -| --- | --- | -| function(app, ...) | -| [getTelemetry(app)](./telemetry.md#gettelemetry_cf608e1) | Returns the default [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance that is associated with the provided [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface). If no instance exists, initializes a new instance with the default settings. | -| function(telemetry, ...) | -| [captureError(telemetry, error, attributes)](./telemetry.md#captureerror_862e6b3) | Enqueues an error to be uploaded to the Firebase Telemetry API. | -| [flush(telemetry)](./telemetry.md#flush_8975134) | Flushes all enqueued telemetry data immediately, instead of waiting for default batching. | - -## Interfaces - -| Interface | Description | -| --- | --- | -| [Telemetry](./telemetry.telemetry.md#telemetry_interface) | An instance of the Firebase Telemetry SDK.Do not create this instance directly. Instead, use [getTelemetry()](./telemetry.md#gettelemetry_cf608e1). | - -## Namespaces - -| Namespace | Description | -| --- | --- | -| [Instrumentation](./telemetry.instrumentation.md#instrumentation_namespace) | | - -## Variables - -| Variable | Description | -| --- | --- | -| [nextOnRequestError](./telemetry.md#nextonrequesterror) | Automatically report uncaught errors from server routes to Firebase Telemetry. | - -## function(app, ...) - -### getTelemetry(app) {:#gettelemetry_cf608e1} - -Returns the default [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance that is associated with the provided [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface). If no instance exists, initializes a new instance with the default settings. - -Signature: - -```typescript -export declare function getTelemetry(app?: FirebaseApp): Telemetry; -``` - -#### Parameters - -| Parameter | Type | Description | -| --- | --- | --- | -| app | [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) | The [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) to use. | - -Returns: - -[Telemetry](./telemetry.telemetry.md#telemetry_interface) - -The default [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance for the given [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface). - -### Example - - -```javascript -const telemetry = getTelemetry(app); - -``` - -## function(telemetry, ...) - -### captureError(telemetry, error, attributes) {:#captureerror_862e6b3} - -Enqueues an error to be uploaded to the Firebase Telemetry API. - -Signature: - -```typescript -export declare function captureError(telemetry: Telemetry, error: unknown, attributes?: AnyValueMap): void; -``` - -#### Parameters - -| Parameter | Type | Description | -| --- | --- | --- | -| telemetry | [Telemetry](./telemetry.telemetry.md#telemetry_interface) | The [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance. | -| error | unknown | The caught exception, typically an | -| attributes | AnyValueMap | = Optional, arbitrary attributes to attach to the error log | - -Returns: - -void - -### flush(telemetry) {:#flush_8975134} - -Flushes all enqueued telemetry data immediately, instead of waiting for default batching. - -Signature: - -```typescript -export declare function flush(telemetry: Telemetry): Promise; -``` - -#### Parameters - -| Parameter | Type | Description | -| --- | --- | --- | -| telemetry | [Telemetry](./telemetry.telemetry.md#telemetry_interface) | The [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance. | - -Returns: - -Promise<void> - -a promise which is resolved when all flushes are complete - -## nextOnRequestError - -Automatically report uncaught errors from server routes to Firebase Telemetry. - -Signature: - -```typescript -nextOnRequestError: Instrumentation.onRequestError -``` - -### Example - - -```javascript -// In instrumentation.ts (https://nextjs.org/docs/app/guides/instrumentation): -export { nextOnRequestError as onRequestError } from 'firebase/telemetry' - -``` - diff --git a/docs-devsite/telemetry.telemetry.md b/docs-devsite/telemetry.telemetry.md deleted file mode 100644 index 3e7396f79d6..00000000000 --- a/docs-devsite/telemetry.telemetry.md +++ /dev/null @@ -1,48 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# Telemetry interface -An instance of the Firebase Telemetry SDK. - -Do not create this instance directly. Instead, use [getTelemetry()](./telemetry.md#gettelemetry_cf608e1). - -Signature: - -```typescript -export interface Telemetry -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [app](./telemetry.telemetry.md#telemetryapp) | [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) | The [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) this [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance is associated with. | -| [loggerProvider](./telemetry.telemetry.md#telemetryloggerprovider) | LoggerProvider | The this [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance uses. | - -## Telemetry.app - -The [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface) this [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance is associated with. - -Signature: - -```typescript -app: FirebaseApp; -``` - -## Telemetry.loggerProvider - -The this [Telemetry](./telemetry.telemetry.md#telemetry_interface) instance uses. - -Signature: - -```typescript -loggerProvider: LoggerProvider; -``` diff --git a/packages/ai/src/types/content.ts b/packages/ai/src/types/content.ts index 56a159e30ce..b51dcd9c69f 100644 --- a/packages/ai/src/types/content.ts +++ b/packages/ai/src/types/content.ts @@ -215,7 +215,7 @@ export interface ExecutableCode { */ export interface CodeExecutionResult { /** - * The result of the execution. + * The result of the code execution. */ outcome?: Outcome; /** From 33bd2572b272f995862ce0741654183f0d94729c Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Fri, 12 Sep 2025 09:54:17 -0700 Subject: [PATCH 3/3] Add integration test --- .../ai/integration/generate-content.test.ts | 28 +++++++++++++++++++ .../ai/src/methods/generate-content.test.ts | 1 - 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 0b83df38ecb..a827a447d90 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -21,7 +21,9 @@ import { GenerationConfig, HarmBlockThreshold, HarmCategory, + Language, Modality, + Outcome, SafetySetting, getGenerativeModel } from '../src'; @@ -188,6 +190,32 @@ describe('Generate Content', () => { }); }); + it('generateContent: code execution', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + tools: [{ codeExecution: {} }] + }); + const prompt = + 'What is the sum of the first 50 prime numbers? ' + + 'Generate and run code for the calculation, and make sure you get all 50.'; + + const result = await model.generateContent(prompt); + const parts = result.response.candidates?.[0].content.parts; + expect( + parts?.some(part => part.executableCode?.language === Language.PYTHON) + ).to.be.true; + expect( + parts?.some(part => part.codeExecutionResult?.outcome === Outcome.OK) + ).to.be.true; + // Expect these to be truthy (!= null) + expect(parts?.some(part => part.executableCode?.code != null)).to.be + .true; + expect(parts?.some(part => part.codeExecutionResult?.output != null)).to + .be.true; + }); + it('generateContentStream: text input, text output', async () => { const model = getGenerativeModel(testConfig.ai, { model: testConfig.model, diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index e9298c75349..3bb396ac6d8 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -262,7 +262,6 @@ describe('generateContent()', () => { fakeRequestParams ); const parts = result.response.candidates?.[0].content.parts; - parts?.forEach(part => console.log(Object.keys(part))); expect( parts?.some(part => part.codeExecutionResult?.outcome === Outcome.OK) ).to.be.true;