Skip to content

Commit 64844f1

Browse files
stainless-app[bot]stainless-app[bot]
committed
feat(api): Add tools and structured outputs to evals
1 parent b3d488f commit 64844f1

File tree

2 files changed

+302
-2
lines changed

2 files changed

+302
-2
lines changed

‎.stats.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
configured_endpoints: 111
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-0205acb1015d29b2312a48526734c0399f93026d4fe2dff5c7768f566e333fd2.yml
3-
openapi_spec_hash: 1772cc9056c2f6dfb2a4e9cb77ee6343
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-4865dda2b62927bd141cbc85f81be3d88602f103e2c581e15eb1caded3e3aaa2.yml
3+
openapi_spec_hash: 7d14a9b23ef4ac93ea46d629601b6f6b
44
config_hash: ed1e6b3c5f93d12b80d31167f55c557c

‎src/resources/evals/runs/runs.ts

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import { APIResource } from '../../../core/resource';
44
import * as Shared from '../../shared';
55
import * as ResponsesAPI from '../../responses/responses';
6+
import * as CompletionsAPI from '../../chat/completions/completions';
67
import * as OutputItemsAPI from './output-items';
78
import {
89
OutputItemListParams,
@@ -257,6 +258,23 @@ export namespace CreateEvalCompletionsRunDataSource {
257258
*/
258259
max_completion_tokens?: number;
259260

261+
/**
262+
* An object specifying the format that the model must output.
263+
*
264+
* Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured
265+
* Outputs which ensures the model will match your supplied JSON schema. Learn more
266+
* in the
267+
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
268+
*
269+
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
270+
* ensures the message the model generates is valid JSON. Using `json_schema` is
271+
* preferred for models that support it.
272+
*/
273+
response_format?:
274+
| Shared.ResponseFormatText
275+
| Shared.ResponseFormatJSONSchema
276+
| Shared.ResponseFormatJSONObject;
277+
260278
/**
261279
* A seed value to initialize the randomness, during sampling.
262280
*/
@@ -267,6 +285,13 @@ export namespace CreateEvalCompletionsRunDataSource {
267285
*/
268286
temperature?: number;
269287

288+
/**
289+
* A list of tools the model may call. Currently, only functions are supported as a
290+
* tool. Use this to provide a list of functions the model may generate JSON inputs
291+
* for. A max of 128 functions are supported.
292+
*/
293+
tools?: Array<CompletionsAPI.ChatCompletionTool>;
294+
270295
/**
271296
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
272297
*/
@@ -649,11 +674,66 @@ export namespace RunCreateResponse {
649674
*/
650675
temperature?: number;
651676

677+
/**
678+
* Configuration options for a text response from the model. Can be plain text or
679+
* structured JSON data. Learn more:
680+
*
681+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
682+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
683+
*/
684+
text?: SamplingParams.Text;
685+
686+
/**
687+
* An array of tools the model may call while generating a response. You can
688+
* specify which tool to use by setting the `tool_choice` parameter.
689+
*
690+
* The two categories of tools you can provide the model are:
691+
*
692+
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
693+
* capabilities, like
694+
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
695+
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
696+
* Learn more about
697+
* [built-in tools](https://platform.openai.com/docs/guides/tools).
698+
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
699+
* the model to call your own code. Learn more about
700+
* [function calling](https://platform.openai.com/docs/guides/function-calling).
701+
*/
702+
tools?: Array<ResponsesAPI.Tool>;
703+
652704
/**
653705
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
654706
*/
655707
top_p?: number;
656708
}
709+
710+
export namespace SamplingParams {
711+
/**
712+
* Configuration options for a text response from the model. Can be plain text or
713+
* structured JSON data. Learn more:
714+
*
715+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
716+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
717+
*/
718+
export interface Text {
719+
/**
720+
* An object specifying the format that the model must output.
721+
*
722+
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
723+
* ensures the model will match your supplied JSON schema. Learn more in the
724+
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
725+
*
726+
* The default format is `{ "type": "text" }` with no additional options.
727+
*
728+
* **Not recommended for gpt-4o and newer models:**
729+
*
730+
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
731+
* ensures the message the model generates is valid JSON. Using `json_schema` is
732+
* preferred for models that support it.
733+
*/
734+
format?: ResponsesAPI.ResponseFormatTextConfig;
735+
}
736+
}
657737
}
658738

659739
export interface PerModelUsage {
@@ -1041,11 +1121,66 @@ export namespace RunRetrieveResponse {
10411121
*/
10421122
temperature?: number;
10431123

1124+
/**
1125+
* Configuration options for a text response from the model. Can be plain text or
1126+
* structured JSON data. Learn more:
1127+
*
1128+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
1129+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
1130+
*/
1131+
text?: SamplingParams.Text;
1132+
1133+
/**
1134+
* An array of tools the model may call while generating a response. You can
1135+
* specify which tool to use by setting the `tool_choice` parameter.
1136+
*
1137+
* The two categories of tools you can provide the model are:
1138+
*
1139+
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
1140+
* capabilities, like
1141+
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
1142+
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
1143+
* Learn more about
1144+
* [built-in tools](https://platform.openai.com/docs/guides/tools).
1145+
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
1146+
* the model to call your own code. Learn more about
1147+
* [function calling](https://platform.openai.com/docs/guides/function-calling).
1148+
*/
1149+
tools?: Array<ResponsesAPI.Tool>;
1150+
10441151
/**
10451152
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
10461153
*/
10471154
top_p?: number;
10481155
}
1156+
1157+
export namespace SamplingParams {
1158+
/**
1159+
* Configuration options for a text response from the model. Can be plain text or
1160+
* structured JSON data. Learn more:
1161+
*
1162+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
1163+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
1164+
*/
1165+
export interface Text {
1166+
/**
1167+
* An object specifying the format that the model must output.
1168+
*
1169+
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
1170+
* ensures the model will match your supplied JSON schema. Learn more in the
1171+
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
1172+
*
1173+
* The default format is `{ "type": "text" }` with no additional options.
1174+
*
1175+
* **Not recommended for gpt-4o and newer models:**
1176+
*
1177+
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
1178+
* ensures the message the model generates is valid JSON. Using `json_schema` is
1179+
* preferred for models that support it.
1180+
*/
1181+
format?: ResponsesAPI.ResponseFormatTextConfig;
1182+
}
1183+
}
10491184
}
10501185

10511186
export interface PerModelUsage {
@@ -1430,11 +1565,66 @@ export namespace RunListResponse {
14301565
*/
14311566
temperature?: number;
14321567

1568+
/**
1569+
* Configuration options for a text response from the model. Can be plain text or
1570+
* structured JSON data. Learn more:
1571+
*
1572+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
1573+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
1574+
*/
1575+
text?: SamplingParams.Text;
1576+
1577+
/**
1578+
* An array of tools the model may call while generating a response. You can
1579+
* specify which tool to use by setting the `tool_choice` parameter.
1580+
*
1581+
* The two categories of tools you can provide the model are:
1582+
*
1583+
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
1584+
* capabilities, like
1585+
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
1586+
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
1587+
* Learn more about
1588+
* [built-in tools](https://platform.openai.com/docs/guides/tools).
1589+
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
1590+
* the model to call your own code. Learn more about
1591+
* [function calling](https://platform.openai.com/docs/guides/function-calling).
1592+
*/
1593+
tools?: Array<ResponsesAPI.Tool>;
1594+
14331595
/**
14341596
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
14351597
*/
14361598
top_p?: number;
14371599
}
1600+
1601+
export namespace SamplingParams {
1602+
/**
1603+
* Configuration options for a text response from the model. Can be plain text or
1604+
* structured JSON data. Learn more:
1605+
*
1606+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
1607+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
1608+
*/
1609+
export interface Text {
1610+
/**
1611+
* An object specifying the format that the model must output.
1612+
*
1613+
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
1614+
* ensures the model will match your supplied JSON schema. Learn more in the
1615+
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
1616+
*
1617+
* The default format is `{ "type": "text" }` with no additional options.
1618+
*
1619+
* **Not recommended for gpt-4o and newer models:**
1620+
*
1621+
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
1622+
* ensures the message the model generates is valid JSON. Using `json_schema` is
1623+
* preferred for models that support it.
1624+
*/
1625+
format?: ResponsesAPI.ResponseFormatTextConfig;
1626+
}
1627+
}
14381628
}
14391629

14401630
export interface PerModelUsage {
@@ -1830,11 +2020,66 @@ export namespace RunCancelResponse {
18302020
*/
18312021
temperature?: number;
18322022

2023+
/**
2024+
* Configuration options for a text response from the model. Can be plain text or
2025+
* structured JSON data. Learn more:
2026+
*
2027+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
2028+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
2029+
*/
2030+
text?: SamplingParams.Text;
2031+
2032+
/**
2033+
* An array of tools the model may call while generating a response. You can
2034+
* specify which tool to use by setting the `tool_choice` parameter.
2035+
*
2036+
* The two categories of tools you can provide the model are:
2037+
*
2038+
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
2039+
* capabilities, like
2040+
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
2041+
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
2042+
* Learn more about
2043+
* [built-in tools](https://platform.openai.com/docs/guides/tools).
2044+
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
2045+
* the model to call your own code. Learn more about
2046+
* [function calling](https://platform.openai.com/docs/guides/function-calling).
2047+
*/
2048+
tools?: Array<ResponsesAPI.Tool>;
2049+
18332050
/**
18342051
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
18352052
*/
18362053
top_p?: number;
18372054
}
2055+
2056+
export namespace SamplingParams {
2057+
/**
2058+
* Configuration options for a text response from the model. Can be plain text or
2059+
* structured JSON data. Learn more:
2060+
*
2061+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
2062+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
2063+
*/
2064+
export interface Text {
2065+
/**
2066+
* An object specifying the format that the model must output.
2067+
*
2068+
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
2069+
* ensures the model will match your supplied JSON schema. Learn more in the
2070+
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
2071+
*
2072+
* The default format is `{ "type": "text" }` with no additional options.
2073+
*
2074+
* **Not recommended for gpt-4o and newer models:**
2075+
*
2076+
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
2077+
* ensures the message the model generates is valid JSON. Using `json_schema` is
2078+
* preferred for models that support it.
2079+
*/
2080+
format?: ResponsesAPI.ResponseFormatTextConfig;
2081+
}
2082+
}
18382083
}
18392084

18402085
export interface PerModelUsage {
@@ -2169,11 +2414,66 @@ export namespace RunCreateParams {
21692414
*/
21702415
temperature?: number;
21712416

2417+
/**
2418+
* Configuration options for a text response from the model. Can be plain text or
2419+
* structured JSON data. Learn more:
2420+
*
2421+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
2422+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
2423+
*/
2424+
text?: SamplingParams.Text;
2425+
2426+
/**
2427+
* An array of tools the model may call while generating a response. You can
2428+
* specify which tool to use by setting the `tool_choice` parameter.
2429+
*
2430+
* The two categories of tools you can provide the model are:
2431+
*
2432+
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
2433+
* capabilities, like
2434+
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
2435+
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
2436+
* Learn more about
2437+
* [built-in tools](https://platform.openai.com/docs/guides/tools).
2438+
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
2439+
* the model to call your own code. Learn more about
2440+
* [function calling](https://platform.openai.com/docs/guides/function-calling).
2441+
*/
2442+
tools?: Array<ResponsesAPI.Tool>;
2443+
21722444
/**
21732445
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
21742446
*/
21752447
top_p?: number;
21762448
}
2449+
2450+
export namespace SamplingParams {
2451+
/**
2452+
* Configuration options for a text response from the model. Can be plain text or
2453+
* structured JSON data. Learn more:
2454+
*
2455+
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
2456+
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
2457+
*/
2458+
export interface Text {
2459+
/**
2460+
* An object specifying the format that the model must output.
2461+
*
2462+
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
2463+
* ensures the model will match your supplied JSON schema. Learn more in the
2464+
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
2465+
*
2466+
* The default format is `{ "type": "text" }` with no additional options.
2467+
*
2468+
* **Not recommended for gpt-4o and newer models:**
2469+
*
2470+
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
2471+
* ensures the message the model generates is valid JSON. Using `json_schema` is
2472+
* preferred for models that support it.
2473+
*/
2474+
format?: ResponsesAPI.ResponseFormatTextConfig;
2475+
}
2476+
}
21772477
}
21782478
}
21792479

0 commit comments

Comments
 (0)