Skip to content

Commit ea1d56c

Browse files
feat(api): responses x eval api
1 parent bc9f15f commit ea1d56c

File tree

12 files changed

+375
-1216
lines changed

12 files changed

+375
-1216
lines changed

.stats.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
configured_endpoints: 101
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-794a6ed3c3d3d77887564755168056af8a426b17cf1ec721e3a300503dc22a41.yml
3-
openapi_spec_hash: 25a81c220713cd5b0bafc221d1dfa79a
4-
config_hash: 0b768ed1b56c6d82816f0fa40dc4aaf5
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-161ca7f1cfd7b33c1fc07d0ce25dfe4be5a7271c394f4cb526b7fb21b0729900.yml
3+
openapi_spec_hash: 602e14add4bee018c6774e320ce309b8
4+
config_hash: 7da27f7260075e8813ddcea542fba1bf

api.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -709,6 +709,7 @@ Methods:
709709
Types:
710710

711711
- <code><a href="./src/resources/evals/evals.ts">EvalCustomDataSourceConfig</a></code>
712+
- <code><a href="./src/resources/evals/evals.ts">EvalLogsDataSourceConfig</a></code>
712713
- <code><a href="./src/resources/evals/evals.ts">EvalStoredCompletionsDataSourceConfig</a></code>
713714
- <code><a href="./src/resources/evals/evals.ts">EvalCreateResponse</a></code>
714715
- <code><a href="./src/resources/evals/evals.ts">EvalRetrieveResponse</a></code>
@@ -730,6 +731,7 @@ Types:
730731

731732
- <code><a href="./src/resources/evals/runs/runs.ts">CreateEvalCompletionsRunDataSource</a></code>
732733
- <code><a href="./src/resources/evals/runs/runs.ts">CreateEvalJSONLRunDataSource</a></code>
734+
- <code><a href="./src/resources/evals/runs/runs.ts">CreateEvalResponsesRunDataSource</a></code>
733735
- <code><a href="./src/resources/evals/runs/runs.ts">EvalAPIError</a></code>
734736
- <code><a href="./src/resources/evals/runs/runs.ts">RunCreateResponse</a></code>
735737
- <code><a href="./src/resources/evals/runs/runs.ts">RunRetrieveResponse</a></code>

src/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ import {
7474
EvalListParams,
7575
EvalListResponse,
7676
EvalListResponsesPage,
77+
EvalLogsDataSourceConfig,
7778
EvalRetrieveResponse,
7879
EvalStoredCompletionsDataSourceConfig,
7980
EvalUpdateParams,
@@ -533,6 +534,7 @@ export declare namespace OpenAI {
533534
export {
534535
Evals as Evals,
535536
type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig,
537+
type EvalLogsDataSourceConfig as EvalLogsDataSourceConfig,
536538
type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig,
537539
type EvalCreateResponse as EvalCreateResponse,
538540
type EvalRetrieveResponse as EvalRetrieveResponse,

src/resources/audio/transcriptions.ts

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,14 @@ export interface TranscriptionCreateParamsBase<
324324
*/
325325
model: (string & {}) | AudioAPI.AudioModel;
326326

327+
/**
328+
* Controls how the audio is cut into chunks. When set to `"auto"`, the server
329+
* first normalizes loudness and then uses voice activity detection (VAD) to choose
330+
* boundaries. `server_vad` object can be provided to tweak VAD detection
331+
* parameters manually. If unset, the audio is transcribed as a single block.
332+
*/
333+
chunking_strategy?: 'auto' | TranscriptionCreateParams.VadConfig | null;
334+
327335
/**
328336
* Additional information to include in the transcription response. `logprobs` will
329337
* return the log probabilities of the tokens in the response to understand the
@@ -387,6 +395,32 @@ export interface TranscriptionCreateParamsBase<
387395
}
388396

389397
export namespace TranscriptionCreateParams {
398+
export interface VadConfig {
399+
/**
400+
* Must be set to `server_vad` to enable manual chunking using server side VAD.
401+
*/
402+
type: 'server_vad';
403+
404+
/**
405+
* Amount of audio to include before the VAD detected speech (in milliseconds).
406+
*/
407+
prefix_padding_ms?: number;
408+
409+
/**
410+
* Duration of silence to detect speech stop (in milliseconds). With shorter values
411+
* the model will respond more quickly, but may jump in on short pauses from the
412+
* user.
413+
*/
414+
silence_duration_ms?: number;
415+
416+
/**
417+
* Sensitivity threshold (0.0 to 1.0) for voice activity detection. A higher
418+
* threshold will require louder audio to activate the model, and thus might
419+
* perform better in noisy environments.
420+
*/
421+
threshold?: number;
422+
}
423+
390424
export type TranscriptionCreateParamsNonStreaming = TranscriptionsAPI.TranscriptionCreateParamsNonStreaming;
391425
export type TranscriptionCreateParamsStreaming = TranscriptionsAPI.TranscriptionCreateParamsStreaming;
392426
}

src/resources/embeddings.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,11 +130,12 @@ export interface EmbeddingCreateParams {
130130
* Input text to embed, encoded as a string or array of tokens. To embed multiple
131131
* inputs in a single request, pass an array of strings or array of token arrays.
132132
* The input must not exceed the max input tokens for the model (8192 tokens for
133-
* `text-embedding-ada-002`), cannot be an empty string, and any array must be 2048
133+
* all embedding models), cannot be an empty string, and any array must be 2048
134134
* dimensions or less.
135135
* [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken)
136-
* for counting tokens. Some models may also impose a limit on total number of
137-
* tokens summed across inputs.
136+
* for counting tokens. In addition to the per-input token limit, all embedding
137+
* models enforce a maximum of 300,000 tokens summed across all inputs in a single
138+
* request.
138139
*/
139140
input: string | Array<string> | Array<number> | Array<Array<number>>;
140141

src/resources/evals/evals.ts

Lines changed: 72 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import * as RunsAPI from './runs/runs';
1010
import {
1111
CreateEvalCompletionsRunDataSource,
1212
CreateEvalJSONLRunDataSource,
13+
CreateEvalResponsesRunDataSource,
1314
EvalAPIError,
1415
RunCancelResponse,
1516
RunCreateParams,
@@ -105,11 +106,37 @@ export interface EvalCustomDataSourceConfig {
105106
}
106107

107108
/**
108-
* A StoredCompletionsDataSourceConfig which specifies the metadata property of
109-
* your stored completions query. This is usually metadata like `usecase=chatbot`
110-
* or `prompt-version=v2`, etc. The schema returned by this data source config is
111-
* used to defined what variables are available in your evals. `item` and `sample`
112-
* are both defined when using this data source config.
109+
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
110+
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
111+
* schema returned by this data source config is used to defined what variables are
112+
* available in your evals. `item` and `sample` are both defined when using this
113+
* data source config.
114+
*/
115+
export interface EvalLogsDataSourceConfig {
116+
/**
117+
* The json schema for the run data source items. Learn how to build JSON schemas
118+
* [here](https://json-schema.org/).
119+
*/
120+
schema: Record<string, unknown>;
121+
122+
/**
123+
* The type of data source. Always `logs`.
124+
*/
125+
type: 'logs';
126+
127+
/**
128+
* Set of 16 key-value pairs that can be attached to an object. This can be useful
129+
* for storing additional information about the object in a structured format, and
130+
* querying for objects via API or the dashboard.
131+
*
132+
* Keys are strings with a maximum length of 64 characters. Values are strings with
133+
* a maximum length of 512 characters.
134+
*/
135+
metadata?: Shared.Metadata | null;
136+
}
137+
138+
/**
139+
* @deprecated Deprecated in favor of LogsDataSourceConfig.
113140
*/
114141
export interface EvalStoredCompletionsDataSourceConfig {
115142
/**
@@ -119,9 +146,9 @@ export interface EvalStoredCompletionsDataSourceConfig {
119146
schema: Record<string, unknown>;
120147

121148
/**
122-
* The type of data source. Always `stored_completions`.
149+
* The type of data source. Always `stored-completions`.
123150
*/
124-
type: 'stored_completions';
151+
type: 'stored-completions';
125152

126153
/**
127154
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -156,7 +183,10 @@ export interface EvalCreateResponse {
156183
/**
157184
* Configuration of data sources used in runs of the evaluation.
158185
*/
159-
data_source_config: EvalCustomDataSourceConfig | EvalStoredCompletionsDataSourceConfig;
186+
data_source_config:
187+
| EvalCustomDataSourceConfig
188+
| EvalLogsDataSourceConfig
189+
| EvalStoredCompletionsDataSourceConfig;
160190

161191
/**
162192
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -244,7 +274,10 @@ export interface EvalRetrieveResponse {
244274
/**
245275
* Configuration of data sources used in runs of the evaluation.
246276
*/
247-
data_source_config: EvalCustomDataSourceConfig | EvalStoredCompletionsDataSourceConfig;
277+
data_source_config:
278+
| EvalCustomDataSourceConfig
279+
| EvalLogsDataSourceConfig
280+
| EvalStoredCompletionsDataSourceConfig;
248281

249282
/**
250283
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -332,7 +365,10 @@ export interface EvalUpdateResponse {
332365
/**
333366
* Configuration of data sources used in runs of the evaluation.
334367
*/
335-
data_source_config: EvalCustomDataSourceConfig | EvalStoredCompletionsDataSourceConfig;
368+
data_source_config:
369+
| EvalCustomDataSourceConfig
370+
| EvalLogsDataSourceConfig
371+
| EvalStoredCompletionsDataSourceConfig;
336372

337373
/**
338374
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -420,7 +456,10 @@ export interface EvalListResponse {
420456
/**
421457
* Configuration of data sources used in runs of the evaluation.
422458
*/
423-
data_source_config: EvalCustomDataSourceConfig | EvalStoredCompletionsDataSourceConfig;
459+
data_source_config:
460+
| EvalCustomDataSourceConfig
461+
| EvalLogsDataSourceConfig
462+
| EvalStoredCompletionsDataSourceConfig;
424463

425464
/**
426465
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -498,7 +537,7 @@ export interface EvalCreateParams {
498537
/**
499538
* The configuration for the data source used for the evaluation runs.
500539
*/
501-
data_source_config: EvalCreateParams.Custom | EvalCreateParams.StoredCompletions;
540+
data_source_config: EvalCreateParams.Custom | EvalCreateParams.Logs | EvalCreateParams.StoredCompletions;
502541

503542
/**
504543
* A list of graders for all eval runs in this group.
@@ -555,15 +594,29 @@ export namespace EvalCreateParams {
555594
}
556595

557596
/**
558-
* A data source config which specifies the metadata property of your stored
559-
* completions query. This is usually metadata like `usecase=chatbot` or
560-
* `prompt-version=v2`, etc.
597+
* A data source config which specifies the metadata property of your logs query.
598+
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
599+
*/
600+
export interface Logs {
601+
/**
602+
* The type of data source. Always `logs`.
603+
*/
604+
type: 'logs';
605+
606+
/**
607+
* Metadata filters for the logs data source.
608+
*/
609+
metadata?: Record<string, unknown>;
610+
}
611+
612+
/**
613+
* Deprecated in favor of LogsDataSourceConfig.
561614
*/
562615
export interface StoredCompletions {
563616
/**
564-
* The type of data source. Always `stored_completions`.
617+
* The type of data source. Always `stored-completions`.
565618
*/
566-
type: 'stored_completions';
619+
type: 'stored-completions';
567620

568621
/**
569622
* Metadata filters for the stored completions data source.
@@ -733,6 +786,7 @@ Evals.RunListResponsesPage = RunListResponsesPage;
733786
export declare namespace Evals {
734787
export {
735788
type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig,
789+
type EvalLogsDataSourceConfig as EvalLogsDataSourceConfig,
736790
type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig,
737791
type EvalCreateResponse as EvalCreateResponse,
738792
type EvalRetrieveResponse as EvalRetrieveResponse,
@@ -749,6 +803,7 @@ export declare namespace Evals {
749803
Runs as Runs,
750804
type CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource,
751805
type CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource,
806+
type CreateEvalResponsesRunDataSource as CreateEvalResponsesRunDataSource,
752807
type EvalAPIError as EvalAPIError,
753808
type RunCreateResponse as RunCreateResponse,
754809
type RunRetrieveResponse as RunRetrieveResponse,

src/resources/evals/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ export {
44
EvalListResponsesPage,
55
Evals,
66
type EvalCustomDataSourceConfig,
7+
type EvalLogsDataSourceConfig,
78
type EvalStoredCompletionsDataSourceConfig,
89
type EvalCreateResponse,
910
type EvalRetrieveResponse,
@@ -19,6 +20,7 @@ export {
1920
Runs,
2021
type CreateEvalCompletionsRunDataSource,
2122
type CreateEvalJSONLRunDataSource,
23+
type CreateEvalResponsesRunDataSource,
2224
type EvalAPIError,
2325
type RunCreateResponse,
2426
type RunRetrieveResponse,

src/resources/evals/runs/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ export {
1212
Runs,
1313
type CreateEvalCompletionsRunDataSource,
1414
type CreateEvalJSONLRunDataSource,
15+
type CreateEvalResponsesRunDataSource,
1516
type EvalAPIError,
1617
type RunCreateResponse,
1718
type RunRetrieveResponse,

0 commit comments

Comments
 (0)