@@ -10,6 +10,7 @@ import * as RunsAPI from './runs/runs';
10
10
import {
11
11
CreateEvalCompletionsRunDataSource ,
12
12
CreateEvalJSONLRunDataSource ,
13
+ CreateEvalResponsesRunDataSource ,
13
14
EvalAPIError ,
14
15
RunCancelResponse ,
15
16
RunCreateParams ,
@@ -105,11 +106,37 @@ export interface EvalCustomDataSourceConfig {
105
106
}
106
107
107
108
/**
108
- * A StoredCompletionsDataSourceConfig which specifies the metadata property of
109
- * your stored completions query. This is usually metadata like `usecase=chatbot`
110
- * or `prompt-version=v2`, etc. The schema returned by this data source config is
111
- * used to defined what variables are available in your evals. `item` and `sample`
112
- * are both defined when using this data source config.
109
+ * A LogsDataSourceConfig which specifies the metadata property of your logs query.
110
+ * This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
111
+ * schema returned by this data source config is used to defined what variables are
112
+ * available in your evals. `item` and `sample` are both defined when using this
113
+ * data source config.
114
+ */
115
+ export interface EvalLogsDataSourceConfig {
116
+ /**
117
+ * The json schema for the run data source items. Learn how to build JSON schemas
118
+ * [here](https://json-schema.org/).
119
+ */
120
+ schema : Record < string , unknown > ;
121
+
122
+ /**
123
+ * The type of data source. Always `logs`.
124
+ */
125
+ type : 'logs' ;
126
+
127
+ /**
128
+ * Set of 16 key-value pairs that can be attached to an object. This can be useful
129
+ * for storing additional information about the object in a structured format, and
130
+ * querying for objects via API or the dashboard.
131
+ *
132
+ * Keys are strings with a maximum length of 64 characters. Values are strings with
133
+ * a maximum length of 512 characters.
134
+ */
135
+ metadata ?: Shared . Metadata | null ;
136
+ }
137
+
138
+ /**
139
+ * @deprecated Deprecated in favor of LogsDataSourceConfig.
113
140
*/
114
141
export interface EvalStoredCompletionsDataSourceConfig {
115
142
/**
@@ -119,9 +146,9 @@ export interface EvalStoredCompletionsDataSourceConfig {
119
146
schema : Record < string , unknown > ;
120
147
121
148
/**
122
- * The type of data source. Always `stored_completions `.
149
+ * The type of data source. Always `stored-completions `.
123
150
*/
124
- type : 'stored_completions ' ;
151
+ type : 'stored-completions ' ;
125
152
126
153
/**
127
154
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -156,7 +183,10 @@ export interface EvalCreateResponse {
156
183
/**
157
184
* Configuration of data sources used in runs of the evaluation.
158
185
*/
159
- data_source_config : EvalCustomDataSourceConfig | EvalStoredCompletionsDataSourceConfig ;
186
+ data_source_config :
187
+ | EvalCustomDataSourceConfig
188
+ | EvalLogsDataSourceConfig
189
+ | EvalStoredCompletionsDataSourceConfig ;
160
190
161
191
/**
162
192
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -244,7 +274,10 @@ export interface EvalRetrieveResponse {
244
274
/**
245
275
* Configuration of data sources used in runs of the evaluation.
246
276
*/
247
- data_source_config : EvalCustomDataSourceConfig | EvalStoredCompletionsDataSourceConfig ;
277
+ data_source_config :
278
+ | EvalCustomDataSourceConfig
279
+ | EvalLogsDataSourceConfig
280
+ | EvalStoredCompletionsDataSourceConfig ;
248
281
249
282
/**
250
283
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -332,7 +365,10 @@ export interface EvalUpdateResponse {
332
365
/**
333
366
* Configuration of data sources used in runs of the evaluation.
334
367
*/
335
- data_source_config : EvalCustomDataSourceConfig | EvalStoredCompletionsDataSourceConfig ;
368
+ data_source_config :
369
+ | EvalCustomDataSourceConfig
370
+ | EvalLogsDataSourceConfig
371
+ | EvalStoredCompletionsDataSourceConfig ;
336
372
337
373
/**
338
374
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -420,7 +456,10 @@ export interface EvalListResponse {
420
456
/**
421
457
* Configuration of data sources used in runs of the evaluation.
422
458
*/
423
- data_source_config : EvalCustomDataSourceConfig | EvalStoredCompletionsDataSourceConfig ;
459
+ data_source_config :
460
+ | EvalCustomDataSourceConfig
461
+ | EvalLogsDataSourceConfig
462
+ | EvalStoredCompletionsDataSourceConfig ;
424
463
425
464
/**
426
465
* Set of 16 key-value pairs that can be attached to an object. This can be useful
@@ -498,7 +537,7 @@ export interface EvalCreateParams {
498
537
/**
499
538
* The configuration for the data source used for the evaluation runs.
500
539
*/
501
- data_source_config : EvalCreateParams . Custom | EvalCreateParams . StoredCompletions ;
540
+ data_source_config : EvalCreateParams . Custom | EvalCreateParams . Logs | EvalCreateParams . StoredCompletions ;
502
541
503
542
/**
504
543
* A list of graders for all eval runs in this group.
@@ -555,15 +594,29 @@ export namespace EvalCreateParams {
555
594
}
556
595
557
596
/**
558
- * A data source config which specifies the metadata property of your stored
559
- * completions query. This is usually metadata like `usecase=chatbot` or
560
- * `prompt-version=v2`, etc.
597
+ * A data source config which specifies the metadata property of your logs query.
598
+ * This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
599
+ */
600
+ export interface Logs {
601
+ /**
602
+ * The type of data source. Always `logs`.
603
+ */
604
+ type : 'logs' ;
605
+
606
+ /**
607
+ * Metadata filters for the logs data source.
608
+ */
609
+ metadata ?: Record < string , unknown > ;
610
+ }
611
+
612
+ /**
613
+ * Deprecated in favor of LogsDataSourceConfig.
561
614
*/
562
615
export interface StoredCompletions {
563
616
/**
564
- * The type of data source. Always `stored_completions `.
617
+ * The type of data source. Always `stored-completions `.
565
618
*/
566
- type : 'stored_completions ' ;
619
+ type : 'stored-completions ' ;
567
620
568
621
/**
569
622
* Metadata filters for the stored completions data source.
@@ -733,6 +786,7 @@ Evals.RunListResponsesPage = RunListResponsesPage;
733
786
export declare namespace Evals {
734
787
export {
735
788
type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig ,
789
+ type EvalLogsDataSourceConfig as EvalLogsDataSourceConfig ,
736
790
type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig ,
737
791
type EvalCreateResponse as EvalCreateResponse ,
738
792
type EvalRetrieveResponse as EvalRetrieveResponse ,
@@ -749,6 +803,7 @@ export declare namespace Evals {
749
803
Runs as Runs ,
750
804
type CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource ,
751
805
type CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource ,
806
+ type CreateEvalResponsesRunDataSource as CreateEvalResponsesRunDataSource ,
752
807
type EvalAPIError as EvalAPIError ,
753
808
type RunCreateResponse as RunCreateResponse ,
754
809
type RunRetrieveResponse as RunRetrieveResponse ,
0 commit comments