@@ -45,20 +45,14 @@ class EvalCreateResponse < OpenAI::Internal::Type::BaseModel
4545 # @return [Symbol, :eval]
4646 required :object , const : :eval
4747
48- # @!attribute share_with_openai
49- # Indicates whether the evaluation is shared with OpenAI.
50- #
51- # @return [Boolean]
52- required :share_with_openai , OpenAI ::Internal ::Type ::Boolean
53-
5448 # @!attribute testing_criteria
5549 # A list of testing criteria.
5650 #
57- # @return [Array<OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader>]
51+ # @return [Array<OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::Python, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel >]
5852 required :testing_criteria ,
5953 -> { OpenAI ::Internal ::Type ::ArrayOf [ union : OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ] }
6054
61- # @!method initialize(id:, created_at:, data_source_config:, metadata:, name:, share_with_openai:, testing_criteria:, object: :eval)
55+ # @!method initialize(id:, created_at:, data_source_config:, metadata:, name:, testing_criteria:, object: :eval)
6256 # An Eval object with a data source config and testing criteria. An Eval
6357 # represents a task to be done for your LLM integration. Like:
6458 #
@@ -71,8 +65,7 @@ class EvalCreateResponse < OpenAI::Internal::Type::BaseModel
7165 # @param data_source_config [OpenAI::Models::EvalCustomDataSourceConfig, OpenAI::Models::EvalStoredCompletionsDataSourceConfig]
7266 # @param metadata [Hash{Symbol=>String}, nil]
7367 # @param name [String]
74- # @param share_with_openai [Boolean]
75- # @param testing_criteria [Array<OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader>]
68+ # @param testing_criteria [Array<OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::Python, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel>]
7669 # @param object [Symbol, :eval]
7770
7871 # Configuration of data sources used in runs of the evaluation.
@@ -116,8 +109,211 @@ module TestingCriterion
116109 # A TextSimilarityGrader object which grades text based on similarity metrics.
117110 variant :text_similarity , -> { OpenAI ::Models ::EvalTextSimilarityGrader }
118111
112+ # A PythonGrader object that runs a python script on the input.
113+ variant :python , -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::Python }
114+
115+ # A ScoreModelGrader object that uses a model to assign a score to the input.
116+ variant :score_model , -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel }
117+
118+ class Python < OpenAI ::Internal ::Type ::BaseModel
119+ # @!attribute name
120+ # The name of the grader.
121+ #
122+ # @return [String]
123+ required :name , String
124+
125+ # @!attribute source
126+ # The source code of the python script.
127+ #
128+ # @return [String]
129+ required :source , String
130+
131+ # @!attribute type
132+ # The object type, which is always `python`.
133+ #
134+ # @return [Symbol, :python]
135+ required :type , const : :python
136+
137+ # @!attribute image_tag
138+ # The image tag to use for the python script.
139+ #
140+ # @return [String, nil]
141+ optional :image_tag , String
142+
143+ # @!attribute pass_threshold
144+ # The threshold for the score.
145+ #
146+ # @return [Float, nil]
147+ optional :pass_threshold , Float
148+
149+ # @!method initialize(name:, source:, image_tag: nil, pass_threshold: nil, type: :python)
150+ # A PythonGrader object that runs a python script on the input.
151+ #
152+ # @param name [String]
153+ # @param source [String]
154+ # @param image_tag [String]
155+ # @param pass_threshold [Float]
156+ # @param type [Symbol, :python]
157+ end
158+
159+ class ScoreModel < OpenAI ::Internal ::Type ::BaseModel
160+ # @!attribute input
161+ # The input text. This may include template strings.
162+ #
163+ # @return [Array<OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input>]
164+ required :input ,
165+ -> { OpenAI ::Internal ::Type ::ArrayOf [ OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ] }
166+
167+ # @!attribute model
168+ # The model to use for the evaluation.
169+ #
170+ # @return [String]
171+ required :model , String
172+
173+ # @!attribute name
174+ # The name of the grader.
175+ #
176+ # @return [String]
177+ required :name , String
178+
179+ # @!attribute type
180+ # The object type, which is always `score_model`.
181+ #
182+ # @return [Symbol, :score_model]
183+ required :type , const : :score_model
184+
185+ # @!attribute pass_threshold
186+ # The threshold for the score.
187+ #
188+ # @return [Float, nil]
189+ optional :pass_threshold , Float
190+
191+ # @!attribute range
192+ # The range of the score. Defaults to `[0, 1]`.
193+ #
194+ # @return [Array<Float>, nil]
195+ optional :range , OpenAI ::Internal ::Type ::ArrayOf [ Float ]
196+
197+ # @!attribute sampling_params
198+ # The sampling parameters for the model.
199+ #
200+ # @return [Object, nil]
201+ optional :sampling_params , OpenAI ::Internal ::Type ::Unknown
202+
203+ # @!method initialize(input:, model:, name:, pass_threshold: nil, range: nil, sampling_params: nil, type: :score_model)
204+ # A ScoreModelGrader object that uses a model to assign a score to the input.
205+ #
206+ # @param input [Array<OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input>]
207+ # @param model [String]
208+ # @param name [String]
209+ # @param pass_threshold [Float]
210+ # @param range [Array<Float>]
211+ # @param sampling_params [Object]
212+ # @param type [Symbol, :score_model]
213+
214+ class Input < OpenAI ::Internal ::Type ::BaseModel
215+ # @!attribute content
216+ # Text inputs to the model - can contain template strings.
217+ #
218+ # @return [String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Content::OutputText]
219+ required :content ,
220+ union : -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ::Content }
221+
222+ # @!attribute role
223+ # The role of the message input. One of `user`, `assistant`, `system`, or
224+ # `developer`.
225+ #
226+ # @return [Symbol, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Role]
227+ required :role , enum : -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ::Role }
228+
229+ # @!attribute type
230+ # The type of the message input. Always `message`.
231+ #
232+ # @return [Symbol, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Type, nil]
233+ optional :type , enum : -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ::Type }
234+
235+ # @!method initialize(content:, role:, type: nil)
236+ # A message input to the model with a role indicating instruction following
237+ # hierarchy. Instructions given with the `developer` or `system` role take
238+ # precedence over instructions given with the `user` role. Messages with the
239+ # `assistant` role are presumed to have been generated by the model in previous
240+ # interactions.
241+ #
242+ # @param content [String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Content::OutputText]
243+ # @param role [Symbol, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Role]
244+ # @param type [Symbol, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Type]
245+
246+ # Text inputs to the model - can contain template strings.
247+ #
248+ # @see OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input#content
249+ module Content
250+ extend OpenAI ::Internal ::Type ::Union
251+
252+ # A text input to the model.
253+ variant String
254+
255+ # A text input to the model.
256+ variant -> { OpenAI ::Models ::Responses ::ResponseInputText }
257+
258+ # A text output from the model.
259+ variant -> { OpenAI ::Models ::EvalCreateResponse ::TestingCriterion ::ScoreModel ::Input ::Content ::OutputText }
260+
261+ class OutputText < OpenAI ::Internal ::Type ::BaseModel
262+ # @!attribute text
263+ # The text output from the model.
264+ #
265+ # @return [String]
266+ required :text , String
267+
268+ # @!attribute type
269+ # The type of the output text. Always `output_text`.
270+ #
271+ # @return [Symbol, :output_text]
272+ required :type , const : :output_text
273+
274+ # @!method initialize(text:, type: :output_text)
275+ # A text output from the model.
276+ #
277+ # @param text [String]
278+ # @param type [Symbol, :output_text]
279+ end
280+
281+ # @!method self.variants
282+ # @return [Array(String, OpenAI::Models::Responses::ResponseInputText, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input::Content::OutputText)]
283+ end
284+
285+ # The role of the message input. One of `user`, `assistant`, `system`, or
286+ # `developer`.
287+ #
288+ # @see OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input#role
289+ module Role
290+ extend OpenAI ::Internal ::Type ::Enum
291+
292+ USER = :user
293+ ASSISTANT = :assistant
294+ SYSTEM = :system
295+ DEVELOPER = :developer
296+
297+ # @!method self.values
298+ # @return [Array<Symbol>]
299+ end
300+
301+ # The type of the message input. Always `message`.
302+ #
303+ # @see OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel::Input#type
304+ module Type
305+ extend OpenAI ::Internal ::Type ::Enum
306+
307+ MESSAGE = :message
308+
309+ # @!method self.values
310+ # @return [Array<Symbol>]
311+ end
312+ end
313+ end
314+
119315 # @!method self.variants
120- # @return [Array(OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader)]
316+ # @return [Array(OpenAI::Models::EvalLabelModelGrader, OpenAI::Models::EvalStringCheckGrader, OpenAI::Models::EvalTextSimilarityGrader, OpenAI::Models::EvalCreateResponse::TestingCriterion::Python, OpenAI::Models::EvalCreateResponse::TestingCriterion::ScoreModel )]
121317 end
122318 end
123319 end
0 commit comments