Skip to content

Commit cf0244c

Browse files
gandhiprgandhipr
authored
[Evals Svc Team] Evaluation Target typespec for modelResponseGenerationTarget (Azure#35021)
* eval target typespec * review changes * review comments after api review * rebase and compile * remove from v1 version * tsp format
1 parent c73ff8f commit cf0244c

File tree

2 files changed

+260
-0
lines changed

2 files changed

+260
-0
lines changed

‎specification/ai/Azure.AI.Projects/evaluations/models.tsp

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ model Evaluation {
7878

7979
@doc("Evaluators to be used for the evaluation.")
8080
evaluators: Record<EvaluatorConfiguration>;
81+
82+
@doc("Specifies the type and configuration of the entity used for this evaluation.")
83+
target?: EvaluationTarget;
8184
}
8285

8386
@doc("Definition for sampling strategy.")
@@ -176,3 +179,92 @@ model AgentEvaluation {
176179
@doc("The agent evaluation result.")
177180
result?: Array<AgentEvaluationResult>;
178181
}
182+
183+
@doc("Abstract base model representing a single message in a conversation.")
184+
@discriminator("role")
185+
@removed(Versions.v1)
186+
@added(Versions.v2025_05_15_preview)
187+
model Message {
188+
@doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.")
189+
role: "system" | "assistant" | "developer" | "user" | string;
190+
}
191+
192+
@doc("A message authored by the system to guide model behavior.")
193+
@removed(Versions.v1)
194+
@added(Versions.v2025_05_15_preview)
195+
model SystemMessage extends Message {
196+
@doc("Indicates this is a system message.")
197+
role: "system";
198+
199+
@doc("Plain text instructions provided by the system to steer model behavior.")
200+
content: string;
201+
}
202+
203+
@doc("A message authored by a developer to guide the model during evaluation.")
204+
@removed(Versions.v1)
205+
@added(Versions.v2025_05_15_preview)
206+
model DeveloperMessage extends Message {
207+
@doc("Indicates this is a developer message.")
208+
role: "developer";
209+
210+
@doc("Content provided by a developer to guide model behavior in an evaluation context.")
211+
content: string;
212+
}
213+
214+
@doc("A message authored by the end user as input to the model.")
215+
@removed(Versions.v1)
216+
@added(Versions.v2025_05_15_preview)
217+
model UserMessage extends Message {
218+
@doc("Indicates this is a user message.")
219+
role: "user";
220+
221+
@doc("Input content or question provided by the end user.")
222+
content: string;
223+
}
224+
225+
@doc("A message generated by the assistant in response to previous messages.")
226+
@removed(Versions.v1)
227+
@added(Versions.v2025_05_15_preview)
228+
model AssistantMessage extends Message {
229+
@doc("Indicates this is an assistant message.")
230+
role: "assistant";
231+
232+
@doc("Response content generated by the assistant.")
233+
content: string;
234+
}
235+
236+
@doc("Allowed types of evaluation targets.")
237+
@removed(Versions.v1)
238+
@added(Versions.v2025_05_15_preview)
239+
union EvaluationTargetType {
240+
@doc("Evaluation target that uses a model for response generation.")
241+
modelResponseGeneration: "modelResponseGeneration",
242+
243+
string,
244+
}
245+
246+
@doc("Abstract base model for defining evaluation targets.")
247+
@discriminator("type")
248+
@removed(Versions.v1)
249+
@added(Versions.v2025_05_15_preview)
250+
model EvaluationTarget {
251+
@doc("Discriminator that defines the type of the evaluation target.")
252+
type: EvaluationTargetType;
253+
}
254+
255+
@doc("Evaluation target for generating responses using a given model and dataset.")
256+
@removed(Versions.v1)
257+
@added(Versions.v2025_05_15_preview)
258+
model modelResponseGenerationTarget extends EvaluationTarget {
259+
@doc("The type of evaluation target. Always 'modelResponseGeneration'.")
260+
type: EvaluationTargetType.modelResponseGeneration;
261+
262+
@doc("A list of messages comprising the conversation so far.")
263+
baseMessages: Message[];
264+
265+
@doc("The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'.")
266+
modelDeploymentName: string;
267+
268+
@doc("Optional parameters passed to the model for evaluation.")
269+
modelParams: Record<unknown>;
270+
}

‎specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1695,6 +1695,25 @@
16951695
"blobReference"
16961696
]
16971697
},
1698+
"AssistantMessage": {
1699+
"type": "object",
1700+
"description": "A message generated by the assistant in response to previous messages.",
1701+
"properties": {
1702+
"content": {
1703+
"type": "string",
1704+
"description": "Response content generated by the assistant."
1705+
}
1706+
},
1707+
"required": [
1708+
"content"
1709+
],
1710+
"allOf": [
1711+
{
1712+
"$ref": "#/definitions/Message"
1713+
}
1714+
],
1715+
"x-ms-discriminator-value": "assistant"
1716+
},
16981717
"AttackStrategy": {
16991718
"type": "string",
17001719
"description": "Strategies for attacks.",
@@ -2378,6 +2397,25 @@
23782397
]
23792398
}
23802399
},
2400+
"DeveloperMessage": {
2401+
"type": "object",
2402+
"description": "A message authored by a developer to guide the model during evaluation.",
2403+
"properties": {
2404+
"content": {
2405+
"type": "string",
2406+
"description": "Content provided by a developer to guide model behavior in an evaluation context."
2407+
}
2408+
},
2409+
"required": [
2410+
"content"
2411+
],
2412+
"allOf": [
2413+
{
2414+
"$ref": "#/definitions/Message"
2415+
}
2416+
],
2417+
"x-ms-discriminator-value": "developer"
2418+
},
23812419
"EmbeddingConfiguration": {
23822420
"type": "object",
23832421
"description": "Embedding configuration class",
@@ -2459,6 +2497,10 @@
24592497
"additionalProperties": {
24602498
"$ref": "#/definitions/EvaluatorConfiguration"
24612499
}
2500+
},
2501+
"target": {
2502+
"$ref": "#/definitions/EvaluationTarget",
2503+
"description": "Specifies the type and configuration of the entity used for this evaluation."
24622504
}
24632505
},
24642506
"required": [
@@ -2467,6 +2509,38 @@
24672509
"evaluators"
24682510
]
24692511
},
2512+
"EvaluationTarget": {
2513+
"type": "object",
2514+
"description": "Abstract base model for defining evaluation targets.",
2515+
"properties": {
2516+
"type": {
2517+
"$ref": "#/definitions/EvaluationTargetType",
2518+
"description": "Discriminator that defines the type of the evaluation target."
2519+
}
2520+
},
2521+
"discriminator": "type",
2522+
"required": [
2523+
"type"
2524+
]
2525+
},
2526+
"EvaluationTargetType": {
2527+
"type": "string",
2528+
"description": "Allowed types of evaluation targets.",
2529+
"enum": [
2530+
"modelResponseGeneration"
2531+
],
2532+
"x-ms-enum": {
2533+
"name": "EvaluationTargetType",
2534+
"modelAsString": true,
2535+
"values": [
2536+
{
2537+
"name": "modelResponseGeneration",
2538+
"value": "modelResponseGeneration",
2539+
"description": "Evaluation target that uses a model for response generation."
2540+
}
2541+
]
2542+
}
2543+
},
24702544
"EvaluatorConfiguration": {
24712545
"type": "object",
24722546
"description": "Evaluator Configuration",
@@ -2706,6 +2780,29 @@
27062780
],
27072781
"x-ms-discriminator-value": "ManagedAzureSearch"
27082782
},
2783+
"Message": {
2784+
"type": "object",
2785+
"description": "Abstract base model representing a single message in a conversation.",
2786+
"properties": {
2787+
"role": {
2788+
"type": "string",
2789+
"description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.",
2790+
"enum": [
2791+
"system",
2792+
"assistant",
2793+
"developer",
2794+
"user"
2795+
],
2796+
"x-ms-enum": {
2797+
"modelAsString": true
2798+
}
2799+
}
2800+
},
2801+
"discriminator": "role",
2802+
"required": [
2803+
"role"
2804+
]
2805+
},
27092806
"ModelDeployment": {
27102807
"type": "object",
27112808
"description": "Model Deployment Definition",
@@ -3181,6 +3278,25 @@
31813278
"tier"
31823279
]
31833280
},
3281+
"SystemMessage": {
3282+
"type": "object",
3283+
"description": "A message authored by the system to guide model behavior.",
3284+
"properties": {
3285+
"content": {
3286+
"type": "string",
3287+
"description": "Plain text instructions provided by the system to steer model behavior."
3288+
}
3289+
},
3290+
"required": [
3291+
"content"
3292+
],
3293+
"allOf": [
3294+
{
3295+
"$ref": "#/definitions/Message"
3296+
}
3297+
],
3298+
"x-ms-discriminator-value": "system"
3299+
},
31843300
"TargetConfig": {
31853301
"type": "object",
31863302
"description": "Abstract class for target configuration.",
@@ -3194,6 +3310,58 @@
31943310
"required": [
31953311
"type"
31963312
]
3313+
},
3314+
"UserMessage": {
3315+
"type": "object",
3316+
"description": "A message authored by the end user as input to the model.",
3317+
"properties": {
3318+
"content": {
3319+
"type": "string",
3320+
"description": "Input content or question provided by the end user."
3321+
}
3322+
},
3323+
"required": [
3324+
"content"
3325+
],
3326+
"allOf": [
3327+
{
3328+
"$ref": "#/definitions/Message"
3329+
}
3330+
],
3331+
"x-ms-discriminator-value": "user"
3332+
},
3333+
"modelResponseGenerationTarget": {
3334+
"type": "object",
3335+
"description": "Evaluation target for generating responses using a given model and dataset.",
3336+
"properties": {
3337+
"baseMessages": {
3338+
"type": "array",
3339+
"description": "A list of messages comprising the conversation so far.",
3340+
"items": {
3341+
"$ref": "#/definitions/Message"
3342+
}
3343+
},
3344+
"modelDeploymentName": {
3345+
"type": "string",
3346+
"description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'."
3347+
},
3348+
"modelParams": {
3349+
"type": "object",
3350+
"description": "Optional parameters passed to the model for evaluation.",
3351+
"additionalProperties": {}
3352+
}
3353+
},
3354+
"required": [
3355+
"baseMessages",
3356+
"modelDeploymentName",
3357+
"modelParams"
3358+
],
3359+
"allOf": [
3360+
{
3361+
"$ref": "#/definitions/EvaluationTarget"
3362+
}
3363+
],
3364+
"x-ms-discriminator-value": "modelResponseGeneration"
31973365
}
31983366
},
31993367
"parameters": {

0 commit comments

Comments
 (0)