Update evaluator - add is_primary and remove OpenAI style pagination

specification/ai/Azure.AI.Projects/evaluations/models.tsp

@@ -103,13 +103,13 @@ model EvalRunOutputItem {
   eval_id?: string;
 
   /** Unix timestamp (in seconds) when the evaluation run was created. */
-  created_at: integer;
+  created_at: int64;
 
   /** The status of the evaluation run. */
   status: string;
 
   /** The identifier for the data source item. */
-  datasource_item_id?: integer;
+  datasource_item_id?: int32;
 
   /** Details of the input data source item. */
   datasource_item: {};

specification/ai/Azure.AI.Projects/evaluations/routes.tsp

@@ -19,6 +19,7 @@ alias EvaluationsOperations = Azure.Core.ResourceOperations<ServiceTraits>;
 @added(Versions.v2025_10_15_preview)
 interface SyncEvals {
   /** Synchronize evaluation runs from connected resources. */
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations"
   @route("sync_evals:run")
   @post
   create is Azure.Core.Foundations.Operation<

specification/ai/Azure.AI.Projects/evaluators/models.tsp

@@ -121,11 +121,11 @@ model EvaluatorVersion {
 
   @visibility(Lifecycle.Read)
   @doc("Creation date/time of the evaluator")
-  created_at: string;
+  created_at: int64;
 
   @visibility(Lifecycle.Read)
   @doc("Last modified date/time of the evaluator")
-  modified_at: string;
+  modified_at: int64;
 
   ...AssetBase;
 }
@@ -144,6 +144,9 @@ model EvaluatorMetric {
 
   @doc("Maximum value for the metric. If not specified, it is assumed to be unbounded.")
   max_value?: float32;
+
+  @doc("Indicates if this metric is primary when there are multiple metrics.")
+  is_primary?: boolean;
 }
 
 @doc("The type of the evaluator")
@@ -174,4 +177,4 @@ union EvaluatorMetricDirection {
 
   @doc("It indicates no preference for this metric direction")
   neutral: "neutral",
-}
+}

specification/ai/Azure.AI.Projects/evaluators/routes.tsp

@@ -7,41 +7,44 @@ import "../common/models.tsp";
 import "./models.tsp";
 import "../servicepatterns.tsp";
 
-using TypeSpec.Http;
 using TypeSpec.Versioning;
 
 namespace Azure.AI.Projects;
 
-alias ListEvaluatorVersionsParameters = {};
-alias EvaluatorInputParameters = InputParameters<
-  EvaluatorVersion,
-  "evaluatorVersion"
->;
+alias ListEvaluatorVersionsParameters = {
+  @doc("Filter evaluators by type. Possible values: 'all', 'custom', 'builtin'.")
+  @Http.query
+  type?: EvaluatorType | "all";
+
+  @doc("A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20.")
+  @Http.query
+  limit?: int32 = 20;
+};
 
 @added(Versions.v2025_10_15_preview)
 interface Evaluators {
-  #suppress "@azure-tools/typespec-azure-core/use-standard-operations"
-  #suppress "@azure-tools/typespec-azure-core/operation-missing-api-version" "not yet versioned"
-  @get
-  @doc("List all versions of the given Evaluator name", EvaluatorVersion)
-  @route("/evaluators/{name}/versions")
-  op listEvaluatorVersions(
-    @doc("Name of the evaluator")
-    @path name: string,
-    ...OpenAIListRequestOptions,
-  ): OpenAIPageableListOf<EvaluatorVersion>;
-
-
+  // GET /evaluators/{name}/versions
+  #suppress "@azure-tools/typespec-azure-core/verb-conflict" "Resource action used as a building block."
+  @doc("List all versions of the given evaluator")
+  @Rest.action("versions")
+  @Rest.actionSeparator("/")
+  @Http.get
+  @list
+  listVersions is Azure.Core.StandardResourceOperations.ResourceAction<
+    EvaluatorVersion,
+    ListEvaluatorVersionsParameters,
+    Azure.Core.Page<EvaluatorVersion>
+  >;
 
+  // GET /evaluators
   #suppress "@azure-tools/typespec-azure-core/use-standard-operations"
-  #suppress "@azure-tools/typespec-azure-core/operation-missing-api-version" "not yet versioned"
-  @get
-  @doc("List the latest version of each Evaluator", EvaluatorVersion)
-  @route("/evaluators")
-  op listVersions(
-    ...OpenAIListRequestOptions,
-  ): OpenAIPageableListOf<EvaluatorVersion>;
-
+  @doc("List the latest version of each evaluator")
+  @Rest.listsResource(EvaluatorVersion)
+  listLatestVersions is Azure.Core.Foundations.ResourceList<
+    EvaluatorVersion,
+    ListEvaluatorVersionsParameters,
+    Azure.Core.Page<EvaluatorVersion>
+  >;
 
   // GET /evaluators/{name}/versions/{version}
   #suppress "@azure-tools/typespec-azure-core/use-standard-operations"

specification/ai/Azure.AI.Projects/examples/2025-10-15-preview/Evaluators_CreateEvaluatorVersion_MaximumSet_Gen.json

@@ -0,0 +1,95 @@
+{
+  "title": "Evaluators_CreateEvaluatorVersion_MaximumSet",
+  "operationId": "Evaluators_CreateEvaluatorVersion",
+  "parameters": {
+    "api-version": "2025-10-15-preview",
+    "name": "my_evaluator",
+    "type": "custom",
+    "evaluator": {
+      "display_name": "lirxubatpcivlmicfrzfoiotoxjf",
+      "metadata": {
+        "key8046": "nvatafccfjdslevuxmcmhsg"
+      },
+      "evaluator_type": "custom",
+      "categories": [
+        "quality"
+      ],
+      "definition": {
+        "type": "prompt",
+        "prompt_text": "Rate the response on a scale from 1 to 10.",
+        "data_schema": {
+          "type": "object",
+          "properties": {
+            "query": {
+              "type": "string"
+            },
+            "response": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "query",
+            "response"
+          ]
+        },
+        "metrics": {
+          "key2527": {
+            "type": "ordinal",
+            "desirable_direction": "increase",
+            "min_value": 19,
+            "max_value": 23,
+            "is_primary": true
+          }
+        }
+      }
+    }
+  },
+  "responses": {
+    "201": {
+      "body": {
+        "display_name": "lirxubatpcivlmicfrzfoiotoxjf",
+        "metadata": {
+          "key8046": "nvatafccfjdslevuxmcmhsg"
+        },
+        "evaluator_type": "custom",
+        "categories": [
+          "quality"
+        ],
+        "definition": {
+          "type": "prompt",
+          "prompt_text": "Rate the response on a scale from 1 to 10.",
+          "data_schema": {
+            "type": "object",
+            "properties": {
+              "query": {
+                "type": "string"
+              },
+              "response": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "query",
+              "response"
+            ]
+          },
+          "metrics": {
+            "key2527": {
+              "type": "ordinal",
+              "desirable_direction": "increase",
+              "min_value": 19,
+              "max_value": 23,
+              "is_primary": true
+            }
+          }
+        },
+        "created_by": "vetzbotxmwuscjezswaxzcnkc",
+        "created_at": 1759999990,
+        "modified_at": 1759999992,
+        "id": "azureai://accounts/account_1/projects/project_1/evaluators/custom_evaluator/versions/1",
+        "name": "my_evaluator",
+        "version": "v1"
+      }
+    }
+  }
+}

specification/ai/Azure.AI.Projects/examples/2025-10-15-preview/Evaluators_DeleteEvaluatorVersion_MaximumSet_Gen.json

@@ -0,0 +1,12 @@
+{
+  "title": "Evaluators_DeleteEvaluatorVersion_MaximumSet",
+  "operationId": "Evaluators_DeleteEvaluatorVersion",
+  "parameters": {
+    "api-version": "2025-10-15-preview",
+    "name": "my_evaluator",
+    "version": "v1"
+  },
+  "responses": {
+    "204": {}
+  }
+}

specification/ai/Azure.AI.Projects/examples/2025-10-15-preview/Evaluators_GetEvaluatorVersion_MaximumSet_Gen.json

@@ -0,0 +1,57 @@
+{
+  "title": "Evaluators_GetEvaluatorVersion_MaximumSet",
+  "operationId": "Evaluators_GetEvaluatorVersion",
+  "parameters": {
+    "api-version": "2025-10-15-preview",
+    "name": "my_evaluator",
+    "version": "v1"
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "display_name": "lirxubatpcivlmicfrzfoiotoxjf",
+        "metadata": {
+          "key8046": "nvatafccfjdslevuxmcmhsg"
+        },
+        "evaluator_type": "custom",
+        "categories": [
+          "quality"
+        ],
+        "definition": {
+          "type": "prompt",
+          "prompt_text": "Rate the response on a scale from 1 to 10.",
+          "data_schema": {
+            "type": "object",
+            "properties": {
+              "query": {
+                "type": "string"
+              },
+              "response": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "query",
+              "response"
+            ]
+          },
+          "metrics": {
+            "key2527": {
+              "type": "ordinal",
+              "desirable_direction": "increase",
+              "min_value": 19,
+              "max_value": 23,
+              "is_primary": true
+            }
+          }
+        },
+        "created_by": "vetzbotxmwuscjezswaxzcnkc",
+        "created_at": 1759999990,
+        "modified_at": 1759999992,
+        "id": "azureai://accounts/account_1/projects/project_1/evaluators/custom_evaluator/versions/1",
+        "name": "my_evaluator",
+        "version": "v1"
+      }
+    }
+  }
+}

specification/ai/Azure.AI.Projects/examples/2025-10-15-preview/Evaluators_ListLatestVersions_MaximumSet_Gen.json

@@ -0,0 +1,62 @@
+{
+  "title": "Evaluators_ListLatestVersions_MaximumSet",
+  "operationId": "Evaluators_ListLatestVersions",
+  "parameters": {
+    "api-version": "2025-10-15-preview",
+    "type": "custom",
+    "limit": 21
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "value": [
+          {
+            "display_name": "lirxubatpcivlmicfrzfoiotoxjf",
+            "metadata": {
+              "key8046": "nvatafccfjdslevuxmcmhsg"
+            },
+            "evaluator_type": "custom",
+            "categories": [
+              "quality"
+            ],
+            "definition": {
+              "type": "prompt",
+              "prompt_text": "Rate the response on a scale from 1 to 10.",
+              "data_schema": {
+                "type": "object",
+                "properties": {
+                  "query": {
+                    "type": "string"
+                  },
+                  "response": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "query",
+                  "response"
+                ]
+              },
+              "metrics": {
+                "key2527": {
+                  "type": "ordinal",
+                  "desirable_direction": "increase",
+                  "min_value": 19,
+                  "max_value": 23,
+                  "is_primary": true
+                }
+              }
+            },
+            "created_by": "vetzbotxmwuscjezswaxzcnkc",
+            "created_at": 1759999990,
+            "modified_at": 1759999992,
+            "id": "azureai://accounts/account_1/projects/project_1/evaluators/custom_evaluator/versions/1",
+            "name": "my_evaluator",
+            "version": "v1"
+          }
+        ],
+        "nextLink": "https://microsoft.com/askdath"
+      }
+    }
+  }
+}