diff --git a/specification/ai/Foundry/datasets/routes.tsp b/specification/ai/Foundry/datasets/routes.tsp index cf271c5c886d..8b53eda1f884 100644 --- a/specification/ai/Foundry/datasets/routes.tsp +++ b/specification/ai/Foundry/datasets/routes.tsp @@ -13,7 +13,15 @@ using Azure.ClientGenerator.Core; namespace Azure.AI.Projects; -alias ListDatasetVersionsParameters = {}; +alias ListDatasetVersionsParameters = { + @doc("Filter datasets by taxonomy ID. Returns datasets where tags['taxonomy_id'] matches the provided value.") + @Http.query + taxonomyId?: string; + + @doc("Filter datasets by evaluation run ID. Returns datasets where tags['eval_run_id'] matches the provided value.") + @Http.query + evalRunId?: string; +}; #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "We are using service specific operation templates" @added(Versions.v2025_05_01) diff --git a/specification/ai/Foundry/evaluation_taxonomies/models.tsp b/specification/ai/Foundry/evaluation_taxonomies/models.tsp index 450f87a28e30..ddf575026af5 100644 --- a/specification/ai/Foundry/evaluation_taxonomies/models.tsp +++ b/specification/ai/Foundry/evaluation_taxonomies/models.tsp @@ -40,6 +40,9 @@ union EvaluationTaxonomyInputType { @doc("Policy.") policy: "policy", + + @doc("Behavior-based taxonomy input for custom behavior definitions") + behavior: "behavior", } @added(Versions.v2025_11_15_preview) @@ -65,6 +68,22 @@ model AgentTaxonomyInput extends EvaluationTaxonomyInput { riskCategories: RiskCategory[]; } +@added(Versions.v2025_11_15_preview) +@removed(Versions.v1) +@doc("Input configuration for the evaluation taxonomy when the input type is behavior.") +model BehaviorTaxonomyInput extends EvaluationTaxonomyInput { + @doc("Input type of the evaluation taxonomy.") + type: EvaluationTaxonomyInputType.behavior; + + @doc("Name of the behavior to evaluate") + behavior_name: string; + + @doc("Detailed definition of the behavior (10-2000 characters)") + @minLength(10) + @maxLength(2000) + behavior_definition: string; +} + @added(Versions.v2025_11_15_preview) @removed(Versions.v1) @doc("Taxonomy category definition.") @@ -101,9 +120,12 @@ model TaxonomySubCategory { @doc("Description of the taxonomy sub-category.") description?: string; - @doc("List of taxonomy items under this sub-category.") + @doc("Whether this sub-category is enabled for evaluation.") enabled: boolean; + @doc("List of subtopics for this sub-category.") + subtopics?: string[]; + @doc("Additional properties for the taxonomy sub-category.") properties?: Record; } diff --git a/specification/ai/Foundry/evaluation_taxonomies/routes.tsp b/specification/ai/Foundry/evaluation_taxonomies/routes.tsp index 26988e23b71e..0cc0a985eece 100644 --- a/specification/ai/Foundry/evaluation_taxonomies/routes.tsp +++ b/specification/ai/Foundry/evaluation_taxonomies/routes.tsp @@ -3,6 +3,9 @@ import "@azure-tools/typespec-autorest"; import "@typespec/versioning"; import "@azure-tools/typespec-azure-core"; import "./models.tsp"; +import "../datasets/models.tsp"; +import "../evaluators/models.tsp"; +import "../openai-evaluations/models.tsp"; using TypeSpec.Http; using TypeSpec.Versioning; @@ -73,4 +76,52 @@ interface EvaluationTaxonomies { }, OkResponse >; + + @doc("List datasets generated from this taxonomy.") + @get + @route("/evaluationtaxonomies/{name}/datasets") + listDatasets is Azure.Core.Foundations.Operation< + { + @doc("The name of the evaluation taxonomy.") + @path + name: string; + + @doc("Filter by taxonomy version.") + @query + version?: string; + }, + Azure.Core.Page + >; + + @doc("List evaluation runs that used this taxonomy.") + @get + @route("/evaluationtaxonomies/{name}/runs") + listRuns is Azure.Core.Foundations.Operation< + { + @doc("The name of the evaluation taxonomy.") + @path + name: string; + + @doc("Filter by taxonomy version.") + @query + version?: string; + }, + AgentsPagedResult + >; + + @doc("List evaluators referencing this taxonomy.") + @get + @route("/evaluationtaxonomies/{name}/evaluators") + listEvaluators is Azure.Core.Foundations.Operation< + { + @doc("The name of the evaluation taxonomy.") + @path + name: string; + + @doc("Filter by taxonomy version.") + @query + version?: string; + }, + Azure.Core.Page + >; } diff --git a/specification/ai/Foundry/evaluators/models.tsp b/specification/ai/Foundry/evaluators/models.tsp index 2ceb0b1ecfa6..dca3cc2ab392 100644 --- a/specification/ai/Foundry/evaluators/models.tsp +++ b/specification/ai/Foundry/evaluators/models.tsp @@ -60,6 +60,9 @@ union EvaluatorDefinitionType { @doc("OpenAI graders") openai_graders: "openai_graders", + + @doc("Behavior-based definition that auto-creates a linked taxonomy") + behavior: "behavior", } #suppress "@azure-tools/typespec-azure-core/no-unknown" @@ -101,6 +104,29 @@ model PromptBasedEvaluatorDefinition extends EvaluatorDefinition { prompt_text: string; } +@doc("Behavior-based evaluator definition that auto-creates a linked taxonomy") +@added(Versions.v2025_11_15_preview) +@removed(Versions.v1) +model BehaviorBasedEvaluatorDefinition extends EvaluatorDefinition { + type: EvaluatorDefinitionType.behavior; + + @doc("Name of the behavior to evaluate") + behavior_name: string; + + @doc("Detailed definition of the behavior (10-2000 characters)") + @minLength(10) + @maxLength(2000) + behavior_definition: string; + + @doc("ID of the auto-created linked taxonomy (read-only, set by service)") + @visibility(Lifecycle.Read) + taxonomy_id?: string; + + @doc("Current version of the linked taxonomy (read-only, set by service)") + @visibility(Lifecycle.Read) + taxonomy_version?: string; +} + @doc("Evaluator Definition") @resource("evaluators") @added(Versions.v2025_11_15_preview) diff --git a/specification/ai/Foundry/evaluators/routes.tsp b/specification/ai/Foundry/evaluators/routes.tsp index 8f5de3db47f0..df5ae0b64f68 100644 --- a/specification/ai/Foundry/evaluators/routes.tsp +++ b/specification/ai/Foundry/evaluators/routes.tsp @@ -17,6 +17,10 @@ alias ListEvaluatorVersionsParameters = { @Http.query type?: EvaluatorType | "all"; + @doc("Filter evaluators by taxonomy ID. Returns only evaluators with definition.type='taxonomy' and matching taxonomy_id.") + @Http.query + taxonomyId?: string; + @doc("A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20.") @Http.query limit?: int32 = 20; diff --git a/specification/ai/Foundry/openai-evaluations/models.tsp b/specification/ai/Foundry/openai-evaluations/models.tsp index ff79b1488d44..6483321c1030 100644 --- a/specification/ai/Foundry/openai-evaluations/models.tsp +++ b/specification/ai/Foundry/openai-evaluations/models.tsp @@ -198,6 +198,9 @@ model RedTeamTaxonomyItemGenerationParams extends RedTeamItemGenerationParams { /** The source from which JSONL content is read. */ source: OpenAI.EvalJsonlFileContentSource; + + /** Description of the application for context-aware seed generation. */ + application_description?: string; } /** Represents the set of parameters used to control item generation operations. */