Skip to content

intial p2m spec #39472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
slister1001 wants to merge 11 commits into
base: feature/foundry
Choose a base branch
from
+116 −2
Draft

intial p2m spec #39472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
b11b363
intial p2m spec
slister1001 Jan 6, 2026
017d5ee
updates
slister1001 Jan 6, 2026
19bc6e7
updates
slister1001 Jan 6, 2026
aaf0e5f
updates
slister1001 Jan 7, 2026
e561f85
updates from team review
slister1001 Jan 8, 2026
a6f90f1
updates from team review pt 2
slister1001 Jan 8, 2026
0dca3ea
updates
slister1001 Jan 8, 2026
9ab844f
Add taxonomy version pinning and taxonomyId filter for custom behavio…
slister1001 Jan 12, 2026
306d091
updates
slister1001 Jan 14, 2026
cc8e659
Add nested resource endpoints for taxonomy lineage
slister1001 Jan 20, 2026
2114351
Add BehaviorBasedEvaluatorDefinition for simplified evaluator creatio…
slister1001 Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion specification/ai/Foundry/datasets/routes.tsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,15 @@ using Azure.ClientGenerator.Core;

namespace Azure.AI.Projects;

alias ListDatasetVersionsParameters = {};
alias ListDatasetVersionsParameters = {
@doc("Filter datasets by taxonomy ID. Returns datasets where tags['taxonomy_id'] matches the provided value.")
@Http.query
taxonomyId?: string;

@doc("Filter datasets by evaluation run ID. Returns datasets where tags['eval_run_id'] matches the provided value.")
@Http.query
evalRunId?: string;
};

#suppress "@azure-tools/typespec-azure-core/use-standard-operations" "We are using service specific operation templates"
@added(Versions.v2025_05_01)
Expand Down
24 changes: 23 additions & 1 deletion specification/ai/Foundry/evaluation_taxonomies/models.tsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ union EvaluationTaxonomyInputType {

@doc("Policy.")
policy: "policy",

@doc("Behavior-based taxonomy input for custom behavior definitions")
behavior: "behavior",
}

@added(Versions.v2025_11_15_preview)
Expand All @@ -65,6 +68,22 @@ model AgentTaxonomyInput extends EvaluationTaxonomyInput {
riskCategories: RiskCategory[];
}

@added(Versions.v2025_11_15_preview)
@removed(Versions.v1)
@doc("Input configuration for the evaluation taxonomy when the input type is behavior.")
model BehaviorTaxonomyInput extends EvaluationTaxonomyInput {
@doc("Input type of the evaluation taxonomy.")
type: EvaluationTaxonomyInputType.behavior;

@doc("Name of the behavior to evaluate")
behavior_name: string;

@doc("Detailed definition of the behavior (10-2000 characters)")
@minLength(10)
@maxLength(2000)
behavior_definition: string;
}

@added(Versions.v2025_11_15_preview)
@removed(Versions.v1)
@doc("Taxonomy category definition.")
Expand Down Expand Up @@ -101,9 +120,12 @@ model TaxonomySubCategory {
@doc("Description of the taxonomy sub-category.")
description?: string;

@doc("List of taxonomy items under this sub-category.")
@doc("Whether this sub-category is enabled for evaluation.")
enabled: boolean;

@doc("List of subtopics for this sub-category.")
subtopics?: string[];

@doc("Additional properties for the taxonomy sub-category.")
properties?: Record<string>;
}
51 changes: 51 additions & 0 deletions specification/ai/Foundry/evaluation_taxonomies/routes.tsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ import "@azure-tools/typespec-autorest";
import "@typespec/versioning";
import "@azure-tools/typespec-azure-core";
import "./models.tsp";
import "../datasets/models.tsp";
import "../evaluators/models.tsp";
import "../openai-evaluations/models.tsp";

using TypeSpec.Http;
using TypeSpec.Versioning;
Expand Down Expand Up @@ -73,4 +76,52 @@ interface EvaluationTaxonomies {
},
OkResponse<EvaluationTaxonomy>
>;

@doc("List datasets generated from this taxonomy.")
@get
@route("/evaluationtaxonomies/{name}/datasets")
listDatasets is Azure.Core.Foundations.Operation<
{
@doc("The name of the evaluation taxonomy.")
@path
name: string;

@doc("Filter by taxonomy version.")
@query
version?: string;
},
Azure.Core.Page<DatasetVersion>
>;

@doc("List evaluation runs that used this taxonomy.")
@get
@route("/evaluationtaxonomies/{name}/runs")
listRuns is Azure.Core.Foundations.Operation<
{
@doc("The name of the evaluation taxonomy.")
@path
name: string;

@doc("Filter by taxonomy version.")
@query
version?: string;
},
AgentsPagedResult<EvalRun>
>;

@doc("List evaluators referencing this taxonomy.")
@get
@route("/evaluationtaxonomies/{name}/evaluators")
listEvaluators is Azure.Core.Foundations.Operation<
{
@doc("The name of the evaluation taxonomy.")
@path
name: string;

@doc("Filter by taxonomy version.")
@query
version?: string;
},
Azure.Core.Page<EvaluatorVersion>
>;
}
26 changes: 26 additions & 0 deletions specification/ai/Foundry/evaluators/models.tsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ union EvaluatorDefinitionType {

@doc("OpenAI graders")
openai_graders: "openai_graders",

@doc("Behavior-based definition that auto-creates a linked taxonomy")
behavior: "behavior",
}

#suppress "@azure-tools/typespec-azure-core/no-unknown"
Expand Down Expand Up @@ -101,6 +104,29 @@ model PromptBasedEvaluatorDefinition extends EvaluatorDefinition {
prompt_text: string;
}

@doc("Behavior-based evaluator definition that auto-creates a linked taxonomy")
@added(Versions.v2025_11_15_preview)
@removed(Versions.v1)
model BehaviorBasedEvaluatorDefinition extends EvaluatorDefinition {
type: EvaluatorDefinitionType.behavior;

@doc("Name of the behavior to evaluate")
behavior_name: string;

@doc("Detailed definition of the behavior (10-2000 characters)")
@minLength(10)
@maxLength(2000)
behavior_definition: string;

@doc("ID of the auto-created linked taxonomy (read-only, set by service)")
@visibility(Lifecycle.Read)
taxonomy_id?: string;

@doc("Current version of the linked taxonomy (read-only, set by service)")
@visibility(Lifecycle.Read)
taxonomy_version?: string;
}

@doc("Evaluator Definition")
@resource("evaluators")
@added(Versions.v2025_11_15_preview)
Expand Down
4 changes: 4 additions & 0 deletions specification/ai/Foundry/evaluators/routes.tsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ alias ListEvaluatorVersionsParameters = {
@Http.query
type?: EvaluatorType | "all";

@doc("Filter evaluators by taxonomy ID. Returns only evaluators with definition.type='taxonomy' and matching taxonomy_id.")
@Http.query
taxonomyId?: string;

@doc("A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20.")
@Http.query
limit?: int32 = 20;
Expand Down
3 changes: 3 additions & 0 deletions specification/ai/Foundry/openai-evaluations/models.tsp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ model RedTeamTaxonomyItemGenerationParams extends RedTeamItemGenerationParams {

/** The source from which JSONL content is read. */
source: OpenAI.EvalJsonlFileContentSource;

/** Description of the application for context-aware seed generation. */
application_description?: string;
}

/** Represents the set of parameters used to control item generation operations. */
Expand Down
Loading