From e05b8b8d3cbb7ddc41eed960df3ce08717927d35 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Tue, 22 Jul 2025 14:11:01 -0700 Subject: [PATCH 01/50] Unified Evaluation API Spec (Version upgrade) --- .../evaluations/chat_messages.tsp | 117 + .../Azure.AI.Projects/evaluations/models.tsp | 601 ++- .../Azure.AI.Projects/evaluations/routes.tsp | 77 +- specification/ai/Azure.AI.Projects/main.tsp | 8 +- .../ai/Azure.AI.Projects/red-teams/models.tsp | 5 - .../ai/Azure.AI.Projects/red-teams/routes.tsp | 1 - .../2025-05-15-preview/azure-ai-projects.json | 43 +- .../2025-07-31-preview/azure-ai-projects.json | 4689 +++++++++++++++++ 8 files changed, 5428 insertions(+), 113 deletions(-) create mode 100644 specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json diff --git a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp new file mode 100644 index 000000000000..53fd391ed2d0 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp @@ -0,0 +1,117 @@ +import "@typespec/versioning"; +import "@azure-tools/typespec-azure-core"; +import "../common/models.tsp"; +import "../main.tsp"; +import "@typespec/openapi"; + +using TypeSpec.Versioning; + +namespace Azure.AI.Projects; + +@doc("Content for AI requests and responses.") +@discriminator("type") +@added(Versions.v2025_07_31_preview) +model AIContent { + @doc("The content of the message.") + type: "text" | "image_url" | "tool_call" | "tool_result" | string; +} + +@doc("Content for text messages in AI conversations.") +@added(Versions.v2025_07_31_preview) +model TextContent extends AIContent { + @doc("The content of the text message.") + type: "text"; + + @doc("The text content of the message.") + text: string; +} + +@doc("Content for image URL messages in AI conversations.") +@added(Versions.v2025_07_31_preview) +model ImageUrlContent extends AIContent { + @doc("The content of the image URL message.") + type: "image_url"; + + @doc("The URL of the image.") + imageUrl: string; +} + +@doc("Content for text messages in AI conversations.") +@added(Versions.v2025_07_31_preview) +model ToolCallContent extends AIContent { + @doc("The content of the tool call.") + type: "tool_call"; + + @doc("The name of the tool being called.") + name: string; + + @doc("The unique identifier of the tool call.") + toolCallId: string; + + @doc("The parameters for the tool call in JSON format.") + arguments: Record; +} + +@doc("Content for tool results in AI conversations.") +@added(Versions.v2025_07_31_preview) +model ToolResultContent extends AIContent { + @doc("The content of the tool result.") + type: "tool_result"; + + @doc("The result of the tool call in JSON format.") + results: Record; +} + +@doc("Abstract base model representing a single message in a conversation.") +@discriminator("role") +@added(Versions.v2025_05_15_preview) +model Message { + @doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.") + role: "system" | "assistant" | "developer" | "user" | string; + + #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." + @doc("The content of the message.") + @typeChangedFrom(Versions.v2025_07_31_preview, string) + content: AIContent[] | string; +} + +@doc("A message authored by the system to guide model behavior.") +@added(Versions.v2025_05_15_preview) +model SystemMessage extends Message { + @doc("Indicates this is a system message.") + role: "system"; +} + +@doc("A message authored by a developer to guide the model during evaluation.") +@added(Versions.v2025_05_15_preview) +model DeveloperMessage extends Message { + @doc("Indicates this is a developer message.") + role: "developer"; +} + +@doc("A message authored by the end user as input to the model.") +@added(Versions.v2025_05_15_preview) +model UserMessage extends Message { + @doc("Indicates this is a user message.") + role: "user"; +} + +@doc("A message generated by the assistant in response to previous messages.") +@added(Versions.v2025_05_15_preview) +model AssistantMessage extends Message { + @doc("Indicates this is an assistant message.") + role: "assistant"; +} + +@doc("Definition of a tool that can be used by the agent.") +@added(Versions.v2025_07_31_preview) +model AgentToolDefinition { + @doc("The name of the tool.") + name: string; + + @doc("The description of the tool.") + description?: string; + + @doc("The parameters for the tool in JSON Schema format.") + parameters: Record; +} diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index ab1ad5338c1a..c88c21b36f8e 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -6,22 +6,29 @@ import "@azure-tools/typespec-azure-core"; import "../common/models.tsp"; import "../main.tsp"; import "@typespec/openapi"; +import "./chat_messages.tsp"; using TypeSpec.Rest; using TypeSpec.Versioning; +using Azure.Core.Foundations; namespace Azure.AI.Projects; @doc("Evaluator Configuration") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model EvaluatorConfiguration { @doc("Identifier of the evaluator.") id: string; @doc("Initialization parameters of the evaluator.") + @removed(Versions.v2025_07_31_preview) initParams?: Record; + @doc("Initial parameters of the evaluator.") + @added(Versions.v2025_07_31_preview) + @renamedFrom(Versions.v2025_07_31_preview, "initParams") + initialParameters?: Record; + @doc("Data parameters of the evaluator.") dataMapping?: Record; } @@ -30,7 +37,6 @@ model EvaluatorConfiguration { @doc("Abstract data class.") @discriminator("type") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model InputData { @doc("Type of the data") type: string; @@ -38,7 +44,6 @@ model InputData { @doc("Dataset as source for evaluation.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model InputDataset extends InputData { type: "dataset"; @@ -46,10 +51,383 @@ model InputDataset extends InputData { id: string; } +@doc("Unique identifier of a dataset registered in AI Foundry workspace. The identifier follows the format: azureai://accounts/{resourceName}/projects/{projectName}/data/{datasetName}/versions/{versionNumber}") +@added(Versions.v2025_07_31_preview) +scalar DatasetId extends string; + +@doc("Specifies the type of data source used for evaluation. Different types support various data input methods and formats.") +@added(Versions.v2025_07_31_preview) +union EvaluationDataSourceType { + @doc("Use inline JSON data provided directly in the request") + InlineJsonData: "inlineJsonData", + + @doc("Use a dataset that has been registered and stored in AI Foundry workspace") + Dataset: "dataset", + + @doc("Use conversation data from a specific agent run execution") + FoundryAgentRun: "foundryAgentRun", + + @doc("Use data generated by running a model deployment against inline queries provided in the request") + FoundryModelDeploymentInline: "FoundryModelDeploymentInline", + + @doc("Use data generated by running a model deployment against a registered dataset") + FoundryModelDeploymentDataset: "foundryModelDeploymentDataset", + + string, +} + +@doc("Controls the level of data redaction applied to evaluation results to protect sensitive information and ensure compliance with privacy requirements.") +@added(Versions.v2025_07_31_preview) +union RedactionLevel { + @doc("Apply redaction to sensitive data in evaluation results. This is the recommended setting for production environments to ensure data privacy.") + Sensitive: "sensitive", + + @doc("Do not apply any data redaction to evaluation results. Use this setting only in development or testing environments where data privacy is not a concern.") + None: "none", + + string, +} + +#suppress "@azure-tools/typespec-azure-core/no-string-discriminator" +@doc("Base class for different types of evaluation data sources. Use the discriminator field 'type' to specify the specific data source implementation.") +@discriminator("type") +@added(Versions.v2025_07_31_preview) +model EvaluationDataSource { + @doc("Specifies the type of data source being used for evaluation") + type: EvaluationDataSourceType; +} + +@doc("Data source that uses conversation data from a specific agent run execution for evaluation. This allows evaluating agent performance based on actual conversation history.") +@added(Versions.v2025_07_31_preview) +model FoundryAgentRunSource extends EvaluationDataSource { + @doc("Specifies that this data source uses agent run data") + type: EvaluationDataSourceType.FoundryAgentRun; + + @doc("Unique identifier for the agent to evaluate.") + id: string; + + @doc("Unique identifier of the agent run to evaluate. Example: run_1234") + runId: string; + + @doc("Unique identifier of the conversation thread within the agent run. This is required when evaluating OpenAI-based agents or models to isolate specific conversation contexts.") + threadId?: string; +} + +@doc("Represents a message that contains a query and its corresponding response.") +@added(Versions.v2025_07_31_preview) +model QueryResponseMessage { + @doc("The query message sent to the model.") + query: string; + + @doc("The response generated by the model in reply to the query.") + response?: string; + + @doc("Optional context information that may include additional details about the query or response, such as metadata or processing instructions.") + context?: string; + + @doc("Optional ground truth value for the query, which can be used to compare against the model's response during evaluation.") + ground_truth?: string; +} + +@doc("The type of inline messages used for evaluation. This union allows for different inline message formats to be specified.") +@added(Versions.v2025_07_31_preview) +union InlineMessagesType { + @doc("Inline messages that consist of query-response pairs for evaluation. Each pair includes a query string and the corresponding response generated by the model.") + queryResponseMessage: "queryResponseMessage", + + @doc("role content type message") + roleContentType: "roleContentType", + + string, +} + +@doc("Available formats for structuring inline evaluation data.") +@added(Versions.v2025_07_31_preview) +union InlineDataFormat { + @doc("Query-response pairs for Q&A and chatbot evaluations") + queryResponseMessage: "queryResponseMessage", + + @doc("Multi-turn conversations with role-based messages and tool definitions") + chatMessages: "chatMessages", + + @doc("Flexible JSON format for custom data structures") + inlineJson: "inlineJson", + + string, +} + +@added(Versions.v2025_07_31_preview) +@doc("Base class for inline evaluation data with format discrimination.") +@discriminator("dataFormat") +model InlineData { + @doc("Format of the inline data structure") + dataFormat: InlineDataFormat; +} + +@doc("Query-response pairs for evaluating Q&A systems and response accuracy.") +@added(Versions.v2025_07_31_preview) +model QueryResponseInlineMessages extends InlineData { + @doc("Specifies query-response format") + dataFormat: InlineDataFormat.queryResponseMessage; + + @doc("Array of query-response pairs with optional context and ground truth") + items: QueryResponseMessage[]; +} + +@added(Versions.v2025_07_31_preview) +@doc("Multi-turn conversations for evaluating dialogue systems and context awareness.") +model ChatMessages extends InlineData { + @doc("Specifies chat messages format") + dataFormat: InlineDataFormat.chatMessages; + + @doc("Array of messages representing representing queries") + query: Message[]; + + @doc("Array of messages representing responses") + response: Message[]; + + @doc("Array of tool definitions that are used in the conversation") + toolDefinitions: AgentToolDefinition[]; +} + +@added(Versions.v2025_07_31_preview) +@doc("Custom JSON format for complex evaluation scenarios requiring flexible data structures.") +model InlineJson extends InlineData { + @doc("Specifies JSON format") + dataFormat: InlineDataFormat.inlineJson; + + @doc("Array of JSON strings with custom fields and metadata") + items: string[]; +} + +@doc("Data source using inline data provided directly in the request.") +@added(Versions.v2025_07_31_preview) +model InlineJsonDataSource extends EvaluationDataSource { + @doc("Specifies inline JSON data source") + type: EvaluationDataSourceType.InlineJsonData; + + @doc("Optional unique identifier for the inline data source. This can be an agent id or a custom identifier to distinguish between different inline data sources.") + id?: string; + + @doc("Inline data structured according to the specified format") + data: InlineData; +} + +@doc("Data source that uses a dataset registered and stored in AI Foundry workspace. This is the recommended approach for large datasets or reusable evaluation data.") +@added(Versions.v2025_07_31_preview) +model FoundryDatasetDataSource extends EvaluationDataSource { + @doc("Specifies that this data source uses a registered dataset") + type: EvaluationDataSourceType.Dataset; + + @doc("Unique identifier of the dataset registered in AI Foundry workspace") + datasetId: DatasetId; +} + +@doc("Comprehensive configuration for a model deployment used in evaluation scenarios. This defines how the model will process inputs and generate responses for evaluation.") +@added(Versions.v2025_07_31_preview) +model EvaluationModelConfiguration { + @doc("The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'.") + modelDeploymentName: string; + + @doc("Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.") + modelParameters?: Record; +} + +@doc("Common properties shared across model deployment configurations used in evaluations.") +@added(Versions.v2025_07_31_preview) +model EvaluationModelSourceCommon { + @doc("Configuration for the model deployment used in evaluation.") + modelConfiguration: EvaluationModelConfiguration; + + @doc("A list of messages comprising the conversation so far. Each message can be a json string with role and content to specify the conversation context.") + baseMessages: Message[]; +} + +@doc("Data source that uses a model deployment with inline queries. The specified model processes each query to generate responses, which are then evaluated against the configured evaluators.") +@added(Versions.v2025_07_31_preview) +model FoundryModelInlineSource extends EvaluationDataSource { + @doc("Specifies that this data source uses a model deployment with inline queries") + type: EvaluationDataSourceType.FoundryModelDeploymentInline; + + ...EvaluationModelSourceCommon; + + @doc("Inline queries to be processed by the model deployment. The response is then evaluated against the configured evaluators.") + queries: Array; +} + +@doc("Data source that uses a model deployment with a dataset containing prompts. The model processes each prompt from the dataset to generate responses, which are then evaluated against the configured evaluators.") +@added(Versions.v2025_07_31_preview) +model FoundryModelDatasetSource extends EvaluationDataSource { + @doc("Specifies that this data source uses a model deployment with a dataset") + type: EvaluationDataSourceType.FoundryModelDeploymentDataset; + + ...EvaluationModelSourceCommon; + + @doc("Unique identifier of the dataset containing prompts that will be processed by the model deployment") + datasetId: DatasetId; + + @doc("Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '{{item.messages}}'") + queryField: string; +} + +@doc("Statistical aggregation methods available for summarizing evaluation results across multiple data points.") +@added(Versions.v2025_07_31_preview) +union AggregationMethod { + @doc("Calculate the sum of all values") + sum: "sum", + + @doc("Calculate the arithmetic mean of all values") + average: "average", + + @doc("Find the maximum value") + max: "max", + + @doc("Find the minimum value") + min: "min", + + @doc("Count the total number of values") + count: "count", + + @doc("Calculate the nth percentile value. Needs a parameter to specify the percentile level (e.g., level: 95 for 95th percentile analysis)") + percentile: "percentile", + + @doc("Calculate the standard deviation") + standardDeviation: "standardDeviation", + + string, +} + +@doc("Configuration for a specific aggregation method, allowing customization of how the aggregation is calculated.") +@added(Versions.v2025_07_31_preview) +model AggregationConfiguration { + @doc("The statistical aggregation method to apply (e.g., 'average', 'percentile', 'standardDeviation')") + method: AggregationMethod; + + @doc("Optional parameters specific to the aggregation method. For example, 'populationVariance: true' for standard deviation, or 'interpolation: linear' for percentiles.") + parameters?: Record; +} + +@doc("Represents system Metadata about the evaluation execution environment and resource usage.") +@added(Versions.v2025_07_31_preview) +model EvaluatorResultSystemData { + @doc("Total number of input tokens consumed during the evaluation process. This helps track resource usage and costs.") + inputTokenCount: int64; + + @doc("Total number of output tokens generated during the evaluation process. This helps track resource usage and costs.") + outputTokenCount: int64; + + @doc("Total time taken to complete the evaluation, measured in milliseconds. This includes processing time and any network latency.") + duration: int64; + + @doc("Additional details about the evaluator result") + additionalDetails: Record; +} + +@doc("Common properties shared across all evaluator results, including metadata about the evaluation execution and resource consumption.") +@added(Versions.v2025_07_31_preview) +model EvaluatorResultCommon { + @doc("Unique identifier of the evaluator that produced this result") + evaluatorId: string; + + @doc("Metadata about the evaluation execution environment and resource usage.") + systemData?: EvaluatorResultSystemData; + + @doc("Current state of the evaluation execution. Tracks progress from initiation to completion or failure.") + state: OperationState; +} + +@added(Versions.v2025_07_31_preview) +@doc("Represents an aggregated score calculated using a specific statistical method across all rows of data.") +model AggregatedScore { + @doc("The calculated aggregated value based on the specified method. This value is undefined in case aggregation is not applicable or the method does not produce a numeric result.") + value?: float32; + + @doc("Additional metadata about the aggregation calculation") + metadata?: Record; +} + +@added(Versions.v2025_07_31_preview) +@doc("Statistical summary of evaluation results including aggregated scores and pass/fail rates.") +model EvaluationSummaryStatistics { + @doc("Aggregated scores calculated using specified aggregation methods. This provides a summary of the evaluator's performance across all data rows.") + aggregatedScores: Record; + + @doc("Value between 0 and 1 representing the proportion of rows where outcome is 'pass'.") + passRate: float32; + + @doc("Total number of rows that were evaluated.") + sampleCount: int64; +} + +@doc("Aggregated summary of evaluation results for a specific evaluator across all data rows. This provides a high-level overview of the evaluator's performance.") +@added(Versions.v2025_07_31_preview) +model EvaluatorResultSummary { + ...EvaluatorResultCommon; + + @doc("Statistical summary of evaluation results.") + statistics?: EvaluationSummaryStatistics; +} + +@doc("Specifies the type of external storage destination where evaluation results can be exported for further analysis or long-term retention.") +@added(Versions.v2025_07_31_preview) +union EvaluationDestinationConfigurationType { + @doc("Export evaluation results to Azure Application Insights for monitoring and analytics") + AppInsightsStorage: "appInsightsStorage", + + @doc("Export evaluation results to Azure Blob Storage for data archival and custom processing") + StorageAccountStorage: "storageAccountStorage", + + string, +} + +@doc("Base configuration for exporting evaluation results to external storage destinations. This enables integration with external analytics and monitoring systems.") +@added(Versions.v2025_07_31_preview) +@discriminator("type") +model EvaluationDestinationConfiguration { + @doc("Specifies the type of external storage destination") + type: EvaluationDestinationConfigurationType; + + @doc("Name of the connection resource configured in the AI Foundry workspace. This connection must be properly configured with appropriate credentials before use.") + connectionName: string; + + @doc("Current state of the export operation to this destination. This field is managed by the service and cannot be modified by clients.") + @visibility(Lifecycle.Read) + state?: OperationState; +} + +@doc("Configuration for exporting evaluation results to Azure Application Insights. This enables monitoring, alerting, and analytics on evaluation performance and trends.") +@added(Versions.v2025_07_31_preview) +model AppInsightsDestinationConfiguration + extends EvaluationDestinationConfiguration { + @doc("Specifies Azure Application Insights as the export destination") + type: EvaluationDestinationConfigurationType.AppInsightsStorage; +} + +@doc("Configuration for exporting evaluation results to Azure Blob Storage. This provides long-term storage and enables custom data processing workflows.") +@added(Versions.v2025_07_31_preview) +model StorageAccountDestinationConfiguration + extends EvaluationDestinationConfiguration { + @doc("Specifies Azure Blob Storage as the export destination") + type: EvaluationDestinationConfigurationType.StorageAccountStorage; + + @doc("Name of the container within the Azure Blob Storage account where evaluation results will be stored") + containerName: string; +} + +@doc("Configuration settings that control how evaluation results are processed, stored, and exported. These settings affect data privacy, retention, and integration with external systems.") +@added(Versions.v2025_07_31_preview) +model EvaluationResultSettings { + @doc("Level of data redaction to apply to evaluation results. Defaults to 'sensitive' in production environments to protect confidential information.") + redactionLevel?: RedactionLevel; + + @doc("List of external storage destinations where evaluation results should be exported in addition to the default AI Foundry storage") + @added(Versions.v2025_07_31_preview) + additionalDestinations?: Array; +} + @doc("Evaluation Definition") @resource("runs") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model Evaluation { @doc("Identifier of the evaluation.") @key("name") @@ -58,8 +436,17 @@ model Evaluation { name: string; @doc("Data for evaluation.") + @removed(Versions.v2025_07_31_preview) // Replaced by `dataSource` data: InputData; + @doc("Data source configuration that specifies where the evaluation data comes from. This replaces the legacy 'data' field and provides more flexible data source options.") + @added(Versions.v2025_07_31_preview) + dataSource: EvaluationDataSource; + + @doc("Configuration settings that control how evaluation results are processed, stored, and exported") + @added(Versions.v2025_07_31_preview) + resultSettings?: EvaluationResultSettings; + @doc("Display Name for evaluation. It helps to find the evaluation easily in AI Foundry. It does not need to be unique.") displayName?: string; @@ -68,8 +455,14 @@ model Evaluation { @doc("Status of the evaluation. It is set by service and is read-only.") @visibility(Lifecycle.Read) + @removed(Versions.v2025_07_31_preview) // Replaced by `state` in 2025-07-31-preview status?: string; + @doc("Current operational state of the evaluation. This field is managed by the service and reflects the evaluation's progress from initiation to completion.") + @visibility(Lifecycle.Read) + @added(Versions.v2025_07_31_preview) + state: OperationState; + @doc("Evaluation's tags. Unlike properties, tags are fully mutable.") tags?: Record; @@ -79,13 +472,42 @@ model Evaluation { @doc("Evaluators to be used for the evaluation.") evaluators: Record; + @doc(""" + Statistical aggregation methods to apply when summarizing evaluation results. Specify which aggregation functions should be calculated across all evaluation scores. For example, use 'average' to get mean scores, 'percentile' with parameter level: 95 for 95th percentile analysis, or 'standardDeviation' to measure score variability. + If not specified, average and count aggregations will be applied by default. + Key is the any label for the aggregation, which can be used to identify it in results. + Available methods: sum, average, max, min, count, percentile, standardDeviation. + Note: Aggregations are only calculated for evaluators that produce numeric scores. Non-numeric evaluators will not contribute to these aggregations. + """) + @added(Versions.v2025_07_31_preview) + aggregations?: Record; + @doc("Specifies the type and configuration of the entity used for this evaluation.") + @removed(Versions.v2025_07_31_preview) target?: EvaluationTarget; + + @doc("Aggregated summary of evaluation results for each configured evaluator. This provides a high-level overview of performance across all data rows and is available once the evaluation completes.") + @visibility(Lifecycle.Read) + @added(Versions.v2025_07_31_preview) + summary?: Record; + + @doc(""" + Unique identifier of the dataset containing detailed evaluation results. This dataset is created automatically upon evaluation completion and contains row-by-row results for analysis. + The identifier follows the format: azureai://accounts/{resourceName}/projects/{projectName}/datasets/{datasetName}/versions/{versionNumber}. + Note: This dataset is only available when the result retention policy permits detailed result storage. + """) + @added(Versions.v2025_07_31_preview) + @visibility(Lifecycle.Read) + resultDatasetId?: DatasetId; + + @doc("System-generated metadata containing internal service information for debugging and operational purposes. This field is managed by the service and cannot be modified by clients.") + @visibility(Lifecycle.Read) + @added(Versions.v2025_07_31_preview) + systemData?: Record; } @doc("Definition for sampling strategy.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model AgentEvaluationSamplingConfiguration { @doc("Name of the sampling strategy.") name: string; @@ -99,7 +521,6 @@ model AgentEvaluationSamplingConfiguration { @doc("The redaction configuration will allow the user to control what is redacted.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model AgentEvaluationRedactionConfiguration { @doc("Redact score properties. If not specified, the default is to redact in production.") redactScoreProperties?: boolean; @@ -107,7 +528,6 @@ model AgentEvaluationRedactionConfiguration { @doc("Evaluation request for agent run.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model AgentEvaluationRequest { @doc("Identifier of the agent run.") runId: string; @@ -130,7 +550,6 @@ model AgentEvaluationRequest { @doc("Result for the agent evaluation evaluator run.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model AgentEvaluationResult { @doc("Evaluator's name. This is the name of the evaluator that was used to evaluate the agent's completion.") evaluator: string; @@ -165,7 +584,6 @@ model AgentEvaluationResult { @doc("Evaluation response for agent evaluation run.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model AgentEvaluation { @doc("Identifier of the agent evaluation run.") id: string; @@ -180,62 +598,9 @@ model AgentEvaluation { result?: Array; } -@doc("Abstract base model representing a single message in a conversation.") -@discriminator("role") -@removed(Versions.v1) -@added(Versions.v2025_05_15_preview) -model Message { - @doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.") - role: "system" | "assistant" | "developer" | "user" | string; -} - -@doc("A message authored by the system to guide model behavior.") -@removed(Versions.v1) -@added(Versions.v2025_05_15_preview) -model SystemMessage extends Message { - @doc("Indicates this is a system message.") - role: "system"; - - @doc("Plain text instructions provided by the system to steer model behavior.") - content: string; -} - -@doc("A message authored by a developer to guide the model during evaluation.") -@removed(Versions.v1) -@added(Versions.v2025_05_15_preview) -model DeveloperMessage extends Message { - @doc("Indicates this is a developer message.") - role: "developer"; - - @doc("Content provided by a developer to guide model behavior in an evaluation context.") - content: string; -} - -@doc("A message authored by the end user as input to the model.") -@removed(Versions.v1) -@added(Versions.v2025_05_15_preview) -model UserMessage extends Message { - @doc("Indicates this is a user message.") - role: "user"; - - @doc("Input content or question provided by the end user.") - content: string; -} - -@doc("A message generated by the assistant in response to previous messages.") -@removed(Versions.v1) -@added(Versions.v2025_05_15_preview) -model AssistantMessage extends Message { - @doc("Indicates this is an assistant message.") - role: "assistant"; - - @doc("Response content generated by the assistant.") - content: string; -} - @doc("Allowed types of evaluation targets.") -@removed(Versions.v1) @added(Versions.v2025_05_15_preview) +@removed(Versions.v2025_07_31_preview) union EvaluationTargetType { @doc("Evaluation target that uses a model for response generation.") modelResponseGeneration: "modelResponseGeneration", @@ -245,16 +610,16 @@ union EvaluationTargetType { @doc("Abstract base model for defining evaluation targets.") @discriminator("type") -@removed(Versions.v1) @added(Versions.v2025_05_15_preview) +@removed(Versions.v2025_07_31_preview) model EvaluationTarget { @doc("Discriminator that defines the type of the evaluation target.") type: EvaluationTargetType; } @doc("Evaluation target for generating responses using a given model and dataset.") -@removed(Versions.v1) @added(Versions.v2025_05_15_preview) +@removed(Versions.v2025_07_31_preview) model modelResponseGenerationTarget extends EvaluationTarget { @doc("The type of evaluation target. Always 'modelResponseGeneration'.") type: EvaluationTargetType.modelResponseGeneration; @@ -268,3 +633,109 @@ model modelResponseGenerationTarget extends EvaluationTarget { @doc("Optional parameters passed to the model for evaluation.") modelParams: Record; } + +@doc("Defines the possible outcomes of an evaluation result, indicating whether the evaluated data meets the specified criteria or standards.") +@added(Versions.v2025_07_31_preview) +union EvaluationResultOutcome { + @doc("The evaluation completed successfully and met the specified criteria or threshold") + pass: "pass", + + @doc("The evaluation completed but did not meet the specified criteria or threshold") + fail: "fail", + + @doc("The evaluation has not been processed yet or is still in progress") + notEvaluated: "notEvaluated", + + @doc("The evaluation criteria do not apply to this particular data row or context") + notApplicable: "notApplicable", + + string, +} + +@doc("Detailed result of a specific evaluator's assessment on a single data row. Contains the evaluation outcome, score, reasoning, and additional metrics.") +@added(Versions.v2025_07_31_preview) +model EvaluatorResult { + ...EvaluatorResultCommon; + + @doc("Score for the evaluation result") + score: EvaluationScore; +} + +@doc("Describes the desirable direction for the evaluation score. This indicates whether a higher or lower score is preferred for this evaluator.") +@added(Versions.v2025_07_31_preview) +union EvaluatorDesirableDirection { + @doc("Indicates that a higher score is desirable for this evaluator") + increase: "increase", + + @doc("Indicates that a lower score is desirable for this evaluator") + decrease: "decrease", + + @doc("Indicates that the score should be neutral, meaning it does not have a preferred direction") + neutral: "neutral", + + string, +} + +@added(Versions.v2025_07_31_preview) +@doc("Type of score assigned by the evaluator. This indicates whether the score is boolean, continuous, or ordinal.") +union EvaluatorScoreType { + @doc("Boolean score type, where the score is either true or false") + boolean: "boolean", + + @doc("Continuous score type, where the score is a floating-point number representing a continuous value") + continuous: "continuous", + + @doc("Ordinal score type, where the score represents an ordered category or rank") + ordinal: "ordinal", + + string, +} + +@added(Versions.v2025_07_31_preview) +@doc("Score assigned by the evaluator to a specific data row") +model EvaluationScore { + #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." + @doc("Score of the evaluation.") + label?: boolean | string | float64; + + @doc("Justification for the score, providing context on how it was derived or what it represents") + justification: string; + + @doc("Threshold value that this score is compared against to determine the evaluation outcome") + threshold?: float32; + + @doc("Direction of the score that indicates whether a higher or lower score is desirable for this evaluator") + desirableDirection?: EvaluatorDesirableDirection; + + @doc("Type of the score that indicates whether it is a boolean, continuous, or ordinal score") + scoreType?: EvaluatorScoreType; + + @doc("Outcome of the evaluation based on the score and threshold. Indicates whether the score meets, exceeds, or falls below expectations") + outcome?: EvaluationResultOutcome; + + @doc("Optional additional details that may include metadata or other relevant information about the score") + additionalMetrics?: Record; +} + +@doc("Comprehensive evaluation results for a single data row, including the input data, processing state, and results from all configured evaluators.") +@added(Versions.v2025_07_31_preview) +@resource("runs/{evaluationId}/results") +model EvaluationResult { + @doc("Unique identifier for this evaluation result row") + @key("id") + @visibility(Lifecycle.Read) + id: string; + + @doc("Current processing state of this data row. Tracks progress from initial processing through completion or failure.") + state: OperationState; + + @doc("Error message describing why the evaluation failed for this data row, if applicable") + error?: string; + + @doc("Original input data for this row in JSON string format. This preserves the exact data that was evaluated for reference and debugging.") + @added(Versions.v2025_07_31_preview) + inputDataJson?: string; + + @doc("Collection of evaluation results from each configured evaluator for this data row. The key is the evaluator identifier and the value contains the detailed evaluation results.") + evaluatorResults: Record; +} diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 19c42276ebeb..b2852bbdc54d 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -5,6 +5,7 @@ import "@azure-tools/typespec-azure-core"; import "./models.tsp"; using TypeSpec.Http; +using Azure.Core; using Azure.Core.Traits; using TypeSpec.Versioning; @@ -14,21 +15,39 @@ alias ServiceTraits = SupportsClientRequestId & NoRepeatableRequests & NoConditionalRequests; +alias RetryServiceTrait = { + @header("Retry-After") + @doc("Recommended time interval in seconds before making another request when the evaluation is still processing. This helps prevent excessive polling and reduces server load during long-running evaluations.") + @added(Versions.v2025_07_31_preview) + @visibility(Lifecycle.Read) + retryAfter?: string; +}; + +alias DisplayPreferencesTypeParameterTrait = { + @doc("Filter expression to narrow down the list of evaluations based on specific criteria. Supports filtering by properties such as status, tags, or properties to help manage large evaluation collections.") + @query + @added(Versions.v2025_07_31_preview) + filter?: string; +}; + alias EvaluationsOperations = Azure.Core.ResourceOperations; @route("evaluations") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) interface Evaluations { @doc("Get an evaluation run by name.") - get is EvaluationsOperations.ResourceRead; + get is EvaluationsOperations.ResourceRead; @doc("List evaluation runs") - list is EvaluationsOperations.ResourceList; + list is EvaluationsOperations.ResourceList< + Evaluation, + ListQueryParametersTrait + >; #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates an evaluation run.") @route("runs:run") + @removed(Versions.v2025_07_31_preview) @post create is Azure.Core.Foundations.Operation< { @@ -42,6 +61,7 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates an agent evaluation run.") @route("runs:runAgent") + @removed(Versions.v2025_07_31_preview) // Use createOrReplace instead @post createAgentEvaluation is Azure.Core.Foundations.Operation< { @@ -52,6 +72,57 @@ interface Evaluations { ResourceCreatedResponse >; + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" + @doc("Creates a new evaluation with the specified configuration.") + @added(Versions.v2025_07_31_preview) + @route("runs") + @post + createEvaluation is Azure.Core.Foundations.Operation< + { + @header("Repeatability-Request-ID") + @doc("Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.") + repeatabilityRequestId?: string; + + @doc("Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.") + @header("Repeatability-First-Sent") + repeatabilityFirstSent?: utcDateTime; + + @doc("Complete evaluation configuration including data source, evaluators, and result settings") + @body + evaluation: Evaluation; + }, + ResourceCreatedResponse + >; + + @doc("Updates specific properties of an existing evaluation. Supports modification of metadata fields including description, display name, and tags. Note: Core evaluation configuration such as data sources and evaluators cannot be modified after creation.") + @added(Versions.v2025_07_31_preview) + update is EvaluationsOperations.ResourceUpdate; + + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" + @doc("Retrieves detailed row-by-row evaluation results with pagination support. This endpoint provides access to individual evaluation outcomes, scores, and reasoning for each data row processed.") + @route("runs/{name}/results") + @get + @added(Versions.v2025_07_31_preview) + getEvaluationResults( + @doc("Unique identifier of the evaluation whose results are being retrieved") + @path + name: string, + + @doc("API version specifier for this operation") + @query("api-version") + apiVersion: string, + + @query + @doc("Maximum number of result items to return in a single response. Must be a multiple of 10 (e.g., 10, 20, 100, 1000). Defaults to 10 if not specified.") + top?: int32, + + @query + @doc("Number of result items to skip before returning results, enabling pagination through large result sets. Must be a multiple of 10. Defaults to 0.") + skip?: int32 = 0, + + ...ClientRequestIdHeader, + ): Azure.Core.Page; + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Cancel an evaluation run by name") @post diff --git a/specification/ai/Azure.AI.Projects/main.tsp b/specification/ai/Azure.AI.Projects/main.tsp index eb7edfc0664c..8e5165c4e593 100644 --- a/specification/ai/Azure.AI.Projects/main.tsp +++ b/specification/ai/Azure.AI.Projects/main.tsp @@ -50,12 +50,16 @@ namespace Azure.AI.Projects { @useDependency(Azure.Core.Versions.v1_0_Preview_2) v2025_05_01: "2025-05-01", + @doc("Azure AI API version v1.") + @useDependency(Azure.Core.Versions.v1_0_Preview_2) + v1: "v1", + @doc("Azure AI API version 2025-05-15-preview.") @useDependency(Azure.Core.Versions.v1_0_Preview_2) v2025_05_15_preview: "2025-05-15-preview", - @doc("Azure AI API version v1.") + @doc("Azure AI API version 2025-07-31-preview.") @useDependency(Azure.Core.Versions.v1_0_Preview_2) - v1: "v1", + v2025_07_31_preview: "2025-07-31-preview", } } diff --git a/specification/ai/Azure.AI.Projects/red-teams/models.tsp b/specification/ai/Azure.AI.Projects/red-teams/models.tsp index b6ec4eb4dc58..f43edf233dbc 100644 --- a/specification/ai/Azure.AI.Projects/red-teams/models.tsp +++ b/specification/ai/Azure.AI.Projects/red-teams/models.tsp @@ -13,7 +13,6 @@ namespace Azure.AI.Projects; @doc("Strategies for attacks.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) union AttackStrategy { string, @@ -92,7 +91,6 @@ union AttackStrategy { @doc("Risk category for the attack objective.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) union RiskCategory { string, @@ -111,7 +109,6 @@ union RiskCategory { @doc("Azure OpenAI model configuration. The API version would be selected by the service for querying the model.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model AzureOpenAIModelConfiguration extends TargetConfig { @visibility(Lifecycle.Read) type: "AzureOpenAIModel"; @@ -122,7 +119,6 @@ model AzureOpenAIModelConfiguration extends TargetConfig { @doc("Abstract class for target configuration.") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) @discriminator("type") model TargetConfig { @doc("Type of the model configuration.") @@ -132,7 +128,6 @@ model TargetConfig { @doc("Red team details.") @resource("runs") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) model RedTeam { @doc("Identifier of the red team run.") @key("name") diff --git a/specification/ai/Azure.AI.Projects/red-teams/routes.tsp b/specification/ai/Azure.AI.Projects/red-teams/routes.tsp index 9fcdf116796d..cd29328cfdf6 100644 --- a/specification/ai/Azure.AI.Projects/red-teams/routes.tsp +++ b/specification/ai/Azure.AI.Projects/red-teams/routes.tsp @@ -18,7 +18,6 @@ alias RedTeamOperations = Azure.Core.ResourceOperations; @route("redTeams") @added(Versions.v2025_05_15_preview) -@removed(Versions.v1) interface RedTeams { @doc("Get a redteam by name.") get is RedTeamOperations.ResourceRead; diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index b4cfc17440bb..6563d5e464d7 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1698,15 +1698,6 @@ "AssistantMessage": { "type": "object", "description": "A message generated by the assistant in response to previous messages.", - "properties": { - "content": { - "type": "string", - "description": "Response content generated by the assistant." - } - }, - "required": [ - "content" - ], "allOf": [ { "$ref": "#/definitions/Message" @@ -2400,15 +2391,6 @@ "DeveloperMessage": { "type": "object", "description": "A message authored by a developer to guide the model during evaluation.", - "properties": { - "content": { - "type": "string", - "description": "Content provided by a developer to guide model behavior in an evaluation context." - } - }, - "required": [ - "content" - ], "allOf": [ { "$ref": "#/definitions/Message" @@ -2796,11 +2778,16 @@ "x-ms-enum": { "modelAsString": true } + }, + "content": { + "type": "string", + "description": "The content of the message." } }, "discriminator": "role", "required": [ - "role" + "role", + "content" ] }, "ModelDeployment": { @@ -3281,15 +3268,6 @@ "SystemMessage": { "type": "object", "description": "A message authored by the system to guide model behavior.", - "properties": { - "content": { - "type": "string", - "description": "Plain text instructions provided by the system to steer model behavior." - } - }, - "required": [ - "content" - ], "allOf": [ { "$ref": "#/definitions/Message" @@ -3314,15 +3292,6 @@ "UserMessage": { "type": "object", "description": "A message authored by the end user as input to the model.", - "properties": { - "content": { - "type": "string", - "description": "Input content or question provided by the end user." - } - }, - "required": [ - "content" - ], "allOf": [ { "$ref": "#/definitions/Message" diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json new file mode 100644 index 000000000000..41997dbd0151 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -0,0 +1,4689 @@ +{ + "swagger": "2.0", + "info": { + "title": "Azure AI", + "version": "2025-07-31-preview", + "x-typespec-generated": [ + { + "emitter": "@azure-tools/typespec-autorest" + } + ] + }, + "schemes": [ + "https" + ], + "x-ms-parameterized-host": { + "hostTemplate": "{endpoint}", + "useSchemePrefix": false, + "parameters": [ + { + "name": "endpoint", + "in": "path", + "description": "Project endpoint. In the form \"https://.services.ai.azure.com/api/projects/_project\"\nif your Foundry Hub has only one Project, or to use the default Project in your Hub. Or in the form \n\"https://.services.ai.azure.com/api/projects/\" if you want to explicitly\nspecify the Foundry Project name.", + "required": true, + "type": "string", + "format": "uri", + "x-ms-skip-url-encoding": true + } + ] + }, + "produces": [ + "application/json" + ], + "consumes": [ + "application/json" + ], + "security": [ + { + "OAuth2Auth": [ + "https://ai.azure.com/.default" + ] + } + ], + "securityDefinitions": { + "OAuth2Auth": { + "type": "oauth2", + "flow": "implicit", + "authorizationUrl": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", + "scopes": { + "https://ai.azure.com/.default": "" + } + } + }, + "tags": [], + "paths": { + "/connections": { + "get": { + "operationId": "Connections_List", + "description": "List all connections in the project, without populating connection credentials", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "connectionType", + "in": "query", + "description": "List connections of this specific type", + "required": false, + "type": "string", + "enum": [ + "AzureOpenAI", + "AzureBlob", + "AzureStorageAccount", + "CognitiveSearch", + "CosmosDB", + "ApiKey", + "AppConfig", + "AppInsights", + "CustomKeys" + ], + "x-ms-enum": { + "name": "ConnectionType", + "modelAsString": true, + "values": [ + { + "name": "AzureOpenAI", + "value": "AzureOpenAI", + "description": "Azure OpenAI Service" + }, + { + "name": "AzureBlobStorage", + "value": "AzureBlob", + "description": "Azure Blob Storage, with specified container" + }, + { + "name": "AzureStorageAccount", + "value": "AzureStorageAccount", + "description": "Azure Blob Storage, with container not specified (used by Agents)" + }, + { + "name": "AzureAISearch", + "value": "CognitiveSearch", + "description": "Azure AI Search" + }, + { + "name": "CosmosDB", + "value": "CosmosDB", + "description": "CosmosDB" + }, + { + "name": "APIKey", + "value": "ApiKey", + "description": "Generic connection that uses API Key authentication" + }, + { + "name": "ApplicationConfiguration", + "value": "AppConfig", + "description": "Application Configuration" + }, + { + "name": "ApplicationInsights", + "value": "AppInsights", + "description": "Application Insights" + }, + { + "name": "Custom", + "value": "CustomKeys", + "description": "Custom Keys" + } + ] + } + }, + { + "name": "defaultConnection", + "in": "query", + "description": "List connections that are default connections", + "required": false, + "type": "boolean" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedConnection" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + } + }, + "/connections/{name}": { + "get": { + "operationId": "Connections_Get", + "description": "Get a connection by name, without populating connection credentials", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The friendly name of the connection, provided by the user.", + "required": true, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/Connection" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/connections/{name}/getConnectionWithCredentials": { + "post": { + "operationId": "Connections_GetWithCredentials", + "description": "Get a connection by name, with its connection credentials", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The friendly name of the connection, provided by the user.", + "required": true, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/Connection" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/datasets": { + "get": { + "operationId": "Datasets_ListLatest", + "description": "List the latest version of each DatasetVersion", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedDatasetVersion" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + } + }, + "/datasets/{name}/versions": { + "get": { + "operationId": "Datasets_ListVersions", + "description": "List all versions of the given DatasetVersion", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedDatasetVersion" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + } + }, + "/datasets/{name}/versions/{version}": { + "get": { + "operationId": "Datasets_GetVersion", + "description": "Get the specific version of the DatasetVersion", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + }, + { + "name": "version", + "in": "path", + "description": "The specific version id of the DatasetVersion to retrieve.", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/DatasetVersion" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + }, + "patch": { + "operationId": "Datasets_CreateOrUpdateVersion", + "description": "Create a new or update an existing DatasetVersion with the given version id", + "consumes": [ + "application/merge-patch+json" + ], + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + }, + { + "name": "version", + "in": "path", + "description": "The specific version id of the DatasetVersion to create or update.", + "required": true, + "type": "string" + }, + { + "name": "datasetVersion", + "in": "body", + "description": "The DatasetVersion to create or update.", + "required": true, + "schema": { + "$ref": "#/definitions/DatasetVersion" + } + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/DatasetVersion" + } + }, + "201": { + "description": "The request has succeeded and a new resource has been created as a result.", + "schema": { + "$ref": "#/definitions/DatasetVersion" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + }, + "delete": { + "operationId": "Datasets_DeleteVersion", + "description": "Delete the specific version of the DatasetVersion", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + }, + { + "name": "version", + "in": "path", + "description": "The version of the DatasetVersion to delete.", + "required": true, + "type": "string" + } + ], + "responses": { + "204": { + "description": "There is no content to send for this request, but the headers may be useful." + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/datasets/{name}/versions/{version}/credentials": { + "post": { + "operationId": "Datasets_GetCredentials", + "description": "Get the SAS credential to access the storage account associated with a Dataset version.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + }, + { + "name": "version", + "in": "path", + "description": "The specific version id of the DatasetVersion to operate on.", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/AssetCredentialResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/datasets/{name}/versions/{version}/startPendingUpload": { + "post": { + "operationId": "Datasets_StartPendingUploadVersion", + "description": "Start a new or get an existing pending upload of a dataset for a specific version.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + }, + { + "name": "version", + "in": "path", + "description": "The specific version id of the DatasetVersion to operate on.", + "required": true, + "type": "string" + }, + { + "name": "pendingUploadRequest", + "in": "body", + "description": "The pending upload request parameters", + "required": true, + "schema": { + "$ref": "#/definitions/PendingUploadRequest" + } + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PendingUploadResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/deployments": { + "get": { + "operationId": "Deployments_List", + "description": "List all deployed models in the project", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "modelPublisher", + "in": "query", + "description": "Model publisher to filter models by", + "required": false, + "type": "string" + }, + { + "name": "modelName", + "in": "query", + "description": "Model name (the publisher specific name) to filter models by", + "required": false, + "type": "string" + }, + { + "name": "deploymentType", + "in": "query", + "description": "Type of deployment to filter list by", + "required": false, + "type": "string", + "enum": [ + "ModelDeployment" + ], + "x-ms-enum": { + "name": "DeploymentType", + "modelAsString": true, + "values": [ + { + "name": "ModelDeployment", + "value": "ModelDeployment", + "description": "Model deployment" + } + ] + } + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedDeployment" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + } + }, + "/deployments/{name}": { + "get": { + "operationId": "Deployments_Get", + "description": "Get a deployed model.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "Name of the deployment", + "required": true, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/Deployment" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/evaluations/runs": { + "get": { + "operationId": "Evaluations_List", + "description": "List evaluation runs", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "filter", + "in": "query", + "description": "Filter expression to narrow down the list of evaluations based on specific criteria. Supports filtering by properties such as status, tags, or properties to help manage large evaluation collections.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedEvaluation" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + }, + "post": { + "operationId": "Evaluations_CreateEvaluation", + "description": "Creates a new evaluation with the specified configuration.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "Repeatability-Request-ID", + "in": "header", + "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", + "required": false, + "type": "string", + "x-ms-client-name": "repeatabilityRequestId" + }, + { + "name": "Repeatability-First-Sent", + "in": "header", + "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", + "required": false, + "type": "string", + "format": "date-time", + "x-ms-client-name": "repeatabilityFirstSent" + }, + { + "name": "evaluation", + "in": "body", + "description": "Complete evaluation configuration including data source, evaluators, and result settings", + "required": true, + "schema": { + "$ref": "#/definitions/Evaluation" + } + } + ], + "responses": { + "201": { + "description": "The request has succeeded and a new resource has been created as a result.", + "schema": { + "$ref": "#/definitions/Evaluation" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/evaluations/runs/{name}": { + "get": { + "operationId": "Evaluations_Get", + "description": "Get an evaluation run by name.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "Identifier of the evaluation.", + "required": true, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/Evaluation" + }, + "headers": { + "Retry-After": { + "type": "string", + "description": "Recommended time interval in seconds before making another request when the evaluation is still processing. This helps prevent excessive polling and reduces server load during long-running evaluations." + }, + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + }, + "patch": { + "operationId": "Evaluations_Update", + "description": "Updates specific properties of an existing evaluation. Supports modification of metadata fields including description, display name, and tags. Note: Core evaluation configuration such as data sources and evaluators cannot be modified after creation.", + "consumes": [ + "application/merge-patch+json" + ], + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "Identifier of the evaluation.", + "required": true, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + }, + { + "name": "resource", + "in": "body", + "description": "The resource instance.", + "required": true, + "schema": { + "$ref": "#/definitions/EvaluationUpdate" + } + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/Evaluation" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + }, + "delete": { + "operationId": "Evaluations_Delete", + "description": "Delete an evaluation run by name", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "Identifier of the evaluation.", + "required": true, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "204": { + "description": "There is no content to send for this request, but the headers may be useful. ", + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/evaluations/runs/{name}:cancel": { + "post": { + "operationId": "Evaluations_Cancel", + "description": "Cancel an evaluation run by name", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "Identifier of the evaluation.", + "required": true, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "204": { + "description": "There is no content to send for this request, but the headers may be useful. ", + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/evaluations/runs/{name}/results": { + "get": { + "operationId": "Evaluations_GetEvaluationResults", + "description": "Retrieves detailed row-by-row evaluation results with pagination support. This endpoint provides access to individual evaluation outcomes, scores, and reasoning for each data row processed.", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "Unique identifier of the evaluation whose results are being retrieved", + "required": true, + "type": "string" + }, + { + "name": "api-version", + "in": "query", + "description": "API version specifier for this operation", + "required": true, + "type": "string", + "x-ms-client-name": "apiVersion" + }, + { + "name": "top", + "in": "query", + "description": "Maximum number of result items to return in a single response. Must be a multiple of 10 (e.g., 10, 20, 100, 1000). Defaults to 10 if not specified.", + "required": false, + "type": "integer", + "format": "int32" + }, + { + "name": "skip", + "in": "query", + "description": "Number of result items to skip before returning results, enabling pagination through large result sets. Must be a multiple of 10. Defaults to 0.", + "required": false, + "type": "integer", + "format": "int32", + "default": 0 + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedEvaluationResult" + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + } + }, + "/indexes": { + "get": { + "operationId": "Indexes_ListLatest", + "description": "List the latest version of each Index", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedIndex" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + } + }, + "/indexes/{name}/versions": { + "get": { + "operationId": "Indexes_ListVersions", + "description": "List all versions of the given Index", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedIndex" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + } + }, + "/indexes/{name}/versions/{version}": { + "get": { + "operationId": "Indexes_GetVersion", + "description": "Get the specific version of the Index", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + }, + { + "name": "version", + "in": "path", + "description": "The specific version id of the Index to retrieve.", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/Index" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + }, + "patch": { + "operationId": "Indexes_CreateOrUpdateVersion", + "description": "Create a new or update an existing Index with the given version id", + "consumes": [ + "application/merge-patch+json" + ], + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + }, + { + "name": "version", + "in": "path", + "description": "The specific version id of the Index to create or update.", + "required": true, + "type": "string" + }, + { + "name": "index", + "in": "body", + "description": "The Index to create or update.", + "required": true, + "schema": { + "$ref": "#/definitions/Index" + } + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/Index" + } + }, + "201": { + "description": "The request has succeeded and a new resource has been created as a result.", + "schema": { + "$ref": "#/definitions/Index" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + }, + "delete": { + "operationId": "Indexes_DeleteVersion", + "description": "Delete the specific version of the Index", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "The name of the resource", + "required": true, + "type": "string" + }, + { + "name": "version", + "in": "path", + "description": "The version of the Index to delete.", + "required": true, + "type": "string" + } + ], + "responses": { + "204": { + "description": "There is no content to send for this request, but the headers may be useful." + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/redTeams/runs": { + "get": { + "operationId": "RedTeams_List", + "description": "List a redteam by name.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/PagedRedTeam" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "x-ms-pageable": { + "nextLinkName": "nextLink" + } + } + }, + "/redTeams/runs/{name}": { + "get": { + "operationId": "RedTeams_Get", + "description": "Get a redteam by name.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "name", + "in": "path", + "description": "Identifier of the red team run.", + "required": true, + "type": "string" + }, + { + "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" + } + ], + "responses": { + "200": { + "description": "The request has succeeded.", + "schema": { + "$ref": "#/definitions/RedTeam" + }, + "headers": { + "x-ms-client-request-id": { + "type": "string", + "format": "uuid", + "description": "An opaque, globally-unique, client-generated string identifier for the request." + } + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, + "/redTeams/runs:run": { + "post": { + "operationId": "RedTeams_Create", + "description": "Creates a redteam run.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "RedTeam", + "in": "body", + "description": "Redteam to be run", + "required": true, + "schema": { + "$ref": "#/definitions/RedTeam" + } + } + ], + "responses": { + "201": { + "description": "The request has succeeded and a new resource has been created as a result.", + "schema": { + "$ref": "#/definitions/RedTeam" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + } + }, + "definitions": { + "AIContent": { + "type": "object", + "description": "Content for AI requests and responses.", + "properties": { + "type": { + "type": "string", + "description": "The content of the message.", + "enum": [ + "text", + "image_url", + "tool_call", + "tool_result" + ], + "x-ms-enum": { + "modelAsString": true + } + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, + "AgentEvaluation": { + "type": "object", + "description": "Evaluation response for agent evaluation run.", + "properties": { + "id": { + "type": "string", + "description": "Identifier of the agent evaluation run." + }, + "status": { + "type": "string", + "description": "Status of the agent evaluation. Options: Running, Completed, Failed." + }, + "error": { + "type": "string", + "description": "The reason of the request failure for the long running process, if applicable." + }, + "result": { + "type": "array", + "description": "The agent evaluation result.", + "items": { + "$ref": "#/definitions/AgentEvaluationResult" + } + } + }, + "required": [ + "id", + "status" + ] + }, + "AgentEvaluationRedactionConfiguration": { + "type": "object", + "description": "The redaction configuration will allow the user to control what is redacted.", + "properties": { + "redactScoreProperties": { + "type": "boolean", + "description": "Redact score properties. If not specified, the default is to redact in production." + } + } + }, + "AgentEvaluationRequest": { + "type": "object", + "description": "Evaluation request for agent run.", + "properties": { + "runId": { + "type": "string", + "description": "Identifier of the agent run." + }, + "threadId": { + "type": "string", + "description": "Identifier of the agent thread. This field is mandatory currently, but it will be optional in the future." + }, + "evaluators": { + "type": "object", + "description": "Evaluators to be used for the evaluation.", + "additionalProperties": { + "$ref": "#/definitions/EvaluatorConfiguration" + } + }, + "samplingConfiguration": { + "$ref": "#/definitions/AgentEvaluationSamplingConfiguration", + "description": "Sampling configuration for the evaluation." + }, + "redactionConfiguration": { + "$ref": "#/definitions/AgentEvaluationRedactionConfiguration", + "description": "Redaction configuration for the evaluation." + }, + "appInsightsConnectionString": { + "type": "string", + "description": "Pass the AppInsights connection string to the agent evaluation for the evaluation results and the errors logs." + } + }, + "required": [ + "runId", + "evaluators", + "appInsightsConnectionString" + ] + }, + "AgentEvaluationResult": { + "type": "object", + "description": "Result for the agent evaluation evaluator run.", + "properties": { + "evaluator": { + "type": "string", + "description": "Evaluator's name. This is the name of the evaluator that was used to evaluate the agent's completion." + }, + "evaluatorId": { + "type": "string", + "description": "Identifier of the evaluator." + }, + "score": { + "type": "number", + "format": "float", + "description": "Score of the given evaluator. No restriction on range." + }, + "status": { + "type": "string", + "description": "Status of the evaluator result. Options: Running, Completed, Failed, NotApplicable." + }, + "reason": { + "type": "string", + "description": "Reasoning for the evaluation result." + }, + "version": { + "type": "string", + "description": "Version of the evaluator that was used to evaluate the agent's completion." + }, + "threadId": { + "type": "string", + "description": "The unique identifier of the thread." + }, + "runId": { + "type": "string", + "description": "The unique identifier of the run." + }, + "error": { + "type": "string", + "description": "A string explaining why there was an error, if applicable." + }, + "additionalDetails": { + "type": "object", + "description": "Additional properties relevant to the evaluator. These will differ between evaluators.", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "evaluator", + "evaluatorId", + "score", + "status", + "runId" + ] + }, + "AgentEvaluationSamplingConfiguration": { + "type": "object", + "description": "Definition for sampling strategy.", + "properties": { + "name": { + "type": "string", + "description": "Name of the sampling strategy." + }, + "samplingPercent": { + "type": "number", + "format": "float", + "description": "Percentage of sampling per hour (0-100)." + }, + "maxRequestRate": { + "type": "number", + "format": "float", + "description": "Maximum request rate per hour (0 to 1000)." + } + }, + "required": [ + "name", + "samplingPercent", + "maxRequestRate" + ] + }, + "AgentToolDefinition": { + "type": "object", + "description": "Definition of a tool that can be used by the agent.", + "properties": { + "name": { + "type": "string", + "description": "The name of the tool." + }, + "description": { + "type": "string", + "description": "The description of the tool." + }, + "parameters": { + "type": "object", + "description": "The parameters for the tool in JSON Schema format.", + "additionalProperties": {} + } + }, + "required": [ + "name", + "parameters" + ] + }, + "AggregatedScore": { + "type": "object", + "description": "Represents an aggregated score calculated using a specific statistical method across all rows of data.", + "properties": { + "value": { + "type": "number", + "format": "float", + "description": "The calculated aggregated value based on the specified method. This value is undefined in case aggregation is not applicable or the method does not produce a numeric result." + }, + "metadata": { + "type": "object", + "description": "Additional metadata about the aggregation calculation", + "additionalProperties": {} + } + } + }, + "AggregationConfiguration": { + "type": "object", + "description": "Configuration for a specific aggregation method, allowing customization of how the aggregation is calculated.", + "properties": { + "method": { + "$ref": "#/definitions/AggregationMethod", + "description": "The statistical aggregation method to apply (e.g., 'average', 'percentile', 'standardDeviation')" + }, + "parameters": { + "type": "object", + "description": "Optional parameters specific to the aggregation method. For example, 'populationVariance: true' for standard deviation, or 'interpolation: linear' for percentiles.", + "additionalProperties": {} + } + }, + "required": [ + "method" + ] + }, + "AggregationConfigurationUpdate": { + "type": "object", + "description": "Configuration for a specific aggregation method, allowing customization of how the aggregation is calculated.", + "properties": { + "method": { + "$ref": "#/definitions/AggregationMethod", + "description": "The statistical aggregation method to apply (e.g., 'average', 'percentile', 'standardDeviation')" + }, + "parameters": { + "type": "object", + "description": "Optional parameters specific to the aggregation method. For example, 'populationVariance: true' for standard deviation, or 'interpolation: linear' for percentiles.", + "additionalProperties": {} + } + } + }, + "AggregationMethod": { + "type": "string", + "description": "Statistical aggregation methods available for summarizing evaluation results across multiple data points.", + "enum": [ + "sum", + "average", + "max", + "min", + "count", + "percentile", + "standardDeviation" + ], + "x-ms-enum": { + "name": "AggregationMethod", + "modelAsString": true, + "values": [ + { + "name": "sum", + "value": "sum", + "description": "Calculate the sum of all values" + }, + { + "name": "average", + "value": "average", + "description": "Calculate the arithmetic mean of all values" + }, + { + "name": "max", + "value": "max", + "description": "Find the maximum value" + }, + { + "name": "min", + "value": "min", + "description": "Find the minimum value" + }, + { + "name": "count", + "value": "count", + "description": "Count the total number of values" + }, + { + "name": "percentile", + "value": "percentile", + "description": "Calculate the nth percentile value. Needs a parameter to specify the percentile level (e.g., level: 95 for 95th percentile analysis)" + }, + { + "name": "standardDeviation", + "value": "standardDeviation", + "description": "Calculate the standard deviation" + } + ] + } + }, + "ApiKeyCredentials": { + "type": "object", + "description": "API Key Credential definition", + "properties": { + "key": { + "type": "string", + "description": "API Key", + "readOnly": true, + "x-ms-client-name": "apiKey" + } + }, + "allOf": [ + { + "$ref": "#/definitions/BaseCredentials" + } + ], + "x-ms-discriminator-value": "ApiKey" + }, + "AppInsightsDestinationConfiguration": { + "type": "object", + "description": "Configuration for exporting evaluation results to Azure Application Insights. This enables monitoring, alerting, and analytics on evaluation performance and trends.", + "allOf": [ + { + "$ref": "#/definitions/EvaluationDestinationConfiguration" + } + ], + "x-ms-discriminator-value": "appInsightsStorage" + }, + "AssetCredentialResponse": { + "type": "object", + "description": "Represents a reference to a blob for consumption", + "properties": { + "blobReference": { + "$ref": "#/definitions/BlobReference", + "description": "Credential info to access the storage account." + } + }, + "required": [ + "blobReference" + ] + }, + "AssistantMessage": { + "type": "object", + "description": "A message generated by the assistant in response to previous messages.", + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "assistant" + }, + "AttackStrategy": { + "type": "string", + "description": "Strategies for attacks.", + "enum": [ + "easy", + "moderate", + "difficult", + "ascii_art", + "ascii_smuggler", + "atbash", + "base64", + "binary", + "caesar", + "character_space", + "jailbreak", + "ansii_attack", + "character_swap", + "suffix_append", + "string_join", + "unicode_confusable", + "unicode_substitution", + "diacritic", + "flip", + "leetspeak", + "rot13", + "morse", + "url", + "baseline" + ], + "x-ms-enum": { + "name": "AttackStrategy", + "modelAsString": true, + "values": [ + { + "name": "Easy", + "value": "easy", + "description": "Represents a default set of easy complexity attacks. Easy complexity attacks require less effort, such as translation of a prompt into some encoding, and does not require any Large Language Model to convert or orchestrate." + }, + { + "name": "Moderate", + "value": "moderate", + "description": "Represents a default set of moderate complexity attacks. Moderate complexity attacks require having access to resources such as another generative AI model." + }, + { + "name": "Difficult", + "value": "difficult", + "description": "Represents a default set of difficult complexity attacks. Difficult complexity attacks include attacks that require access to significant resources and effort to execute an attack such as knowledge of search-based algorithms in addition to a generative AI model." + }, + { + "name": "AsciiArt", + "value": "ascii_art", + "description": "Generates visual art using ASCII characters, often used for creative or obfuscation purposes." + }, + { + "name": "AsciiSmuggler", + "value": "ascii_smuggler", + "description": "Conceals data within ASCII characters, making it harder to detect." + }, + { + "name": "Atbash", + "value": "atbash", + "description": "Implements the Atbash cipher, a simple substitution cipher where each letter is mapped to its reverse." + }, + { + "name": "Base64", + "value": "base64", + "description": "Encodes binary data into a text format using Base64, commonly used for data transmission." + }, + { + "name": "Binary", + "value": "binary", + "description": "Converts text into binary code, representing data in a series of 0s and 1s." + }, + { + "name": "Caesar", + "value": "caesar", + "description": "Applies the Caesar cipher, a substitution cipher that shifts characters by a fixed number of positions." + }, + { + "name": "CharacterSpace", + "value": "character_space", + "description": "Alters text by adding spaces between characters, often used for obfuscation." + }, + { + "name": "Jailbreak", + "value": "jailbreak", + "description": "Injects specially crafted prompts to bypass AI safeguards, known as User Injected Prompt Attacks (UPIA)." + }, + { + "name": "AnsiiAttack", + "value": "ansii_attack", + "description": "Utilizes ANSI escape sequences to manipulate text appearance and behavior." + }, + { + "name": "CharacterSwap", + "value": "character_swap", + "description": "Swaps characters within text to create variations or obfuscate the original content." + }, + { + "name": "SuffixAppend", + "value": "suffix_append", + "description": "Appends an adversarial suffix to the prompt." + }, + { + "name": "StringJoin", + "value": "string_join", + "description": "Joins multiple strings together, often used for concatenation or obfuscation." + }, + { + "name": "UnicodeConfusable", + "value": "unicode_confusable", + "description": "Uses Unicode characters that look similar to standard characters, creating visual confusion." + }, + { + "name": "UnicodeSubstitution", + "value": "unicode_substitution", + "description": "Substitutes standard characters with Unicode equivalents, often for obfuscation." + }, + { + "name": "Diacritic", + "value": "diacritic", + "description": "Adds diacritical marks to characters, changing their appearance and sometimes their meaning." + }, + { + "name": "Flip", + "value": "flip", + "description": "Flips characters from front to back, creating a mirrored effect." + }, + { + "name": "Leetspeak", + "value": "leetspeak", + "description": "Transforms text into Leetspeak, a form of encoding that replaces letters with similar-looking numbers or symbols." + }, + { + "name": "ROT13", + "value": "rot13", + "description": "Applies the ROT13 cipher, a simple substitution cipher that shifts characters by 13 positions." + }, + { + "name": "Morse", + "value": "morse", + "description": "Encodes text into Morse code, using dots and dashes to represent characters." + }, + { + "name": "Url", + "value": "url", + "description": "Encodes text into URL format." + }, + { + "name": "Baseline", + "value": "baseline", + "description": "Represents the baseline direct adversarial probing, which is used by attack strategies as the attack objective." + } + ] + } + }, + "Azure.Core.Foundations.Error": { + "type": "object", + "description": "The error object.", + "properties": { + "code": { + "type": "string", + "description": "One of a server-defined set of error codes." + }, + "message": { + "type": "string", + "description": "A human-readable representation of the error." + }, + "target": { + "type": "string", + "description": "The target of the error." + }, + "details": { + "type": "array", + "description": "An array of details about specific errors that led to this reported error.", + "items": { + "$ref": "#/definitions/Azure.Core.Foundations.Error" + } + }, + "innererror": { + "$ref": "#/definitions/Azure.Core.Foundations.InnerError", + "description": "An object containing more specific information than the current object about the error." + } + }, + "required": [ + "code", + "message" + ] + }, + "Azure.Core.Foundations.ErrorResponse": { + "type": "object", + "description": "A response containing error details.", + "properties": { + "error": { + "$ref": "#/definitions/Azure.Core.Foundations.Error", + "description": "The error object." + } + }, + "required": [ + "error" + ] + }, + "Azure.Core.Foundations.InnerError": { + "type": "object", + "description": "An object containing more specific information about the error. As per Azure REST API guidelines - https://aka.ms/AzureRestApiGuidelines#handling-errors.", + "properties": { + "code": { + "type": "string", + "description": "One of a server-defined set of error codes." + }, + "innererror": { + "$ref": "#/definitions/Azure.Core.Foundations.InnerError", + "description": "Inner error." + } + } + }, + "Azure.Core.Foundations.OperationState": { + "type": "string", + "description": "Enum describing allowed operation states.", + "enum": [ + "NotStarted", + "Running", + "Succeeded", + "Failed", + "Canceled" + ], + "x-ms-enum": { + "name": "OperationState", + "modelAsString": true, + "values": [ + { + "name": "NotStarted", + "value": "NotStarted", + "description": "The operation has not started." + }, + { + "name": "Running", + "value": "Running", + "description": "The operation is in progress." + }, + { + "name": "Succeeded", + "value": "Succeeded", + "description": "The operation has completed successfully." + }, + { + "name": "Failed", + "value": "Failed", + "description": "The operation has failed." + }, + { + "name": "Canceled", + "value": "Canceled", + "description": "The operation has been canceled by the user." + } + ] + } + }, + "AzureAISearchIndex": { + "type": "object", + "description": "Azure AI Search Index Definition", + "properties": { + "connectionName": { + "type": "string", + "description": "Name of connection to Azure AI Search", + "x-ms-mutability": [ + "create" + ] + }, + "indexName": { + "type": "string", + "description": "Name of index in Azure AI Search resource to attach", + "x-ms-mutability": [ + "create" + ] + }, + "fieldMapping": { + "$ref": "#/definitions/FieldMapping", + "description": "Field mapping configuration", + "x-ms-mutability": [ + "create" + ] + } + }, + "required": [ + "connectionName", + "indexName" + ], + "allOf": [ + { + "$ref": "#/definitions/Index" + } + ], + "x-ms-discriminator-value": "AzureSearch" + }, + "AzureOpenAIModelConfiguration": { + "type": "object", + "description": "Azure OpenAI model configuration. The API version would be selected by the service for querying the model.", + "properties": { + "modelDeploymentName": { + "type": "string", + "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`." + } + }, + "required": [ + "modelDeploymentName" + ], + "allOf": [ + { + "$ref": "#/definitions/TargetConfig" + } + ], + "x-ms-discriminator-value": "AzureOpenAIModel" + }, + "BaseCredentials": { + "type": "object", + "description": "A base class for connection credentials", + "properties": { + "type": { + "$ref": "#/definitions/CredentialType", + "description": "The type of credential used by the connection", + "readOnly": true + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, + "BlobReference": { + "type": "object", + "description": "Blob reference details.", + "properties": { + "blobUri": { + "type": "string", + "description": "Blob URI path for client to upload data. Example: https://blob.windows.core.net/Container/Path" + }, + "storageAccountArmId": { + "type": "string", + "description": "ARM ID of the storage account to use." + }, + "credential": { + "$ref": "#/definitions/SasCredential", + "description": "Credential info to access the storage account." + } + }, + "required": [ + "blobUri", + "storageAccountArmId", + "credential" + ] + }, + "ChatMessages": { + "type": "object", + "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", + "properties": { + "query": { + "type": "array", + "description": "Array of messages representing representing queries", + "items": { + "$ref": "#/definitions/Message" + } + }, + "response": { + "type": "array", + "description": "Array of messages representing responses", + "items": { + "$ref": "#/definitions/Message" + } + }, + "toolDefinitions": { + "type": "array", + "description": "Array of tool definitions that are used in the conversation", + "items": { + "$ref": "#/definitions/AgentToolDefinition" + } + } + }, + "required": [ + "query", + "response", + "toolDefinitions" + ], + "allOf": [ + { + "$ref": "#/definitions/InlineData" + } + ], + "x-ms-discriminator-value": "chatMessages" + }, + "ChatMessagesUpdate": { + "type": "object", + "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", + "properties": { + "query": { + "type": "array", + "description": "Array of messages representing representing queries", + "items": { + "$ref": "#/definitions/Message" + } + }, + "response": { + "type": "array", + "description": "Array of messages representing responses", + "items": { + "$ref": "#/definitions/Message" + } + }, + "toolDefinitions": { + "type": "array", + "description": "Array of tool definitions that are used in the conversation", + "items": { + "$ref": "#/definitions/AgentToolDefinition" + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/InlineDataUpdate" + } + ], + "x-ms-discriminator-value": "chatMessages" + }, + "Connection": { + "type": "object", + "description": "Response from the list and get connections operations", + "properties": { + "name": { + "type": "string", + "description": "The friendly name of the connection, provided by the user.", + "readOnly": true + }, + "id": { + "type": "string", + "description": "A unique identifier for the connection, generated by the service", + "readOnly": true + }, + "type": { + "$ref": "#/definitions/ConnectionType", + "description": "Category of the connection", + "readOnly": true + }, + "target": { + "type": "string", + "description": "The connection URL to be used for this service", + "readOnly": true + }, + "isDefault": { + "type": "boolean", + "description": "Whether the connection is tagged as the default connection of its type", + "readOnly": true + }, + "credentials": { + "$ref": "#/definitions/BaseCredentials", + "description": "The credentials used by the connection", + "readOnly": true + }, + "metadata": { + "type": "object", + "description": "Metadata of the connection", + "additionalProperties": { + "type": "string" + }, + "readOnly": true + } + }, + "required": [ + "name", + "id", + "type", + "target", + "isDefault", + "credentials", + "metadata" + ] + }, + "ConnectionType": { + "type": "string", + "description": "The Type (or category) of the connection", + "enum": [ + "AzureOpenAI", + "AzureBlob", + "AzureStorageAccount", + "CognitiveSearch", + "CosmosDB", + "ApiKey", + "AppConfig", + "AppInsights", + "CustomKeys" + ], + "x-ms-enum": { + "name": "ConnectionType", + "modelAsString": true, + "values": [ + { + "name": "AzureOpenAI", + "value": "AzureOpenAI", + "description": "Azure OpenAI Service" + }, + { + "name": "AzureBlobStorage", + "value": "AzureBlob", + "description": "Azure Blob Storage, with specified container" + }, + { + "name": "AzureStorageAccount", + "value": "AzureStorageAccount", + "description": "Azure Blob Storage, with container not specified (used by Agents)" + }, + { + "name": "AzureAISearch", + "value": "CognitiveSearch", + "description": "Azure AI Search" + }, + { + "name": "CosmosDB", + "value": "CosmosDB", + "description": "CosmosDB" + }, + { + "name": "APIKey", + "value": "ApiKey", + "description": "Generic connection that uses API Key authentication" + }, + { + "name": "ApplicationConfiguration", + "value": "AppConfig", + "description": "Application Configuration" + }, + { + "name": "ApplicationInsights", + "value": "AppInsights", + "description": "Application Insights" + }, + { + "name": "Custom", + "value": "CustomKeys", + "description": "Custom Keys" + } + ] + } + }, + "CosmosDBIndex": { + "type": "object", + "description": "CosmosDB Vector Store Index Definition", + "properties": { + "connectionName": { + "type": "string", + "description": "Name of connection to CosmosDB", + "x-ms-mutability": [ + "create" + ] + }, + "databaseName": { + "type": "string", + "description": "Name of the CosmosDB Database", + "x-ms-mutability": [ + "create" + ] + }, + "containerName": { + "type": "string", + "description": "Name of CosmosDB Container", + "x-ms-mutability": [ + "create" + ] + }, + "embeddingConfiguration": { + "$ref": "#/definitions/EmbeddingConfiguration", + "description": "Embedding model configuration", + "x-ms-mutability": [ + "create" + ] + }, + "fieldMapping": { + "$ref": "#/definitions/FieldMapping", + "description": "Field mapping configuration", + "x-ms-mutability": [ + "create" + ] + } + }, + "required": [ + "connectionName", + "databaseName", + "containerName", + "embeddingConfiguration", + "fieldMapping" + ], + "allOf": [ + { + "$ref": "#/definitions/Index" + } + ], + "x-ms-discriminator-value": "CosmosDBNoSqlVectorStore" + }, + "CredentialType": { + "type": "string", + "description": "The credential type used by the connection", + "enum": [ + "ApiKey", + "AAD", + "SAS", + "CustomKeys", + "None" + ], + "x-ms-enum": { + "name": "CredentialType", + "modelAsString": true, + "values": [ + { + "name": "apiKey", + "value": "ApiKey", + "description": "API Key credential" + }, + { + "name": "entraId", + "value": "AAD", + "description": "Entra ID credential (formerly known as AAD)" + }, + { + "name": "SAS", + "value": "SAS", + "description": "Shared Access Signature (SAS) credential" + }, + { + "name": "custom", + "value": "CustomKeys", + "description": "Custom credential" + }, + { + "name": "None", + "value": "None", + "description": "No credential" + } + ] + } + }, + "CustomCredential": { + "type": "object", + "description": "Custom credential definition", + "properties": { + "keys": { + "type": "object", + "description": "The credential type", + "additionalProperties": { + "type": "string" + }, + "readOnly": true + } + }, + "required": [ + "keys" + ], + "allOf": [ + { + "$ref": "#/definitions/BaseCredentials" + } + ], + "x-ms-discriminator-value": "CustomKeys" + }, + "DatasetId": { + "type": "string", + "description": "Unique identifier of a dataset registered in AI Foundry workspace. The identifier follows the format: azureai://accounts/{resourceName}/projects/{projectName}/data/{datasetName}/versions/{versionNumber}" + }, + "DatasetType": { + "type": "string", + "description": "Enum to determine the type of data.", + "enum": [ + "uri_file", + "uri_folder" + ], + "x-ms-enum": { + "name": "DatasetType", + "modelAsString": true, + "values": [ + { + "name": "uri_file", + "value": "uri_file", + "description": "URI file." + }, + { + "name": "uri_folder", + "value": "uri_folder", + "description": "URI folder." + } + ] + } + }, + "DatasetVersion": { + "type": "object", + "description": "DatasetVersion Definition", + "properties": { + "dataUri": { + "type": "string", + "description": "URI of the data. Example: https://go.microsoft.com/fwlink/?linkid=2202330", + "minLength": 1, + "pattern": "[a-zA-Z0-9_]", + "x-ms-mutability": [ + "read", + "create" + ] + }, + "type": { + "$ref": "#/definitions/DatasetType", + "description": "Dataset type" + }, + "isReference": { + "type": "boolean", + "description": "Indicates if the dataset holds a reference to the storage, or the dataset manages storage itself. If true, the underlying data will not be deleted when the dataset version is deleted", + "readOnly": true + }, + "connectionName": { + "type": "string", + "description": "The Azure Storage Account connection name. Required if startPendingUploadVersion was not called before creating the Dataset", + "x-ms-mutability": [ + "read", + "create" + ] + }, + "id": { + "type": "string", + "description": "Asset ID, a unique identifier for the asset", + "readOnly": true + }, + "name": { + "type": "string", + "description": "The name of the resource", + "readOnly": true + }, + "version": { + "type": "string", + "description": "The version of the resource", + "readOnly": true + }, + "description": { + "type": "string", + "description": "The asset description text.", + "x-ms-mutability": [ + "update", + "create" + ] + }, + "tags": { + "type": "object", + "description": "Tag dictionary. Tags can be added, removed, and updated.", + "additionalProperties": { + "type": "string" + }, + "x-ms-mutability": [ + "update", + "create" + ] + } + }, + "discriminator": "type", + "required": [ + "dataUri", + "type", + "name", + "version" + ] + }, + "Deployment": { + "type": "object", + "description": "Model Deployment Definition", + "properties": { + "type": { + "$ref": "#/definitions/DeploymentType", + "description": "The type of the deployment" + }, + "name": { + "type": "string", + "description": "Name of the deployment", + "readOnly": true + } + }, + "discriminator": "type", + "required": [ + "type", + "name" + ] + }, + "DeploymentType": { + "type": "string", + "enum": [ + "ModelDeployment" + ], + "x-ms-enum": { + "name": "DeploymentType", + "modelAsString": true, + "values": [ + { + "name": "ModelDeployment", + "value": "ModelDeployment", + "description": "Model deployment" + } + ] + } + }, + "DeveloperMessage": { + "type": "object", + "description": "A message authored by a developer to guide the model during evaluation.", + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "developer" + }, + "EmbeddingConfiguration": { + "type": "object", + "description": "Embedding configuration class", + "properties": { + "modelDeploymentName": { + "type": "string", + "description": "Deployment name of embedding model. It can point to a model deployment either in the parent AIServices or a connection.", + "x-ms-mutability": [ + "create" + ] + }, + "embeddingField": { + "type": "string", + "description": "Embedding field", + "x-ms-mutability": [ + "create" + ] + } + }, + "required": [ + "modelDeploymentName", + "embeddingField" + ] + }, + "EntraIDCredentials": { + "type": "object", + "description": "Entra ID credential definition", + "allOf": [ + { + "$ref": "#/definitions/BaseCredentials" + } + ], + "x-ms-discriminator-value": "AAD" + }, + "Evaluation": { + "type": "object", + "description": "Evaluation Definition", + "properties": { + "id": { + "type": "string", + "description": "Identifier of the evaluation.", + "readOnly": true, + "x-ms-client-name": "name" + }, + "dataSource": { + "$ref": "#/definitions/EvaluationDataSource", + "description": "Data source configuration that specifies where the evaluation data comes from. This replaces the legacy 'data' field and provides more flexible data source options." + }, + "resultSettings": { + "$ref": "#/definitions/EvaluationResultSettings", + "description": "Configuration settings that control how evaluation results are processed, stored, and exported" + }, + "displayName": { + "type": "string", + "description": "Display Name for evaluation. It helps to find the evaluation easily in AI Foundry. It does not need to be unique." + }, + "description": { + "type": "string", + "description": "Description of the evaluation. It can be used to store additional information about the evaluation and is mutable." + }, + "state": { + "$ref": "#/definitions/Azure.Core.Foundations.OperationState", + "description": "Current operational state of the evaluation. This field is managed by the service and reflects the evaluation's progress from initiation to completion.", + "readOnly": true + }, + "tags": { + "type": "object", + "description": "Evaluation's tags. Unlike properties, tags are fully mutable.", + "additionalProperties": { + "type": "string" + } + }, + "properties": { + "type": "object", + "description": "Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.", + "additionalProperties": { + "type": "string" + } + }, + "evaluators": { + "type": "object", + "description": "Evaluators to be used for the evaluation.", + "additionalProperties": { + "$ref": "#/definitions/EvaluatorConfiguration" + } + }, + "aggregations": { + "type": "object", + "description": "Statistical aggregation methods to apply when summarizing evaluation results. Specify which aggregation functions should be calculated across all evaluation scores. For example, use 'average' to get mean scores, 'percentile' with parameter level: 95 for 95th percentile analysis, or 'standardDeviation' to measure score variability.\nIf not specified, average and count aggregations will be applied by default.\nKey is the any label for the aggregation, which can be used to identify it in results.\nAvailable methods: sum, average, max, min, count, percentile, standardDeviation.\nNote: Aggregations are only calculated for evaluators that produce numeric scores. Non-numeric evaluators will not contribute to these aggregations.", + "additionalProperties": { + "$ref": "#/definitions/AggregationConfiguration" + } + }, + "summary": { + "type": "object", + "description": "Aggregated summary of evaluation results for each configured evaluator. This provides a high-level overview of performance across all data rows and is available once the evaluation completes.", + "additionalProperties": { + "$ref": "#/definitions/EvaluatorResultSummary" + }, + "readOnly": true + }, + "resultDatasetId": { + "$ref": "#/definitions/DatasetId", + "description": " Unique identifier of the dataset containing detailed evaluation results. This dataset is created automatically upon evaluation completion and contains row-by-row results for analysis.\n The identifier follows the format: azureai://accounts/{resourceName}/projects/{projectName}/datasets/{datasetName}/versions/{versionNumber}.\n Note: This dataset is only available when the result retention policy permits detailed result storage.", + "readOnly": true + }, + "systemData": { + "type": "object", + "description": "System-generated metadata containing internal service information for debugging and operational purposes. This field is managed by the service and cannot be modified by clients.", + "additionalProperties": { + "type": "string" + }, + "readOnly": true + } + }, + "required": [ + "id", + "dataSource", + "state", + "evaluators" + ] + }, + "EvaluationDataSource": { + "type": "object", + "description": "Base class for different types of evaluation data sources. Use the discriminator field 'type' to specify the specific data source implementation.", + "properties": { + "type": { + "$ref": "#/definitions/EvaluationDataSourceType", + "description": "Specifies the type of data source being used for evaluation" + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, + "EvaluationDataSourceType": { + "type": "string", + "description": "Specifies the type of data source used for evaluation. Different types support various data input methods and formats.", + "enum": [ + "inlineJsonData", + "dataset", + "foundryAgentRun", + "FoundryModelDeploymentInline", + "foundryModelDeploymentDataset" + ], + "x-ms-enum": { + "name": "EvaluationDataSourceType", + "modelAsString": true, + "values": [ + { + "name": "InlineJsonData", + "value": "inlineJsonData", + "description": "Use inline JSON data provided directly in the request" + }, + { + "name": "Dataset", + "value": "dataset", + "description": "Use a dataset that has been registered and stored in AI Foundry workspace" + }, + { + "name": "FoundryAgentRun", + "value": "foundryAgentRun", + "description": "Use conversation data from a specific agent run execution" + }, + { + "name": "FoundryModelDeploymentInline", + "value": "FoundryModelDeploymentInline", + "description": "Use data generated by running a model deployment against inline queries provided in the request" + }, + { + "name": "FoundryModelDeploymentDataset", + "value": "foundryModelDeploymentDataset", + "description": "Use data generated by running a model deployment against a registered dataset" + } + ] + } + }, + "EvaluationDataSourceUpdate": { + "type": "object", + "description": "Base class for different types of evaluation data sources. Use the discriminator field 'type' to specify the specific data source implementation.", + "properties": { + "type": { + "$ref": "#/definitions/EvaluationDataSourceType", + "description": "Specifies the type of data source being used for evaluation" + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, + "EvaluationDestinationConfiguration": { + "type": "object", + "description": "Base configuration for exporting evaluation results to external storage destinations. This enables integration with external analytics and monitoring systems.", + "properties": { + "type": { + "$ref": "#/definitions/EvaluationDestinationConfigurationType", + "description": "Specifies the type of external storage destination" + }, + "connectionName": { + "type": "string", + "description": "Name of the connection resource configured in the AI Foundry workspace. This connection must be properly configured with appropriate credentials before use." + }, + "state": { + "$ref": "#/definitions/Azure.Core.Foundations.OperationState", + "description": "Current state of the export operation to this destination. This field is managed by the service and cannot be modified by clients.", + "readOnly": true + } + }, + "discriminator": "type", + "required": [ + "type", + "connectionName" + ] + }, + "EvaluationDestinationConfigurationType": { + "type": "string", + "description": "Specifies the type of external storage destination where evaluation results can be exported for further analysis or long-term retention.", + "enum": [ + "appInsightsStorage", + "storageAccountStorage" + ], + "x-ms-enum": { + "name": "EvaluationDestinationConfigurationType", + "modelAsString": true, + "values": [ + { + "name": "AppInsightsStorage", + "value": "appInsightsStorage", + "description": "Export evaluation results to Azure Application Insights for monitoring and analytics" + }, + { + "name": "StorageAccountStorage", + "value": "storageAccountStorage", + "description": "Export evaluation results to Azure Blob Storage for data archival and custom processing" + } + ] + } + }, + "EvaluationModelConfiguration": { + "type": "object", + "description": "Comprehensive configuration for a model deployment used in evaluation scenarios. This defines how the model will process inputs and generate responses for evaluation.", + "properties": { + "modelDeploymentName": { + "type": "string", + "description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'." + }, + "modelParameters": { + "type": "object", + "description": "Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.", + "additionalProperties": {} + } + }, + "required": [ + "modelDeploymentName" + ] + }, + "EvaluationModelConfigurationUpdate": { + "type": "object", + "description": "Comprehensive configuration for a model deployment used in evaluation scenarios. This defines how the model will process inputs and generate responses for evaluation.", + "properties": { + "modelDeploymentName": { + "type": "string", + "description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'." + }, + "modelParameters": { + "type": "object", + "description": "Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.", + "additionalProperties": {} + } + } + }, + "EvaluationModelSourceCommon": { + "type": "object", + "description": "Common properties shared across model deployment configurations used in evaluations.", + "properties": { + "modelConfiguration": { + "$ref": "#/definitions/EvaluationModelConfiguration", + "description": "Configuration for the model deployment used in evaluation." + }, + "baseMessages": { + "type": "array", + "description": "A list of messages comprising the conversation so far. Each message can be a json string with role and content to specify the conversation context.", + "items": { + "$ref": "#/definitions/Message" + } + } + }, + "required": [ + "modelConfiguration", + "baseMessages" + ] + }, + "EvaluationResult": { + "type": "object", + "description": "Comprehensive evaluation results for a single data row, including the input data, processing state, and results from all configured evaluators.", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for this evaluation result row", + "readOnly": true + }, + "state": { + "$ref": "#/definitions/Azure.Core.Foundations.OperationState", + "description": "Current processing state of this data row. Tracks progress from initial processing through completion or failure." + }, + "error": { + "type": "string", + "description": "Error message describing why the evaluation failed for this data row, if applicable" + }, + "inputDataJson": { + "type": "string", + "description": "Original input data for this row in JSON string format. This preserves the exact data that was evaluated for reference and debugging." + }, + "evaluatorResults": { + "type": "object", + "description": "Collection of evaluation results from each configured evaluator for this data row. The key is the evaluator identifier and the value contains the detailed evaluation results.", + "additionalProperties": { + "$ref": "#/definitions/EvaluatorResult" + } + } + }, + "required": [ + "id", + "state", + "evaluatorResults" + ] + }, + "EvaluationResultOutcome": { + "type": "string", + "description": "Defines the possible outcomes of an evaluation result, indicating whether the evaluated data meets the specified criteria or standards.", + "enum": [ + "pass", + "fail", + "notEvaluated", + "notApplicable" + ], + "x-ms-enum": { + "name": "EvaluationResultOutcome", + "modelAsString": true, + "values": [ + { + "name": "pass", + "value": "pass", + "description": "The evaluation completed successfully and met the specified criteria or threshold" + }, + { + "name": "fail", + "value": "fail", + "description": "The evaluation completed but did not meet the specified criteria or threshold" + }, + { + "name": "notEvaluated", + "value": "notEvaluated", + "description": "The evaluation has not been processed yet or is still in progress" + }, + { + "name": "notApplicable", + "value": "notApplicable", + "description": "The evaluation criteria do not apply to this particular data row or context" + } + ] + } + }, + "EvaluationResultSettings": { + "type": "object", + "description": "Configuration settings that control how evaluation results are processed, stored, and exported. These settings affect data privacy, retention, and integration with external systems.", + "properties": { + "redactionLevel": { + "$ref": "#/definitions/RedactionLevel", + "description": "Level of data redaction to apply to evaluation results. Defaults to 'sensitive' in production environments to protect confidential information." + }, + "additionalDestinations": { + "type": "array", + "description": "List of external storage destinations where evaluation results should be exported in addition to the default AI Foundry storage", + "items": { + "$ref": "#/definitions/EvaluationDestinationConfiguration" + } + } + } + }, + "EvaluationScore": { + "type": "object", + "description": "Score assigned by the evaluator to a specific data row", + "properties": { + "label": { + "description": "Score of the evaluation." + }, + "justification": { + "type": "string", + "description": "Justification for the score, providing context on how it was derived or what it represents" + }, + "threshold": { + "type": "number", + "format": "float", + "description": "Threshold value that this score is compared against to determine the evaluation outcome" + }, + "desirableDirection": { + "$ref": "#/definitions/EvaluatorDesirableDirection", + "description": "Direction of the score that indicates whether a higher or lower score is desirable for this evaluator" + }, + "scoreType": { + "$ref": "#/definitions/EvaluatorScoreType", + "description": "Type of the score that indicates whether it is a boolean, continuous, or ordinal score" + }, + "outcome": { + "$ref": "#/definitions/EvaluationResultOutcome", + "description": "Outcome of the evaluation based on the score and threshold. Indicates whether the score meets, exceeds, or falls below expectations" + }, + "additionalMetrics": { + "type": "object", + "description": "Optional additional details that may include metadata or other relevant information about the score", + "additionalProperties": {} + } + }, + "required": [ + "justification" + ] + }, + "EvaluationSummaryStatistics": { + "type": "object", + "description": "Statistical summary of evaluation results including aggregated scores and pass/fail rates.", + "properties": { + "aggregatedScores": { + "type": "object", + "description": "Aggregated scores calculated using specified aggregation methods. This provides a summary of the evaluator's performance across all data rows.", + "additionalProperties": { + "$ref": "#/definitions/AggregatedScore" + } + }, + "passRate": { + "type": "number", + "format": "float", + "description": "Value between 0 and 1 representing the proportion of rows where outcome is 'pass'." + }, + "sampleCount": { + "type": "integer", + "format": "int64", + "description": "Total number of rows that were evaluated." + } + }, + "required": [ + "aggregatedScores", + "passRate", + "sampleCount" + ] + }, + "EvaluationUpdate": { + "type": "object", + "description": "Evaluation Definition", + "properties": { + "dataSource": { + "$ref": "#/definitions/EvaluationDataSourceUpdate", + "description": "Data source configuration that specifies where the evaluation data comes from. This replaces the legacy 'data' field and provides more flexible data source options." + }, + "resultSettings": { + "$ref": "#/definitions/EvaluationResultSettings", + "description": "Configuration settings that control how evaluation results are processed, stored, and exported" + }, + "displayName": { + "type": "string", + "description": "Display Name for evaluation. It helps to find the evaluation easily in AI Foundry. It does not need to be unique." + }, + "description": { + "type": "string", + "description": "Description of the evaluation. It can be used to store additional information about the evaluation and is mutable." + }, + "tags": { + "type": "object", + "description": "Evaluation's tags. Unlike properties, tags are fully mutable.", + "additionalProperties": { + "type": "string" + } + }, + "properties": { + "type": "object", + "description": "Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.", + "additionalProperties": { + "type": "string" + } + }, + "evaluators": { + "type": "object", + "description": "Evaluators to be used for the evaluation.", + "additionalProperties": { + "$ref": "#/definitions/EvaluatorConfigurationUpdate" + } + }, + "aggregations": { + "type": "object", + "description": "Statistical aggregation methods to apply when summarizing evaluation results. Specify which aggregation functions should be calculated across all evaluation scores. For example, use 'average' to get mean scores, 'percentile' with parameter level: 95 for 95th percentile analysis, or 'standardDeviation' to measure score variability.\nIf not specified, average and count aggregations will be applied by default.\nKey is the any label for the aggregation, which can be used to identify it in results.\nAvailable methods: sum, average, max, min, count, percentile, standardDeviation.\nNote: Aggregations are only calculated for evaluators that produce numeric scores. Non-numeric evaluators will not contribute to these aggregations.", + "additionalProperties": { + "$ref": "#/definitions/AggregationConfigurationUpdate" + } + } + } + }, + "EvaluatorConfiguration": { + "type": "object", + "description": "Evaluator Configuration", + "properties": { + "id": { + "type": "string", + "description": "Identifier of the evaluator." + }, + "initialParameters": { + "type": "object", + "description": "Initial parameters of the evaluator.", + "additionalProperties": {} + }, + "dataMapping": { + "type": "object", + "description": "Data parameters of the evaluator.", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "id" + ] + }, + "EvaluatorConfigurationUpdate": { + "type": "object", + "description": "Evaluator Configuration", + "properties": { + "id": { + "type": "string", + "description": "Identifier of the evaluator." + }, + "initialParameters": { + "type": "object", + "description": "Initial parameters of the evaluator.", + "additionalProperties": {} + }, + "dataMapping": { + "type": "object", + "description": "Data parameters of the evaluator.", + "additionalProperties": { + "type": "string" + } + } + } + }, + "EvaluatorDesirableDirection": { + "type": "string", + "description": "Describes the desirable direction for the evaluation score. This indicates whether a higher or lower score is preferred for this evaluator.", + "enum": [ + "increase", + "decrease", + "neutral" + ], + "x-ms-enum": { + "name": "EvaluatorDesirableDirection", + "modelAsString": true, + "values": [ + { + "name": "increase", + "value": "increase", + "description": "Indicates that a higher score is desirable for this evaluator" + }, + { + "name": "decrease", + "value": "decrease", + "description": "Indicates that a lower score is desirable for this evaluator" + }, + { + "name": "neutral", + "value": "neutral", + "description": "Indicates that the score should be neutral, meaning it does not have a preferred direction" + } + ] + } + }, + "EvaluatorResult": { + "type": "object", + "description": "Detailed result of a specific evaluator's assessment on a single data row. Contains the evaluation outcome, score, reasoning, and additional metrics.", + "properties": { + "evaluatorId": { + "type": "string", + "description": "Unique identifier of the evaluator that produced this result" + }, + "systemData": { + "$ref": "#/definitions/EvaluatorResultSystemData", + "description": "Metadata about the evaluation execution environment and resource usage." + }, + "state": { + "$ref": "#/definitions/Azure.Core.Foundations.OperationState", + "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." + }, + "score": { + "$ref": "#/definitions/EvaluationScore", + "description": "Score for the evaluation result" + } + }, + "required": [ + "evaluatorId", + "state", + "score" + ] + }, + "EvaluatorResultCommon": { + "type": "object", + "description": "Common properties shared across all evaluator results, including metadata about the evaluation execution and resource consumption.", + "properties": { + "evaluatorId": { + "type": "string", + "description": "Unique identifier of the evaluator that produced this result" + }, + "systemData": { + "$ref": "#/definitions/EvaluatorResultSystemData", + "description": "Metadata about the evaluation execution environment and resource usage." + }, + "state": { + "$ref": "#/definitions/Azure.Core.Foundations.OperationState", + "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." + } + }, + "required": [ + "evaluatorId", + "state" + ] + }, + "EvaluatorResultSummary": { + "type": "object", + "description": "Aggregated summary of evaluation results for a specific evaluator across all data rows. This provides a high-level overview of the evaluator's performance.", + "properties": { + "evaluatorId": { + "type": "string", + "description": "Unique identifier of the evaluator that produced this result" + }, + "systemData": { + "$ref": "#/definitions/EvaluatorResultSystemData", + "description": "Metadata about the evaluation execution environment and resource usage." + }, + "state": { + "$ref": "#/definitions/Azure.Core.Foundations.OperationState", + "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." + }, + "statistics": { + "$ref": "#/definitions/EvaluationSummaryStatistics", + "description": "Statistical summary of evaluation results." + } + }, + "required": [ + "evaluatorId", + "state" + ] + }, + "EvaluatorResultSystemData": { + "type": "object", + "description": "Represents system Metadata about the evaluation execution environment and resource usage.", + "properties": { + "inputTokenCount": { + "type": "integer", + "format": "int64", + "description": "Total number of input tokens consumed during the evaluation process. This helps track resource usage and costs." + }, + "outputTokenCount": { + "type": "integer", + "format": "int64", + "description": "Total number of output tokens generated during the evaluation process. This helps track resource usage and costs." + }, + "duration": { + "type": "integer", + "format": "int64", + "description": "Total time taken to complete the evaluation, measured in milliseconds. This includes processing time and any network latency." + }, + "additionalDetails": { + "type": "object", + "description": "Additional details about the evaluator result", + "additionalProperties": {} + } + }, + "required": [ + "inputTokenCount", + "outputTokenCount", + "duration", + "additionalDetails" + ] + }, + "EvaluatorScoreType": { + "type": "string", + "description": "Type of score assigned by the evaluator. This indicates whether the score is boolean, continuous, or ordinal.", + "enum": [ + "boolean", + "continuous", + "ordinal" + ], + "x-ms-enum": { + "name": "EvaluatorScoreType", + "modelAsString": true, + "values": [ + { + "name": "boolean", + "value": "boolean", + "description": "Boolean score type, where the score is either true or false" + }, + { + "name": "continuous", + "value": "continuous", + "description": "Continuous score type, where the score is a floating-point number representing a continuous value" + }, + { + "name": "ordinal", + "value": "ordinal", + "description": "Ordinal score type, where the score represents an ordered category or rank" + } + ] + } + }, + "FieldMapping": { + "type": "object", + "description": "Field mapping configuration class", + "properties": { + "contentFields": { + "type": "array", + "description": "List of fields with text content", + "items": { + "type": "string" + }, + "x-ms-mutability": [ + "create" + ] + }, + "filepathField": { + "type": "string", + "description": "Path of file to be used as a source of text content", + "x-ms-mutability": [ + "create" + ] + }, + "titleField": { + "type": "string", + "description": "Field containing the title of the document", + "x-ms-mutability": [ + "create" + ] + }, + "urlField": { + "type": "string", + "description": "Field containing the url of the document", + "x-ms-mutability": [ + "create" + ] + }, + "vectorFields": { + "type": "array", + "description": "List of fields with vector content", + "items": { + "type": "string" + }, + "x-ms-mutability": [ + "create" + ] + }, + "metadataFields": { + "type": "array", + "description": "List of fields with metadata content", + "items": { + "type": "string" + }, + "x-ms-mutability": [ + "create" + ] + } + }, + "required": [ + "contentFields" + ] + }, + "FileDatasetVersion": { + "type": "object", + "description": "FileDatasetVersion Definition", + "allOf": [ + { + "$ref": "#/definitions/DatasetVersion" + } + ], + "x-ms-discriminator-value": "uri_file" + }, + "FolderDatasetVersion": { + "type": "object", + "description": "FileDatasetVersion Definition", + "allOf": [ + { + "$ref": "#/definitions/DatasetVersion" + } + ], + "x-ms-discriminator-value": "uri_folder" + }, + "FoundryAgentRunSource": { + "type": "object", + "description": "Data source that uses conversation data from a specific agent run execution for evaluation. This allows evaluating agent performance based on actual conversation history.", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the agent to evaluate." + }, + "runId": { + "type": "string", + "description": "Unique identifier of the agent run to evaluate. Example: run_1234" + }, + "threadId": { + "type": "string", + "description": "Unique identifier of the conversation thread within the agent run. This is required when evaluating OpenAI-based agents or models to isolate specific conversation contexts." + } + }, + "required": [ + "id", + "runId" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSource" + } + ], + "x-ms-discriminator-value": "foundryAgentRun" + }, + "FoundryAgentRunSourceUpdate": { + "type": "object", + "description": "Data source that uses conversation data from a specific agent run execution for evaluation. This allows evaluating agent performance based on actual conversation history.", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the agent to evaluate." + }, + "runId": { + "type": "string", + "description": "Unique identifier of the agent run to evaluate. Example: run_1234" + }, + "threadId": { + "type": "string", + "description": "Unique identifier of the conversation thread within the agent run. This is required when evaluating OpenAI-based agents or models to isolate specific conversation contexts." + } + }, + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSourceUpdate" + } + ], + "x-ms-discriminator-value": "foundryAgentRun" + }, + "FoundryDatasetDataSource": { + "type": "object", + "description": "Data source that uses a dataset registered and stored in AI Foundry workspace. This is the recommended approach for large datasets or reusable evaluation data.", + "properties": { + "datasetId": { + "$ref": "#/definitions/DatasetId", + "description": "Unique identifier of the dataset registered in AI Foundry workspace" + } + }, + "required": [ + "datasetId" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSource" + } + ], + "x-ms-discriminator-value": "dataset" + }, + "FoundryDatasetDataSourceUpdate": { + "type": "object", + "description": "Data source that uses a dataset registered and stored in AI Foundry workspace. This is the recommended approach for large datasets or reusable evaluation data.", + "properties": { + "datasetId": { + "$ref": "#/definitions/DatasetId", + "description": "Unique identifier of the dataset registered in AI Foundry workspace" + } + }, + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSourceUpdate" + } + ], + "x-ms-discriminator-value": "dataset" + }, + "FoundryModelDatasetSource": { + "type": "object", + "description": "Data source that uses a model deployment with a dataset containing prompts. The model processes each prompt from the dataset to generate responses, which are then evaluated against the configured evaluators.", + "properties": { + "modelConfiguration": { + "$ref": "#/definitions/EvaluationModelConfiguration", + "description": "Configuration for the model deployment used in evaluation." + }, + "baseMessages": { + "type": "array", + "description": "A list of messages comprising the conversation so far. Each message can be a json string with role and content to specify the conversation context.", + "items": { + "$ref": "#/definitions/Message" + } + }, + "datasetId": { + "$ref": "#/definitions/DatasetId", + "description": "Unique identifier of the dataset containing prompts that will be processed by the model deployment" + }, + "queryField": { + "type": "string", + "description": "Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '{{item.messages}}'" + } + }, + "required": [ + "modelConfiguration", + "baseMessages", + "datasetId", + "queryField" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSource" + } + ], + "x-ms-discriminator-value": "foundryModelDeploymentDataset" + }, + "FoundryModelDatasetSourceUpdate": { + "type": "object", + "description": "Data source that uses a model deployment with a dataset containing prompts. The model processes each prompt from the dataset to generate responses, which are then evaluated against the configured evaluators.", + "properties": { + "modelConfiguration": { + "$ref": "#/definitions/EvaluationModelConfigurationUpdate", + "description": "Configuration for the model deployment used in evaluation." + }, + "baseMessages": { + "type": "array", + "description": "A list of messages comprising the conversation so far. Each message can be a json string with role and content to specify the conversation context.", + "items": { + "$ref": "#/definitions/Message" + } + }, + "datasetId": { + "$ref": "#/definitions/DatasetId", + "description": "Unique identifier of the dataset containing prompts that will be processed by the model deployment" + }, + "queryField": { + "type": "string", + "description": "Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '{{item.messages}}'" + } + }, + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSourceUpdate" + } + ], + "x-ms-discriminator-value": "foundryModelDeploymentDataset" + }, + "FoundryModelInlineSource": { + "type": "object", + "description": "Data source that uses a model deployment with inline queries. The specified model processes each query to generate responses, which are then evaluated against the configured evaluators.", + "properties": { + "modelConfiguration": { + "$ref": "#/definitions/EvaluationModelConfiguration", + "description": "Configuration for the model deployment used in evaluation." + }, + "baseMessages": { + "type": "array", + "description": "A list of messages comprising the conversation so far. Each message can be a json string with role and content to specify the conversation context.", + "items": { + "$ref": "#/definitions/Message" + } + }, + "queries": { + "type": "array", + "description": "Inline queries to be processed by the model deployment. The response is then evaluated against the configured evaluators.", + "items": { + "type": "string" + } + } + }, + "required": [ + "modelConfiguration", + "baseMessages", + "queries" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSource" + } + ], + "x-ms-discriminator-value": "FoundryModelDeploymentInline" + }, + "FoundryModelInlineSourceUpdate": { + "type": "object", + "description": "Data source that uses a model deployment with inline queries. The specified model processes each query to generate responses, which are then evaluated against the configured evaluators.", + "properties": { + "modelConfiguration": { + "$ref": "#/definitions/EvaluationModelConfigurationUpdate", + "description": "Configuration for the model deployment used in evaluation." + }, + "baseMessages": { + "type": "array", + "description": "A list of messages comprising the conversation so far. Each message can be a json string with role and content to specify the conversation context.", + "items": { + "$ref": "#/definitions/Message" + } + }, + "queries": { + "type": "array", + "description": "Inline queries to be processed by the model deployment. The response is then evaluated against the configured evaluators.", + "items": { + "type": "string" + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSourceUpdate" + } + ], + "x-ms-discriminator-value": "FoundryModelDeploymentInline" + }, + "ImageUrlContent": { + "type": "object", + "description": "Content for image URL messages in AI conversations.", + "properties": { + "imageUrl": { + "type": "string", + "description": "The URL of the image." + } + }, + "required": [ + "imageUrl" + ], + "allOf": [ + { + "$ref": "#/definitions/AIContent" + } + ], + "x-ms-discriminator-value": "image_url" + }, + "Index": { + "type": "object", + "description": "Index resource Definition", + "properties": { + "type": { + "$ref": "#/definitions/IndexType", + "description": "Type of index" + }, + "id": { + "type": "string", + "description": "Asset ID, a unique identifier for the asset", + "readOnly": true + }, + "name": { + "type": "string", + "description": "The name of the resource", + "readOnly": true + }, + "version": { + "type": "string", + "description": "The version of the resource", + "readOnly": true + }, + "description": { + "type": "string", + "description": "The asset description text.", + "x-ms-mutability": [ + "update", + "create" + ] + }, + "tags": { + "type": "object", + "description": "Tag dictionary. Tags can be added, removed, and updated.", + "additionalProperties": { + "type": "string" + }, + "x-ms-mutability": [ + "update", + "create" + ] + } + }, + "discriminator": "type", + "required": [ + "type", + "name", + "version" + ] + }, + "IndexType": { + "type": "string", + "enum": [ + "AzureSearch", + "CosmosDBNoSqlVectorStore", + "ManagedAzureSearch" + ], + "x-ms-enum": { + "name": "IndexType", + "modelAsString": true, + "values": [ + { + "name": "azureSearch", + "value": "AzureSearch", + "description": "Azure search" + }, + { + "name": "cosmosDB", + "value": "CosmosDBNoSqlVectorStore", + "description": "CosmosDB" + }, + { + "name": "managedAzureSearch", + "value": "ManagedAzureSearch", + "description": "Managed Azure Search" + } + ] + } + }, + "InlineData": { + "type": "object", + "description": "Base class for inline evaluation data with format discrimination.", + "properties": { + "dataFormat": { + "$ref": "#/definitions/InlineDataFormat", + "description": "Format of the inline data structure" + } + }, + "discriminator": "dataFormat", + "required": [ + "dataFormat" + ] + }, + "InlineDataFormat": { + "type": "string", + "description": "Available formats for structuring inline evaluation data.", + "enum": [ + "queryResponseMessage", + "chatMessages", + "inlineJson" + ], + "x-ms-enum": { + "name": "InlineDataFormat", + "modelAsString": true, + "values": [ + { + "name": "queryResponseMessage", + "value": "queryResponseMessage", + "description": "Query-response pairs for Q&A and chatbot evaluations" + }, + { + "name": "chatMessages", + "value": "chatMessages", + "description": "Multi-turn conversations with role-based messages and tool definitions" + }, + { + "name": "inlineJson", + "value": "inlineJson", + "description": "Flexible JSON format for custom data structures" + } + ] + } + }, + "InlineDataUpdate": { + "type": "object", + "description": "Base class for inline evaluation data with format discrimination.", + "properties": { + "dataFormat": { + "$ref": "#/definitions/InlineDataFormat", + "description": "Format of the inline data structure" + } + }, + "discriminator": "dataFormat", + "required": [ + "dataFormat" + ] + }, + "InlineJson": { + "type": "object", + "description": "Custom JSON format for complex evaluation scenarios requiring flexible data structures.", + "properties": { + "items": { + "type": "array", + "description": "Array of JSON strings with custom fields and metadata", + "items": { + "type": "string" + } + } + }, + "required": [ + "items" + ], + "allOf": [ + { + "$ref": "#/definitions/InlineData" + } + ], + "x-ms-discriminator-value": "inlineJson" + }, + "InlineJsonDataSource": { + "type": "object", + "description": "Data source using inline data provided directly in the request.", + "properties": { + "id": { + "type": "string", + "description": "Optional unique identifier for the inline data source. This can be an agent id or a custom identifier to distinguish between different inline data sources." + }, + "data": { + "$ref": "#/definitions/InlineData", + "description": "Inline data structured according to the specified format" + } + }, + "required": [ + "data" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSource" + } + ], + "x-ms-discriminator-value": "inlineJsonData" + }, + "InlineJsonDataSourceUpdate": { + "type": "object", + "description": "Data source using inline data provided directly in the request.", + "properties": { + "id": { + "type": "string", + "description": "Optional unique identifier for the inline data source. This can be an agent id or a custom identifier to distinguish between different inline data sources." + }, + "data": { + "$ref": "#/definitions/InlineDataUpdate", + "description": "Inline data structured according to the specified format" + } + }, + "allOf": [ + { + "$ref": "#/definitions/EvaluationDataSourceUpdate" + } + ], + "x-ms-discriminator-value": "inlineJsonData" + }, + "InlineJsonUpdate": { + "type": "object", + "description": "Custom JSON format for complex evaluation scenarios requiring flexible data structures.", + "properties": { + "items": { + "type": "array", + "description": "Array of JSON strings with custom fields and metadata", + "items": { + "type": "string" + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/InlineDataUpdate" + } + ], + "x-ms-discriminator-value": "inlineJson" + }, + "InlineMessagesType": { + "type": "string", + "description": "The type of inline messages used for evaluation. This union allows for different inline message formats to be specified.", + "enum": [ + "queryResponseMessage", + "roleContentType" + ], + "x-ms-enum": { + "name": "InlineMessagesType", + "modelAsString": true, + "values": [ + { + "name": "queryResponseMessage", + "value": "queryResponseMessage", + "description": "Inline messages that consist of query-response pairs for evaluation. Each pair includes a query string and the corresponding response generated by the model." + }, + { + "name": "roleContentType", + "value": "roleContentType", + "description": "role content type message" + } + ] + } + }, + "InputData": { + "type": "object", + "description": "Abstract data class.", + "properties": { + "type": { + "type": "string", + "description": "Type of the data" + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, + "InputDataset": { + "type": "object", + "description": "Dataset as source for evaluation.", + "properties": { + "id": { + "type": "string", + "description": "Evaluation input data" + } + }, + "required": [ + "id" + ], + "allOf": [ + { + "$ref": "#/definitions/InputData" + } + ], + "x-ms-discriminator-value": "dataset" + }, + "ManagedAzureAISearchIndex": { + "type": "object", + "description": "Managed Azure AI Search Index Definition", + "properties": { + "vectorStoreId": { + "type": "string", + "description": "Vector store id of managed index", + "x-ms-mutability": [ + "create" + ] + } + }, + "required": [ + "vectorStoreId" + ], + "allOf": [ + { + "$ref": "#/definitions/Index" + } + ], + "x-ms-discriminator-value": "ManagedAzureSearch" + }, + "Message": { + "type": "object", + "description": "Abstract base model representing a single message in a conversation.", + "properties": { + "role": { + "type": "string", + "description": "The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.", + "enum": [ + "system", + "assistant", + "developer", + "user" + ], + "x-ms-enum": { + "modelAsString": true + } + }, + "content": { + "description": "The content of the message." + } + }, + "discriminator": "role", + "required": [ + "role", + "content" + ] + }, + "ModelDeployment": { + "type": "object", + "description": "Model Deployment Definition", + "properties": { + "modelName": { + "type": "string", + "description": "Publisher-specific name of the deployed model", + "readOnly": true + }, + "modelVersion": { + "type": "string", + "description": "Publisher-specific version of the deployed model", + "readOnly": true + }, + "modelPublisher": { + "type": "string", + "description": "Name of the deployed model's publisher", + "readOnly": true + }, + "capabilities": { + "type": "object", + "description": "Capabilities of deployed model", + "additionalProperties": { + "type": "string" + }, + "readOnly": true + }, + "sku": { + "$ref": "#/definitions/Sku", + "description": "Sku of the model deployment", + "readOnly": true + }, + "connectionName": { + "type": "string", + "description": "Name of the connection the deployment comes from", + "readOnly": true + } + }, + "required": [ + "modelName", + "modelVersion", + "modelPublisher", + "capabilities", + "sku" + ], + "allOf": [ + { + "$ref": "#/definitions/Deployment" + } + ], + "x-ms-discriminator-value": "ModelDeployment" + }, + "NoAuthenticationCredentials": { + "type": "object", + "description": "Credentials that do not require authentication", + "allOf": [ + { + "$ref": "#/definitions/BaseCredentials" + } + ], + "x-ms-discriminator-value": "None" + }, + "PagedConnection": { + "type": "object", + "description": "Paged collection of Connection items", + "properties": { + "value": { + "type": "array", + "description": "The Connection items on this page", + "items": { + "$ref": "#/definitions/Connection" + } + }, + "nextLink": { + "type": "string", + "format": "uri", + "description": "The link to the next page of items" + } + }, + "required": [ + "value" + ] + }, + "PagedDatasetVersion": { + "type": "object", + "description": "Paged collection of DatasetVersion items", + "properties": { + "value": { + "type": "array", + "description": "The DatasetVersion items on this page", + "items": { + "$ref": "#/definitions/DatasetVersion" + } + }, + "nextLink": { + "type": "string", + "format": "uri", + "description": "The link to the next page of items" + } + }, + "required": [ + "value" + ] + }, + "PagedDeployment": { + "type": "object", + "description": "Paged collection of Deployment items", + "properties": { + "value": { + "type": "array", + "description": "The Deployment items on this page", + "items": { + "$ref": "#/definitions/Deployment" + } + }, + "nextLink": { + "type": "string", + "format": "uri", + "description": "The link to the next page of items" + } + }, + "required": [ + "value" + ] + }, + "PagedEvaluation": { + "type": "object", + "description": "Paged collection of Evaluation items", + "properties": { + "value": { + "type": "array", + "description": "The Evaluation items on this page", + "items": { + "$ref": "#/definitions/Evaluation" + } + }, + "nextLink": { + "type": "string", + "format": "uri", + "description": "The link to the next page of items" + } + }, + "required": [ + "value" + ] + }, + "PagedEvaluationResult": { + "type": "object", + "description": "Paged collection of EvaluationResult items", + "properties": { + "value": { + "type": "array", + "description": "The EvaluationResult items on this page", + "items": { + "$ref": "#/definitions/EvaluationResult" + } + }, + "nextLink": { + "type": "string", + "format": "uri", + "description": "The link to the next page of items" + } + }, + "required": [ + "value" + ] + }, + "PagedIndex": { + "type": "object", + "description": "Paged collection of Index items", + "properties": { + "value": { + "type": "array", + "description": "The Index items on this page", + "items": { + "$ref": "#/definitions/Index" + } + }, + "nextLink": { + "type": "string", + "format": "uri", + "description": "The link to the next page of items" + } + }, + "required": [ + "value" + ] + }, + "PagedRedTeam": { + "type": "object", + "description": "Paged collection of RedTeam items", + "properties": { + "value": { + "type": "array", + "description": "The RedTeam items on this page", + "items": { + "$ref": "#/definitions/RedTeam" + } + }, + "nextLink": { + "type": "string", + "format": "uri", + "description": "The link to the next page of items" + } + }, + "required": [ + "value" + ] + }, + "PendingUploadCredentialType": { + "type": "string", + "description": "The type of credential used to access the storage account.", + "enum": [ + "SAS" + ], + "x-ms-enum": { + "name": "PendingUploadCredentialType", + "modelAsString": true, + "values": [ + { + "name": "sas", + "value": "SAS", + "description": "SAS credential type." + } + ] + } + }, + "PendingUploadRequest": { + "type": "object", + "description": "Represents a request for a pending upload.", + "properties": { + "pendingUploadId": { + "type": "string", + "description": "If PendingUploadId is not provided, a random GUID will be used." + }, + "connectionName": { + "type": "string", + "description": "Azure Storage Account connection name to use for generating temporary SAS token" + }, + "pendingUploadType": { + "type": "string", + "description": "BlobReference is the only supported type.", + "enum": [ + "BlobReference" + ], + "x-ms-enum": { + "modelAsString": false + } + } + }, + "required": [ + "pendingUploadType" + ] + }, + "PendingUploadResponse": { + "type": "object", + "description": "Represents the response for a pending upload request", + "properties": { + "blobReference": { + "$ref": "#/definitions/BlobReference", + "description": "Container-level read, write, list SAS." + }, + "pendingUploadId": { + "type": "string", + "description": "ID for this upload request." + }, + "version": { + "type": "string", + "description": "Version of asset to be created if user did not specify version when initially creating upload" + }, + "pendingUploadType": { + "type": "string", + "description": "BlobReference is the only supported type", + "enum": [ + "BlobReference" + ], + "x-ms-enum": { + "modelAsString": false + } + } + }, + "required": [ + "blobReference", + "pendingUploadId", + "pendingUploadType" + ] + }, + "PendingUploadType": { + "type": "string", + "description": "The type of pending upload.", + "enum": [ + "None", + "BlobReference" + ], + "x-ms-enum": { + "name": "PendingUploadType", + "modelAsString": true, + "values": [ + { + "name": "none", + "value": "None", + "description": "No pending upload." + }, + { + "name": "BlobReference", + "value": "BlobReference", + "description": "Blob Reference is the only supported type." + } + ] + } + }, + "QueryResponseInlineMessages": { + "type": "object", + "description": "Query-response pairs for evaluating Q&A systems and response accuracy.", + "properties": { + "items": { + "type": "array", + "description": "Array of query-response pairs with optional context and ground truth", + "items": { + "$ref": "#/definitions/QueryResponseMessage" + } + } + }, + "required": [ + "items" + ], + "allOf": [ + { + "$ref": "#/definitions/InlineData" + } + ], + "x-ms-discriminator-value": "queryResponseMessage" + }, + "QueryResponseInlineMessagesUpdate": { + "type": "object", + "description": "Query-response pairs for evaluating Q&A systems and response accuracy.", + "properties": { + "items": { + "type": "array", + "description": "Array of query-response pairs with optional context and ground truth", + "items": { + "$ref": "#/definitions/QueryResponseMessage" + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/InlineDataUpdate" + } + ], + "x-ms-discriminator-value": "queryResponseMessage" + }, + "QueryResponseMessage": { + "type": "object", + "description": "Represents a message that contains a query and its corresponding response.", + "properties": { + "query": { + "type": "string", + "description": "The query message sent to the model." + }, + "response": { + "type": "string", + "description": "The response generated by the model in reply to the query." + }, + "context": { + "type": "string", + "description": "Optional context information that may include additional details about the query or response, such as metadata or processing instructions." + }, + "ground_truth": { + "type": "string", + "description": "Optional ground truth value for the query, which can be used to compare against the model's response during evaluation." + } + }, + "required": [ + "query" + ] + }, + "RedTeam": { + "type": "object", + "description": "Red team details.", + "properties": { + "id": { + "type": "string", + "description": "Identifier of the red team run.", + "readOnly": true, + "x-ms-client-name": "name" + }, + "displayName": { + "type": "string", + "description": "Name of the red-team run." + }, + "numTurns": { + "type": "integer", + "format": "int32", + "description": "Number of simulation rounds." + }, + "attackStrategies": { + "type": "array", + "description": "List of attack strategies or nested lists of attack strategies.", + "items": { + "$ref": "#/definitions/AttackStrategy" + } + }, + "simulationOnly": { + "type": "boolean", + "description": "Simulation-only or Simulation + Evaluation. Default false, if true the scan outputs conversation not evaluation result.", + "default": false + }, + "riskCategories": { + "type": "array", + "description": "List of risk categories to generate attack objectives for.", + "items": { + "$ref": "#/definitions/RiskCategory" + } + }, + "applicationScenario": { + "type": "string", + "description": "Application scenario for the red team operation, to generate scenario specific attacks." + }, + "tags": { + "type": "object", + "description": "Red team's tags. Unlike properties, tags are fully mutable.", + "additionalProperties": { + "type": "string" + } + }, + "properties": { + "type": "object", + "description": "Red team's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.", + "additionalProperties": { + "type": "string" + } + }, + "status": { + "type": "string", + "description": "Status of the red-team. It is set by service and is read-only.", + "readOnly": true + }, + "target": { + "$ref": "#/definitions/TargetConfig", + "description": "Target configuration for the red-team run." + } + }, + "required": [ + "id", + "target" + ] + }, + "RedactionLevel": { + "type": "string", + "description": "Controls the level of data redaction applied to evaluation results to protect sensitive information and ensure compliance with privacy requirements.", + "enum": [ + "sensitive", + "none" + ], + "x-ms-enum": { + "name": "RedactionLevel", + "modelAsString": true, + "values": [ + { + "name": "Sensitive", + "value": "sensitive", + "description": "Apply redaction to sensitive data in evaluation results. This is the recommended setting for production environments to ensure data privacy." + }, + { + "name": "None", + "value": "none", + "description": "Do not apply any data redaction to evaluation results. Use this setting only in development or testing environments where data privacy is not a concern." + } + ] + } + }, + "RiskCategory": { + "type": "string", + "description": "Risk category for the attack objective.", + "enum": [ + "HateUnfairness", + "Violence", + "Sexual", + "SelfHarm" + ], + "x-ms-enum": { + "name": "RiskCategory", + "modelAsString": true, + "values": [ + { + "name": "HateUnfairness", + "value": "HateUnfairness", + "description": "Represents content related to hate or unfairness." + }, + { + "name": "Violence", + "value": "Violence", + "description": "Represents content related to violence." + }, + { + "name": "Sexual", + "value": "Sexual", + "description": "Represents content of a sexual nature." + }, + { + "name": "SelfHarm", + "value": "SelfHarm", + "description": "Represents content related to self-harm." + } + ] + } + }, + "SASCredentials": { + "type": "object", + "description": "Shared Access Signature (SAS) credential definition", + "properties": { + "SAS": { + "type": "string", + "description": "SAS token", + "readOnly": true, + "x-ms-client-name": "sasToken" + } + }, + "allOf": [ + { + "$ref": "#/definitions/BaseCredentials" + } + ], + "x-ms-discriminator-value": "SAS" + }, + "SasCredential": { + "type": "object", + "description": "SAS Credential definition", + "properties": { + "sasUri": { + "type": "string", + "description": "SAS uri", + "readOnly": true + }, + "type": { + "type": "string", + "description": "Type of credential", + "enum": [ + "SAS" + ], + "x-ms-enum": { + "modelAsString": false + }, + "readOnly": true + } + }, + "required": [ + "sasUri", + "type" + ] + }, + "Sku": { + "type": "object", + "description": "Sku information", + "properties": { + "capacity": { + "type": "integer", + "format": "int64", + "description": "Sku capacity" + }, + "family": { + "type": "string", + "description": "Sku family" + }, + "name": { + "type": "string", + "description": "Sku name" + }, + "size": { + "type": "string", + "description": "Sku size" + }, + "tier": { + "type": "string", + "description": "Sku tier" + } + }, + "required": [ + "capacity", + "family", + "name", + "size", + "tier" + ] + }, + "StorageAccountDestinationConfiguration": { + "type": "object", + "description": "Configuration for exporting evaluation results to Azure Blob Storage. This provides long-term storage and enables custom data processing workflows.", + "properties": { + "containerName": { + "type": "string", + "description": "Name of the container within the Azure Blob Storage account where evaluation results will be stored" + } + }, + "required": [ + "containerName" + ], + "allOf": [ + { + "$ref": "#/definitions/EvaluationDestinationConfiguration" + } + ], + "x-ms-discriminator-value": "storageAccountStorage" + }, + "SystemMessage": { + "type": "object", + "description": "A message authored by the system to guide model behavior.", + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "system" + }, + "TargetConfig": { + "type": "object", + "description": "Abstract class for target configuration.", + "properties": { + "type": { + "type": "string", + "description": "Type of the model configuration." + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, + "TextContent": { + "type": "object", + "description": "Content for text messages in AI conversations.", + "properties": { + "text": { + "type": "string", + "description": "The text content of the message." + } + }, + "required": [ + "text" + ], + "allOf": [ + { + "$ref": "#/definitions/AIContent" + } + ], + "x-ms-discriminator-value": "text" + }, + "ToolCallContent": { + "type": "object", + "description": "Content for text messages in AI conversations.", + "properties": { + "name": { + "type": "string", + "description": "The name of the tool being called." + }, + "toolCallId": { + "type": "string", + "description": "The unique identifier of the tool call." + }, + "arguments": { + "type": "object", + "description": "The parameters for the tool call in JSON format.", + "additionalProperties": {} + } + }, + "required": [ + "name", + "toolCallId", + "arguments" + ], + "allOf": [ + { + "$ref": "#/definitions/AIContent" + } + ], + "x-ms-discriminator-value": "tool_call" + }, + "ToolResultContent": { + "type": "object", + "description": "Content for tool results in AI conversations.", + "properties": { + "results": { + "type": "object", + "description": "The result of the tool call in JSON format.", + "additionalProperties": {} + } + }, + "required": [ + "results" + ], + "allOf": [ + { + "$ref": "#/definitions/AIContent" + } + ], + "x-ms-discriminator-value": "tool_result" + }, + "UserMessage": { + "type": "object", + "description": "A message authored by the end user as input to the model.", + "allOf": [ + { + "$ref": "#/definitions/Message" + } + ], + "x-ms-discriminator-value": "user" + } + }, + "parameters": { + "Azure.Core.ClientRequestIdHeader": { + "name": "x-ms-client-request-id", + "in": "header", + "description": "An opaque, globally-unique, client-generated string identifier for the request.", + "required": false, + "type": "string", + "format": "uuid", + "x-ms-parameter-location": "method", + "x-ms-client-name": "clientRequestId" + }, + "Azure.Core.Foundations.ApiVersionParameter": { + "name": "api-version", + "in": "query", + "description": "The API version to use for this operation.", + "required": true, + "type": "string", + "minLength": 1, + "x-ms-parameter-location": "method", + "x-ms-client-name": "apiVersion" + } + } +} From 8c9d7a2132cfa8fdf77fa436be222d379c9569fc Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Tue, 22 Jul 2025 15:31:27 -0700 Subject: [PATCH 02/50] updating tspconfig.yaml --- specification/ai/Azure.AI.Projects/tspconfig.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/ai/Azure.AI.Projects/tspconfig.yaml b/specification/ai/Azure.AI.Projects/tspconfig.yaml index 4d6ccfcfebd3..01bcac4597c8 100644 --- a/specification/ai/Azure.AI.Projects/tspconfig.yaml +++ b/specification/ai/Azure.AI.Projects/tspconfig.yaml @@ -14,7 +14,7 @@ options: package-dir: "azure-ai-projects" package-name: "{package-dir}" namespace: "azure.ai.projects" - api-version: "2025-05-15-preview" + api-version: "2025-07-31-preview" flavor: azure generate-test: true generate-sample: false From f813b0649a72d569915fb26a16a665781faad88b Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 23 Jul 2025 10:56:43 -0700 Subject: [PATCH 03/50] Use flat scores --- .../Azure.AI.Projects/evaluations/models.tsp | 107 +++++---- .../2025-07-31-preview/azure-ai-projects.json | 221 ++++++------------ 2 files changed, 129 insertions(+), 199 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index c88c21b36f8e..5c35231e29b2 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -270,42 +270,42 @@ model FoundryModelDatasetSource extends EvaluationDataSource { queryField: string; } -@doc("Statistical aggregation methods available for summarizing evaluation results across multiple data points.") -@added(Versions.v2025_07_31_preview) -union AggregationMethod { - @doc("Calculate the sum of all values") - sum: "sum", +// @doc("Statistical aggregation methods available for summarizing evaluation results across multiple data points.") +// @added(Versions.v2025_07_31_preview) +// union AggregationMethod { +// @doc("Calculate the sum of all values") +// sum: "sum", - @doc("Calculate the arithmetic mean of all values") - average: "average", +// @doc("Calculate the arithmetic mean of all values") +// average: "average", - @doc("Find the maximum value") - max: "max", +// @doc("Find the maximum value") +// max: "max", - @doc("Find the minimum value") - min: "min", +// @doc("Find the minimum value") +// min: "min", - @doc("Count the total number of values") - count: "count", +// @doc("Count the total number of values") +// count: "count", - @doc("Calculate the nth percentile value. Needs a parameter to specify the percentile level (e.g., level: 95 for 95th percentile analysis)") - percentile: "percentile", +// @doc("Calculate the nth percentile value. Needs a parameter to specify the percentile level (e.g., level: 95 for 95th percentile analysis)") +// percentile: "percentile", - @doc("Calculate the standard deviation") - standardDeviation: "standardDeviation", +// @doc("Calculate the standard deviation") +// standardDeviation: "standardDeviation", - string, -} +// string, +// } -@doc("Configuration for a specific aggregation method, allowing customization of how the aggregation is calculated.") -@added(Versions.v2025_07_31_preview) -model AggregationConfiguration { - @doc("The statistical aggregation method to apply (e.g., 'average', 'percentile', 'standardDeviation')") - method: AggregationMethod; +// @doc("Configuration for a specific aggregation method, allowing customization of how the aggregation is calculated.") +// @added(Versions.v2025_07_31_preview) +// model AggregationConfiguration { +// @doc("The statistical aggregation method to apply (e.g., 'average', 'percentile', 'standardDeviation')") +// method: AggregationMethod; - @doc("Optional parameters specific to the aggregation method. For example, 'populationVariance: true' for standard deviation, or 'interpolation: linear' for percentiles.") - parameters?: Record; -} +// @doc("Optional parameters specific to the aggregation method. For example, 'populationVariance: true' for standard deviation, or 'interpolation: linear' for percentiles.") +// parameters?: Record; +// } @doc("Represents system Metadata about the evaluation execution environment and resource usage.") @added(Versions.v2025_07_31_preview) @@ -349,23 +349,38 @@ model AggregatedScore { @added(Versions.v2025_07_31_preview) @doc("Statistical summary of evaluation results including aggregated scores and pass/fail rates.") model EvaluationSummaryStatistics { - @doc("Aggregated scores calculated using specified aggregation methods. This provides a summary of the evaluator's performance across all data rows.") - aggregatedScores: Record; - - @doc("Value between 0 and 1 representing the proportion of rows where outcome is 'pass'.") - passRate: float32; - @doc("Total number of rows that were evaluated.") sampleCount: int64; + + @doc("Proportion of evaluation results that failed to meet the specified criteria or threshold, expressed as a value between 0 and 1.") + defectRate?: float32; + + @doc("Lowest score value observed across all evaluation results.") + minScore?: float32; + + @doc("Highest score value observed across all evaluation results.") + maxScore?: float32; + + @doc("Arithmetic mean of all score values in the evaluation results.") + averageScore?: float32; + + @doc("Standard deviation of score values, measuring the variability or spread of the evaluation results.") + standardDeviation?: float32; + + @doc("Lower bound of the 90% confidence interval for the evaluation scores, indicating statistical reliability of the results.") + confidenceIntervalLower?: float32; + + @doc("Upper bound of the 90% confidence interval for the evaluation scores, indicating statistical reliability of the results.") + confidenceIntervalUpper?: float32; } @doc("Aggregated summary of evaluation results for a specific evaluator across all data rows. This provides a high-level overview of the evaluator's performance.") @added(Versions.v2025_07_31_preview) -model EvaluatorResultSummary { +model EvaluatorSummaryResult { ...EvaluatorResultCommon; - @doc("Statistical summary of evaluation results.") - statistics?: EvaluationSummaryStatistics; + @doc("Statistical summary of evaluation results where key is the metric id.") + statistics?: Record; } @doc("Specifies the type of external storage destination where evaluation results can be exported for further analysis or long-term retention.") @@ -472,16 +487,6 @@ model Evaluation { @doc("Evaluators to be used for the evaluation.") evaluators: Record; - @doc(""" - Statistical aggregation methods to apply when summarizing evaluation results. Specify which aggregation functions should be calculated across all evaluation scores. For example, use 'average' to get mean scores, 'percentile' with parameter level: 95 for 95th percentile analysis, or 'standardDeviation' to measure score variability. - If not specified, average and count aggregations will be applied by default. - Key is the any label for the aggregation, which can be used to identify it in results. - Available methods: sum, average, max, min, count, percentile, standardDeviation. - Note: Aggregations are only calculated for evaluators that produce numeric scores. Non-numeric evaluators will not contribute to these aggregations. - """) - @added(Versions.v2025_07_31_preview) - aggregations?: Record; - @doc("Specifies the type and configuration of the entity used for this evaluation.") @removed(Versions.v2025_07_31_preview) target?: EvaluationTarget; @@ -489,7 +494,7 @@ model Evaluation { @doc("Aggregated summary of evaluation results for each configured evaluator. This provides a high-level overview of performance across all data rows and is available once the evaluation completes.") @visibility(Lifecycle.Read) @added(Versions.v2025_07_31_preview) - summary?: Record; + summary?: Record; @doc(""" Unique identifier of the dataset containing detailed evaluation results. This dataset is created automatically upon evaluation completion and contains row-by-row results for analysis. @@ -658,7 +663,7 @@ model EvaluatorResult { ...EvaluatorResultCommon; @doc("Score for the evaluation result") - score: EvaluationScore; + score?: Record; } @doc("Describes the desirable direction for the evaluation score. This indicates whether a higher or lower score is preferred for this evaluator.") @@ -696,7 +701,7 @@ union EvaluatorScoreType { model EvaluationScore { #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." @doc("Score of the evaluation.") - label?: boolean | string | float64; + label?: boolean | string | float32; @doc("Justification for the score, providing context on how it was derived or what it represents") justification: string; @@ -714,7 +719,7 @@ model EvaluationScore { outcome?: EvaluationResultOutcome; @doc("Optional additional details that may include metadata or other relevant information about the score") - additionalMetrics?: Record; + additionalDetails?: Record; } @doc("Comprehensive evaluation results for a single data row, including the input data, processing state, and results from all configured evaluators.") @@ -736,6 +741,6 @@ model EvaluationResult { @added(Versions.v2025_07_31_preview) inputDataJson?: string; - @doc("Collection of evaluation results from each configured evaluator for this data row. The key is the evaluator identifier and the value contains the detailed evaluation results.") + @doc("Evaluation results from all the configured evaluators for this data row.") evaluatorResults: Record; } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 41997dbd0151..f5b3a12c5f09 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1649,93 +1649,6 @@ } } }, - "AggregationConfiguration": { - "type": "object", - "description": "Configuration for a specific aggregation method, allowing customization of how the aggregation is calculated.", - "properties": { - "method": { - "$ref": "#/definitions/AggregationMethod", - "description": "The statistical aggregation method to apply (e.g., 'average', 'percentile', 'standardDeviation')" - }, - "parameters": { - "type": "object", - "description": "Optional parameters specific to the aggregation method. For example, 'populationVariance: true' for standard deviation, or 'interpolation: linear' for percentiles.", - "additionalProperties": {} - } - }, - "required": [ - "method" - ] - }, - "AggregationConfigurationUpdate": { - "type": "object", - "description": "Configuration for a specific aggregation method, allowing customization of how the aggregation is calculated.", - "properties": { - "method": { - "$ref": "#/definitions/AggregationMethod", - "description": "The statistical aggregation method to apply (e.g., 'average', 'percentile', 'standardDeviation')" - }, - "parameters": { - "type": "object", - "description": "Optional parameters specific to the aggregation method. For example, 'populationVariance: true' for standard deviation, or 'interpolation: linear' for percentiles.", - "additionalProperties": {} - } - } - }, - "AggregationMethod": { - "type": "string", - "description": "Statistical aggregation methods available for summarizing evaluation results across multiple data points.", - "enum": [ - "sum", - "average", - "max", - "min", - "count", - "percentile", - "standardDeviation" - ], - "x-ms-enum": { - "name": "AggregationMethod", - "modelAsString": true, - "values": [ - { - "name": "sum", - "value": "sum", - "description": "Calculate the sum of all values" - }, - { - "name": "average", - "value": "average", - "description": "Calculate the arithmetic mean of all values" - }, - { - "name": "max", - "value": "max", - "description": "Find the maximum value" - }, - { - "name": "min", - "value": "min", - "description": "Find the minimum value" - }, - { - "name": "count", - "value": "count", - "description": "Count the total number of values" - }, - { - "name": "percentile", - "value": "percentile", - "description": "Calculate the nth percentile value. Needs a parameter to specify the percentile level (e.g., level: 95 for 95th percentile analysis)" - }, - { - "name": "standardDeviation", - "value": "standardDeviation", - "description": "Calculate the standard deviation" - } - ] - } - }, "ApiKeyCredentials": { "type": "object", "description": "API Key Credential definition", @@ -2683,18 +2596,11 @@ "$ref": "#/definitions/EvaluatorConfiguration" } }, - "aggregations": { - "type": "object", - "description": "Statistical aggregation methods to apply when summarizing evaluation results. Specify which aggregation functions should be calculated across all evaluation scores. For example, use 'average' to get mean scores, 'percentile' with parameter level: 95 for 95th percentile analysis, or 'standardDeviation' to measure score variability.\nIf not specified, average and count aggregations will be applied by default.\nKey is the any label for the aggregation, which can be used to identify it in results.\nAvailable methods: sum, average, max, min, count, percentile, standardDeviation.\nNote: Aggregations are only calculated for evaluators that produce numeric scores. Non-numeric evaluators will not contribute to these aggregations.", - "additionalProperties": { - "$ref": "#/definitions/AggregationConfiguration" - } - }, "summary": { "type": "object", "description": "Aggregated summary of evaluation results for each configured evaluator. This provides a high-level overview of performance across all data rows and is available once the evaluation completes.", "additionalProperties": { - "$ref": "#/definitions/EvaluatorResultSummary" + "$ref": "#/definitions/EvaluatorSummaryResult" }, "readOnly": true }, @@ -2914,7 +2820,7 @@ }, "evaluatorResults": { "type": "object", - "description": "Collection of evaluation results from each configured evaluator for this data row. The key is the evaluator identifier and the value contains the detailed evaluation results.", + "description": "Evaluation results from all the configured evaluators for this data row.", "additionalProperties": { "$ref": "#/definitions/EvaluatorResult" } @@ -3007,7 +2913,7 @@ "$ref": "#/definitions/EvaluationResultOutcome", "description": "Outcome of the evaluation based on the score and threshold. Indicates whether the score meets, exceeds, or falls below expectations" }, - "additionalMetrics": { + "additionalDetails": { "type": "object", "description": "Optional additional details that may include metadata or other relevant information about the score", "additionalProperties": {} @@ -3021,27 +2927,48 @@ "type": "object", "description": "Statistical summary of evaluation results including aggregated scores and pass/fail rates.", "properties": { - "aggregatedScores": { - "type": "object", - "description": "Aggregated scores calculated using specified aggregation methods. This provides a summary of the evaluator's performance across all data rows.", - "additionalProperties": { - "$ref": "#/definitions/AggregatedScore" - } - }, - "passRate": { - "type": "number", - "format": "float", - "description": "Value between 0 and 1 representing the proportion of rows where outcome is 'pass'." - }, "sampleCount": { "type": "integer", "format": "int64", "description": "Total number of rows that were evaluated." + }, + "defectRate": { + "type": "number", + "format": "float", + "description": "Proportion of evaluation results that failed to meet the specified criteria or threshold, expressed as a value between 0 and 1." + }, + "minScore": { + "type": "number", + "format": "float", + "description": "Lowest score value observed across all evaluation results." + }, + "maxScore": { + "type": "number", + "format": "float", + "description": "Highest score value observed across all evaluation results." + }, + "averageScore": { + "type": "number", + "format": "float", + "description": "Arithmetic mean of all score values in the evaluation results." + }, + "standardDeviation": { + "type": "number", + "format": "float", + "description": "Standard deviation of score values, measuring the variability or spread of the evaluation results." + }, + "confidenceIntervalLower": { + "type": "number", + "format": "float", + "description": "Lower bound of the 90% confidence interval for the evaluation scores, indicating statistical reliability of the results." + }, + "confidenceIntervalUpper": { + "type": "number", + "format": "float", + "description": "Upper bound of the 90% confidence interval for the evaluation scores, indicating statistical reliability of the results." } }, "required": [ - "aggregatedScores", - "passRate", "sampleCount" ] }, @@ -3085,13 +3012,6 @@ "additionalProperties": { "$ref": "#/definitions/EvaluatorConfigurationUpdate" } - }, - "aggregations": { - "type": "object", - "description": "Statistical aggregation methods to apply when summarizing evaluation results. Specify which aggregation functions should be calculated across all evaluation scores. For example, use 'average' to get mean scores, 'percentile' with parameter level: 95 for 95th percentile analysis, or 'standardDeviation' to measure score variability.\nIf not specified, average and count aggregations will be applied by default.\nKey is the any label for the aggregation, which can be used to identify it in results.\nAvailable methods: sum, average, max, min, count, percentile, standardDeviation.\nNote: Aggregations are only calculated for evaluators that produce numeric scores. Non-numeric evaluators will not contribute to these aggregations.", - "additionalProperties": { - "$ref": "#/definitions/AggregationConfigurationUpdate" - } } } }, @@ -3189,14 +3109,16 @@ "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." }, "score": { - "$ref": "#/definitions/EvaluationScore", - "description": "Score for the evaluation result" + "type": "object", + "description": "Score for the evaluation result", + "additionalProperties": { + "$ref": "#/definitions/EvaluationScore" + } } }, "required": [ "evaluatorId", - "state", - "score" + "state" ] }, "EvaluatorResultCommon": { @@ -3221,32 +3143,6 @@ "state" ] }, - "EvaluatorResultSummary": { - "type": "object", - "description": "Aggregated summary of evaluation results for a specific evaluator across all data rows. This provides a high-level overview of the evaluator's performance.", - "properties": { - "evaluatorId": { - "type": "string", - "description": "Unique identifier of the evaluator that produced this result" - }, - "systemData": { - "$ref": "#/definitions/EvaluatorResultSystemData", - "description": "Metadata about the evaluation execution environment and resource usage." - }, - "state": { - "$ref": "#/definitions/Azure.Core.Foundations.OperationState", - "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." - }, - "statistics": { - "$ref": "#/definitions/EvaluationSummaryStatistics", - "description": "Statistical summary of evaluation results." - } - }, - "required": [ - "evaluatorId", - "state" - ] - }, "EvaluatorResultSystemData": { "type": "object", "description": "Represents system Metadata about the evaluation execution environment and resource usage.", @@ -3309,6 +3205,35 @@ ] } }, + "EvaluatorSummaryResult": { + "type": "object", + "description": "Aggregated summary of evaluation results for a specific evaluator across all data rows. This provides a high-level overview of the evaluator's performance.", + "properties": { + "evaluatorId": { + "type": "string", + "description": "Unique identifier of the evaluator that produced this result" + }, + "systemData": { + "$ref": "#/definitions/EvaluatorResultSystemData", + "description": "Metadata about the evaluation execution environment and resource usage." + }, + "state": { + "$ref": "#/definitions/Azure.Core.Foundations.OperationState", + "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." + }, + "statistics": { + "type": "object", + "description": "Statistical summary of evaluation results where key is the metric id.", + "additionalProperties": { + "$ref": "#/definitions/EvaluationSummaryStatistics" + } + } + }, + "required": [ + "evaluatorId", + "state" + ] + }, "FieldMapping": { "type": "object", "description": "Field mapping configuration class", From 4f2bac1f895f7057d20829a1711bc3c9aa2f204c Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 23 Jul 2025 11:40:48 -0700 Subject: [PATCH 04/50] Updating routes + Removing Agent run evaluation --- .../Azure.AI.Projects/evaluations/models.tsp | 19 -- .../Azure.AI.Projects/evaluations/routes.tsp | 2 +- .../2025-07-31-preview/azure-ai-projects.json | 170 ++++++------------ 3 files changed, 58 insertions(+), 133 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 5c35231e29b2..d108d9ad5114 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -64,9 +64,6 @@ union EvaluationDataSourceType { @doc("Use a dataset that has been registered and stored in AI Foundry workspace") Dataset: "dataset", - @doc("Use conversation data from a specific agent run execution") - FoundryAgentRun: "foundryAgentRun", - @doc("Use data generated by running a model deployment against inline queries provided in the request") FoundryModelDeploymentInline: "FoundryModelDeploymentInline", @@ -97,22 +94,6 @@ model EvaluationDataSource { type: EvaluationDataSourceType; } -@doc("Data source that uses conversation data from a specific agent run execution for evaluation. This allows evaluating agent performance based on actual conversation history.") -@added(Versions.v2025_07_31_preview) -model FoundryAgentRunSource extends EvaluationDataSource { - @doc("Specifies that this data source uses agent run data") - type: EvaluationDataSourceType.FoundryAgentRun; - - @doc("Unique identifier for the agent to evaluate.") - id: string; - - @doc("Unique identifier of the agent run to evaluate. Example: run_1234") - runId: string; - - @doc("Unique identifier of the conversation thread within the agent run. This is required when evaluating OpenAI-based agents or models to isolate specific conversation contexts.") - threadId?: string; -} - @doc("Represents a message that contains a query and its corresponding response.") @added(Versions.v2025_07_31_preview) model QueryResponseMessage { diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index b2852bbdc54d..a68a5bb84bd9 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -75,7 +75,7 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates a new evaluation with the specified configuration.") @added(Versions.v2025_07_31_preview) - @route("runs") + @route(":runs") @post createEvaluation is Azure.Core.Foundations.Operation< { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index f5b3a12c5f09..6295075582a2 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -764,61 +764,6 @@ "x-ms-pageable": { "nextLinkName": "nextLink" } - }, - "post": { - "operationId": "Evaluations_CreateEvaluation", - "description": "Creates a new evaluation with the specified configuration.", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "Repeatability-Request-ID", - "in": "header", - "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", - "required": false, - "type": "string", - "x-ms-client-name": "repeatabilityRequestId" - }, - { - "name": "Repeatability-First-Sent", - "in": "header", - "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", - "required": false, - "type": "string", - "format": "date-time", - "x-ms-client-name": "repeatabilityFirstSent" - }, - { - "name": "evaluation", - "in": "body", - "description": "Complete evaluation configuration including data source, evaluators, and result settings", - "required": true, - "schema": { - "$ref": "#/definitions/Evaluation" - } - } - ], - "responses": { - "201": { - "description": "The request has succeeded and a new resource has been created as a result.", - "schema": { - "$ref": "#/definitions/Evaluation" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - } } }, "/evaluations/runs/{name}": { @@ -1073,6 +1018,63 @@ } } }, + "/evaluations:runs": { + "post": { + "operationId": "Evaluations_CreateEvaluation", + "description": "Creates a new evaluation with the specified configuration.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "Repeatability-Request-ID", + "in": "header", + "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", + "required": false, + "type": "string", + "x-ms-client-name": "repeatabilityRequestId" + }, + { + "name": "Repeatability-First-Sent", + "in": "header", + "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", + "required": false, + "type": "string", + "format": "date-time", + "x-ms-client-name": "repeatabilityFirstSent" + }, + { + "name": "evaluation", + "in": "body", + "description": "Complete evaluation configuration including data source, evaluators, and result settings", + "required": true, + "schema": { + "$ref": "#/definitions/Evaluation" + } + } + ], + "responses": { + "201": { + "description": "The request has succeeded and a new resource has been created as a result.", + "schema": { + "$ref": "#/definitions/Evaluation" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, "/indexes": { "get": { "operationId": "Indexes_ListLatest", @@ -2645,7 +2647,6 @@ "enum": [ "inlineJsonData", "dataset", - "foundryAgentRun", "FoundryModelDeploymentInline", "foundryModelDeploymentDataset" ], @@ -2663,11 +2664,6 @@ "value": "dataset", "description": "Use a dataset that has been registered and stored in AI Foundry workspace" }, - { - "name": "FoundryAgentRun", - "value": "foundryAgentRun", - "description": "Use conversation data from a specific agent run execution" - }, { "name": "FoundryModelDeploymentInline", "value": "FoundryModelDeploymentInline", @@ -3314,58 +3310,6 @@ ], "x-ms-discriminator-value": "uri_folder" }, - "FoundryAgentRunSource": { - "type": "object", - "description": "Data source that uses conversation data from a specific agent run execution for evaluation. This allows evaluating agent performance based on actual conversation history.", - "properties": { - "id": { - "type": "string", - "description": "Unique identifier for the agent to evaluate." - }, - "runId": { - "type": "string", - "description": "Unique identifier of the agent run to evaluate. Example: run_1234" - }, - "threadId": { - "type": "string", - "description": "Unique identifier of the conversation thread within the agent run. This is required when evaluating OpenAI-based agents or models to isolate specific conversation contexts." - } - }, - "required": [ - "id", - "runId" - ], - "allOf": [ - { - "$ref": "#/definitions/EvaluationDataSource" - } - ], - "x-ms-discriminator-value": "foundryAgentRun" - }, - "FoundryAgentRunSourceUpdate": { - "type": "object", - "description": "Data source that uses conversation data from a specific agent run execution for evaluation. This allows evaluating agent performance based on actual conversation history.", - "properties": { - "id": { - "type": "string", - "description": "Unique identifier for the agent to evaluate." - }, - "runId": { - "type": "string", - "description": "Unique identifier of the agent run to evaluate. Example: run_1234" - }, - "threadId": { - "type": "string", - "description": "Unique identifier of the conversation thread within the agent run. This is required when evaluating OpenAI-based agents or models to isolate specific conversation contexts." - } - }, - "allOf": [ - { - "$ref": "#/definitions/EvaluationDataSourceUpdate" - } - ], - "x-ms-discriminator-value": "foundryAgentRun" - }, "FoundryDatasetDataSource": { "type": "object", "description": "Data source that uses a dataset registered and stored in AI Foundry workspace. This is the recommended approach for large datasets or reusable evaluation data.", From 0468dfb8a03eb002416760ab88bcb68d8063139e Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 23 Jul 2025 13:54:53 -0700 Subject: [PATCH 05/50] rename items to messages --- .../ai/Azure.AI.Projects/evaluations/models.tsp | 4 ++-- .../2025-07-31-preview/azure-ai-projects.json | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index d108d9ad5114..581ac8e6e179 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -152,7 +152,7 @@ model QueryResponseInlineMessages extends InlineData { dataFormat: InlineDataFormat.queryResponseMessage; @doc("Array of query-response pairs with optional context and ground truth") - items: QueryResponseMessage[]; + messages: QueryResponseMessage[]; } @added(Versions.v2025_07_31_preview) @@ -178,7 +178,7 @@ model InlineJson extends InlineData { dataFormat: InlineDataFormat.inlineJson; @doc("Array of JSON strings with custom fields and metadata") - items: string[]; + messages: string[]; } @doc("Data source using inline data provided directly in the request.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 6295075582a2..68e6a038884b 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3638,7 +3638,7 @@ "type": "object", "description": "Custom JSON format for complex evaluation scenarios requiring flexible data structures.", "properties": { - "items": { + "messages": { "type": "array", "description": "Array of JSON strings with custom fields and metadata", "items": { @@ -3647,7 +3647,7 @@ } }, "required": [ - "items" + "messages" ], "allOf": [ { @@ -3703,7 +3703,7 @@ "type": "object", "description": "Custom JSON format for complex evaluation scenarios requiring flexible data structures.", "properties": { - "items": { + "messages": { "type": "array", "description": "Array of JSON strings with custom fields and metadata", "items": { @@ -4139,7 +4139,7 @@ "type": "object", "description": "Query-response pairs for evaluating Q&A systems and response accuracy.", "properties": { - "items": { + "messages": { "type": "array", "description": "Array of query-response pairs with optional context and ground truth", "items": { @@ -4148,7 +4148,7 @@ } }, "required": [ - "items" + "messages" ], "allOf": [ { @@ -4161,7 +4161,7 @@ "type": "object", "description": "Query-response pairs for evaluating Q&A systems and response accuracy.", "properties": { - "items": { + "messages": { "type": "array", "description": "Array of query-response pairs with optional context and ground truth", "items": { From a6a622dcd513beb087bac8b38aa26ca2a0d0387d Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 23 Jul 2025 14:04:13 -0700 Subject: [PATCH 06/50] Updating InlineJsonDataSource to InlineDataSource --- .../Azure.AI.Projects/evaluations/models.tsp | 6 +- .../2025-07-31-preview/azure-ai-projects.json | 86 +++++++++---------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 581ac8e6e179..ec293df1ab41 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -59,7 +59,7 @@ scalar DatasetId extends string; @added(Versions.v2025_07_31_preview) union EvaluationDataSourceType { @doc("Use inline JSON data provided directly in the request") - InlineJsonData: "inlineJsonData", + InlineData: "inlineData", @doc("Use a dataset that has been registered and stored in AI Foundry workspace") Dataset: "dataset", @@ -183,9 +183,9 @@ model InlineJson extends InlineData { @doc("Data source using inline data provided directly in the request.") @added(Versions.v2025_07_31_preview) -model InlineJsonDataSource extends EvaluationDataSource { +model InlineDataSource extends EvaluationDataSource { @doc("Specifies inline JSON data source") - type: EvaluationDataSourceType.InlineJsonData; + type: EvaluationDataSourceType.InlineData; @doc("Optional unique identifier for the inline data source. This can be an agent id or a custom identifier to distinguish between different inline data sources.") id?: string; diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 68e6a038884b..d866f0d3330c 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -2645,7 +2645,7 @@ "type": "string", "description": "Specifies the type of data source used for evaluation. Different types support various data input methods and formats.", "enum": [ - "inlineJsonData", + "inlineData", "dataset", "FoundryModelDeploymentInline", "foundryModelDeploymentDataset" @@ -2655,8 +2655,8 @@ "modelAsString": true, "values": [ { - "name": "InlineJsonData", - "value": "inlineJsonData", + "name": "InlineData", + "value": "inlineData", "description": "Use inline JSON data provided directly in the request" }, { @@ -3620,43 +3620,7 @@ ] } }, - "InlineDataUpdate": { - "type": "object", - "description": "Base class for inline evaluation data with format discrimination.", - "properties": { - "dataFormat": { - "$ref": "#/definitions/InlineDataFormat", - "description": "Format of the inline data structure" - } - }, - "discriminator": "dataFormat", - "required": [ - "dataFormat" - ] - }, - "InlineJson": { - "type": "object", - "description": "Custom JSON format for complex evaluation scenarios requiring flexible data structures.", - "properties": { - "messages": { - "type": "array", - "description": "Array of JSON strings with custom fields and metadata", - "items": { - "type": "string" - } - } - }, - "required": [ - "messages" - ], - "allOf": [ - { - "$ref": "#/definitions/InlineData" - } - ], - "x-ms-discriminator-value": "inlineJson" - }, - "InlineJsonDataSource": { + "InlineDataSource": { "type": "object", "description": "Data source using inline data provided directly in the request.", "properties": { @@ -3677,9 +3641,9 @@ "$ref": "#/definitions/EvaluationDataSource" } ], - "x-ms-discriminator-value": "inlineJsonData" + "x-ms-discriminator-value": "inlineData" }, - "InlineJsonDataSourceUpdate": { + "InlineDataSourceUpdate": { "type": "object", "description": "Data source using inline data provided directly in the request.", "properties": { @@ -3697,7 +3661,43 @@ "$ref": "#/definitions/EvaluationDataSourceUpdate" } ], - "x-ms-discriminator-value": "inlineJsonData" + "x-ms-discriminator-value": "inlineData" + }, + "InlineDataUpdate": { + "type": "object", + "description": "Base class for inline evaluation data with format discrimination.", + "properties": { + "dataFormat": { + "$ref": "#/definitions/InlineDataFormat", + "description": "Format of the inline data structure" + } + }, + "discriminator": "dataFormat", + "required": [ + "dataFormat" + ] + }, + "InlineJson": { + "type": "object", + "description": "Custom JSON format for complex evaluation scenarios requiring flexible data structures.", + "properties": { + "messages": { + "type": "array", + "description": "Array of JSON strings with custom fields and metadata", + "items": { + "type": "string" + } + } + }, + "required": [ + "messages" + ], + "allOf": [ + { + "$ref": "#/definitions/InlineData" + } + ], + "x-ms-discriminator-value": "inlineJson" }, "InlineJsonUpdate": { "type": "object", From c6c4682b22db915634ea29bca0b66253bea2d549 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 23 Jul 2025 14:21:16 -0700 Subject: [PATCH 07/50] Updating discriminator values --- .../Azure.AI.Projects/evaluations/models.tsp | 8 ++++---- .../2025-07-31-preview/azure-ai-projects.json | 20 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index ec293df1ab41..3712da2696c8 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -65,10 +65,10 @@ union EvaluationDataSourceType { Dataset: "dataset", @doc("Use data generated by running a model deployment against inline queries provided in the request") - FoundryModelDeploymentInline: "FoundryModelDeploymentInline", + FoundryModelInline: "FoundryModelInline", @doc("Use data generated by running a model deployment against a registered dataset") - FoundryModelDeploymentDataset: "foundryModelDeploymentDataset", + FoundryModelDataset: "FoundryModelDataset", string, } @@ -228,7 +228,7 @@ model EvaluationModelSourceCommon { @added(Versions.v2025_07_31_preview) model FoundryModelInlineSource extends EvaluationDataSource { @doc("Specifies that this data source uses a model deployment with inline queries") - type: EvaluationDataSourceType.FoundryModelDeploymentInline; + type: EvaluationDataSourceType.FoundryModelInline; ...EvaluationModelSourceCommon; @@ -240,7 +240,7 @@ model FoundryModelInlineSource extends EvaluationDataSource { @added(Versions.v2025_07_31_preview) model FoundryModelDatasetSource extends EvaluationDataSource { @doc("Specifies that this data source uses a model deployment with a dataset") - type: EvaluationDataSourceType.FoundryModelDeploymentDataset; + type: EvaluationDataSourceType.FoundryModelDataset; ...EvaluationModelSourceCommon; diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index d866f0d3330c..a17c5cc1b6a5 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -2647,8 +2647,8 @@ "enum": [ "inlineData", "dataset", - "FoundryModelDeploymentInline", - "foundryModelDeploymentDataset" + "FoundryModelInline", + "FoundryModelDataset" ], "x-ms-enum": { "name": "EvaluationDataSourceType", @@ -2665,13 +2665,13 @@ "description": "Use a dataset that has been registered and stored in AI Foundry workspace" }, { - "name": "FoundryModelDeploymentInline", - "value": "FoundryModelDeploymentInline", + "name": "FoundryModelInline", + "value": "FoundryModelInline", "description": "Use data generated by running a model deployment against inline queries provided in the request" }, { - "name": "FoundryModelDeploymentDataset", - "value": "foundryModelDeploymentDataset", + "name": "FoundryModelDataset", + "value": "FoundryModelDataset", "description": "Use data generated by running a model deployment against a registered dataset" } ] @@ -3380,7 +3380,7 @@ "$ref": "#/definitions/EvaluationDataSource" } ], - "x-ms-discriminator-value": "foundryModelDeploymentDataset" + "x-ms-discriminator-value": "FoundryModelDataset" }, "FoundryModelDatasetSourceUpdate": { "type": "object", @@ -3411,7 +3411,7 @@ "$ref": "#/definitions/EvaluationDataSourceUpdate" } ], - "x-ms-discriminator-value": "foundryModelDeploymentDataset" + "x-ms-discriminator-value": "FoundryModelDataset" }, "FoundryModelInlineSource": { "type": "object", @@ -3446,7 +3446,7 @@ "$ref": "#/definitions/EvaluationDataSource" } ], - "x-ms-discriminator-value": "FoundryModelDeploymentInline" + "x-ms-discriminator-value": "FoundryModelInline" }, "FoundryModelInlineSourceUpdate": { "type": "object", @@ -3476,7 +3476,7 @@ "$ref": "#/definitions/EvaluationDataSourceUpdate" } ], - "x-ms-discriminator-value": "FoundryModelDeploymentInline" + "x-ms-discriminator-value": "FoundryModelInline" }, "ImageUrlContent": { "type": "object", From f0063795c62b03ed55cfe5f17deb7b898fa14c09 Mon Sep 17 00:00:00 2001 From: Kayla Seager Date: Thu, 24 Jul 2025 13:56:51 -0700 Subject: [PATCH 08/50] initial modeling for real-time api evaluation (#36147) * initial modeling for real-time api: single evaluation * update given comments * add optional storage for customers * remove plural * updates given the SDK review * remove the tsp generated json for merge ease * OneEval is not a separate resource type * remove additional storage option for sync. Clients will have to manage this --- .../Azure.AI.Projects/evaluations/models.tsp | 13 ++++++++++ .../Azure.AI.Projects/evaluations/routes.tsp | 25 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 3712da2696c8..123108aaed53 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -421,6 +421,19 @@ model EvaluationResultSettings { additionalDestinations?: Array; } +@doc("One Evaluation Definition") +@added(Versions.v2025_07_31_preview) +model OneEvaluation { + @doc("Input data to evaluate. Single row support only.") + dataSource: InlineDataSource; + + @doc("Evaluators to be used for the evaluation.") + evaluators: Record; + + @doc("Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.") + properties?: Record; +} + @doc("Evaluation Definition") @resource("runs") @added(Versions.v2025_05_15_preview) diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index a68a5bb84bd9..bb87451a89ce 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -3,6 +3,7 @@ import "@azure-tools/typespec-autorest"; import "@typespec/versioning"; import "@azure-tools/typespec-azure-core"; import "./models.tsp"; +import "@typespec/http"; using TypeSpec.Http; using Azure.Core; @@ -30,7 +31,7 @@ alias DisplayPreferencesTypeParameterTrait = { filter?: string; }; -alias EvaluationsOperations = Azure.Core.ResourceOperations; +alias EvaluationsOperations = ResourceOperations; @route("evaluations") @added(Versions.v2025_05_15_preview) @@ -94,6 +95,28 @@ interface Evaluations { ResourceCreatedResponse >; + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" + @doc("Independent API operation to perform a single evaluation and immediately get a result.") + @added(Versions.v2025_07_31_preview) + @route(":evaluateOne") + @post + createOneEvaluation is Azure.Core.Foundations.Operation< + { + @header("Repeatability-Request-ID") + @doc("Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.") + repeatabilityRequestId?: string; + + @doc("Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.") + @header("Repeatability-First-Sent") + repeatabilityFirstSent?: utcDateTime; + + @doc("Complete evaluation configuration including data source, evaluators, and result settings") + @body + oneEvaluation: OneEvaluation; + }, + ResourceCreatedResponse + >; + @doc("Updates specific properties of an existing evaluation. Supports modification of metadata fields including description, display name, and tags. Note: Core evaluation configuration such as data sources and evaluators cannot be modified after creation.") @added(Versions.v2025_07_31_preview) update is EvaluationsOperations.ResourceUpdate; From 321cb1ab4a48cd8c650729119a8cf6d602b5e422 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Sun, 27 Jul 2025 22:02:01 -0700 Subject: [PATCH 09/50] Merge --- .../2025-07-31-preview/azure-ai-projects.json | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index a17c5cc1b6a5..080fb9744a16 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1018,6 +1018,63 @@ } } }, + "/evaluations:evaluateOne": { + "post": { + "operationId": "Evaluations_CreateOneEvaluation", + "description": "Independent API operation to perform a single evaluation and immediately get a result.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "Repeatability-Request-ID", + "in": "header", + "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", + "required": false, + "type": "string", + "x-ms-client-name": "repeatabilityRequestId" + }, + { + "name": "Repeatability-First-Sent", + "in": "header", + "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", + "required": false, + "type": "string", + "format": "date-time", + "x-ms-client-name": "repeatabilityFirstSent" + }, + { + "name": "oneEvaluation", + "in": "body", + "description": "Complete evaluation configuration including data source, evaluators, and result settings", + "required": true, + "schema": { + "$ref": "#/definitions/OneEvaluation" + } + } + ], + "responses": { + "201": { + "description": "The request has succeeded and a new resource has been created as a result.", + "schema": { + "$ref": "#/definitions/EvaluationResult" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + } + } + }, "/evaluations:runs": { "post": { "operationId": "Evaluations_CreateEvaluation", @@ -3886,6 +3943,34 @@ ], "x-ms-discriminator-value": "None" }, + "OneEvaluation": { + "type": "object", + "description": "One Evaluation Definition", + "properties": { + "dataSource": { + "$ref": "#/definitions/InlineDataSource", + "description": "Input data to evaluate. Single row support only." + }, + "evaluators": { + "type": "object", + "description": "Evaluators to be used for the evaluation.", + "additionalProperties": { + "$ref": "#/definitions/EvaluatorConfiguration" + } + }, + "properties": { + "type": "object", + "description": "Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "dataSource", + "evaluators" + ] + }, "PagedConnection": { "type": "object", "description": "Paged collection of Connection items", From ec4154a2504366d08a81047c661358bac7281451 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 28 Jul 2025 12:52:46 -0700 Subject: [PATCH 10/50] Update result model --- .../Azure.AI.Projects/evaluations/models.tsp | 245 ++++------ .../2025-07-31-preview/azure-ai-projects.json | 431 +++++++----------- 2 files changed, 246 insertions(+), 430 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 123108aaed53..4b251f7503ef 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -73,18 +73,6 @@ union EvaluationDataSourceType { string, } -@doc("Controls the level of data redaction applied to evaluation results to protect sensitive information and ensure compliance with privacy requirements.") -@added(Versions.v2025_07_31_preview) -union RedactionLevel { - @doc("Apply redaction to sensitive data in evaluation results. This is the recommended setting for production environments to ensure data privacy.") - Sensitive: "sensitive", - - @doc("Do not apply any data redaction to evaluation results. Use this setting only in development or testing environments where data privacy is not a concern.") - None: "none", - - string, -} - #suppress "@azure-tools/typespec-azure-core/no-string-discriminator" @doc("Base class for different types of evaluation data sources. Use the discriminator field 'type' to specify the specific data source implementation.") @discriminator("type") @@ -110,29 +98,17 @@ model QueryResponseMessage { ground_truth?: string; } -@doc("The type of inline messages used for evaluation. This union allows for different inline message formats to be specified.") -@added(Versions.v2025_07_31_preview) -union InlineMessagesType { - @doc("Inline messages that consist of query-response pairs for evaluation. Each pair includes a query string and the corresponding response generated by the model.") - queryResponseMessage: "queryResponseMessage", - - @doc("role content type message") - roleContentType: "roleContentType", - - string, -} - @doc("Available formats for structuring inline evaluation data.") @added(Versions.v2025_07_31_preview) union InlineDataFormat { @doc("Query-response pairs for Q&A and chatbot evaluations") - queryResponseMessage: "queryResponseMessage", + queryResponseMessageFormat: "queryResponseMessageFormat", @doc("Multi-turn conversations with role-based messages and tool definitions") - chatMessages: "chatMessages", + chatMessagesFormat: "chatMessagesFormat", @doc("Flexible JSON format for custom data structures") - inlineJson: "inlineJson", + inlineJsonFormat: "inlineJsonFormat", string, } @@ -149,7 +125,7 @@ model InlineData { @added(Versions.v2025_07_31_preview) model QueryResponseInlineMessages extends InlineData { @doc("Specifies query-response format") - dataFormat: InlineDataFormat.queryResponseMessage; + dataFormat: InlineDataFormat.queryResponseMessageFormat; @doc("Array of query-response pairs with optional context and ground truth") messages: QueryResponseMessage[]; @@ -159,7 +135,7 @@ model QueryResponseInlineMessages extends InlineData { @doc("Multi-turn conversations for evaluating dialogue systems and context awareness.") model ChatMessages extends InlineData { @doc("Specifies chat messages format") - dataFormat: InlineDataFormat.chatMessages; + dataFormat: InlineDataFormat.chatMessagesFormat; @doc("Array of messages representing representing queries") query: Message[]; @@ -175,7 +151,7 @@ model ChatMessages extends InlineData { @doc("Custom JSON format for complex evaluation scenarios requiring flexible data structures.") model InlineJson extends InlineData { @doc("Specifies JSON format") - dataFormat: InlineDataFormat.inlineJson; + dataFormat: InlineDataFormat.inlineJsonFormat; @doc("Array of JSON strings with custom fields and metadata") messages: string[]; @@ -251,82 +227,6 @@ model FoundryModelDatasetSource extends EvaluationDataSource { queryField: string; } -// @doc("Statistical aggregation methods available for summarizing evaluation results across multiple data points.") -// @added(Versions.v2025_07_31_preview) -// union AggregationMethod { -// @doc("Calculate the sum of all values") -// sum: "sum", - -// @doc("Calculate the arithmetic mean of all values") -// average: "average", - -// @doc("Find the maximum value") -// max: "max", - -// @doc("Find the minimum value") -// min: "min", - -// @doc("Count the total number of values") -// count: "count", - -// @doc("Calculate the nth percentile value. Needs a parameter to specify the percentile level (e.g., level: 95 for 95th percentile analysis)") -// percentile: "percentile", - -// @doc("Calculate the standard deviation") -// standardDeviation: "standardDeviation", - -// string, -// } - -// @doc("Configuration for a specific aggregation method, allowing customization of how the aggregation is calculated.") -// @added(Versions.v2025_07_31_preview) -// model AggregationConfiguration { -// @doc("The statistical aggregation method to apply (e.g., 'average', 'percentile', 'standardDeviation')") -// method: AggregationMethod; - -// @doc("Optional parameters specific to the aggregation method. For example, 'populationVariance: true' for standard deviation, or 'interpolation: linear' for percentiles.") -// parameters?: Record; -// } - -@doc("Represents system Metadata about the evaluation execution environment and resource usage.") -@added(Versions.v2025_07_31_preview) -model EvaluatorResultSystemData { - @doc("Total number of input tokens consumed during the evaluation process. This helps track resource usage and costs.") - inputTokenCount: int64; - - @doc("Total number of output tokens generated during the evaluation process. This helps track resource usage and costs.") - outputTokenCount: int64; - - @doc("Total time taken to complete the evaluation, measured in milliseconds. This includes processing time and any network latency.") - duration: int64; - - @doc("Additional details about the evaluator result") - additionalDetails: Record; -} - -@doc("Common properties shared across all evaluator results, including metadata about the evaluation execution and resource consumption.") -@added(Versions.v2025_07_31_preview) -model EvaluatorResultCommon { - @doc("Unique identifier of the evaluator that produced this result") - evaluatorId: string; - - @doc("Metadata about the evaluation execution environment and resource usage.") - systemData?: EvaluatorResultSystemData; - - @doc("Current state of the evaluation execution. Tracks progress from initiation to completion or failure.") - state: OperationState; -} - -@added(Versions.v2025_07_31_preview) -@doc("Represents an aggregated score calculated using a specific statistical method across all rows of data.") -model AggregatedScore { - @doc("The calculated aggregated value based on the specified method. This value is undefined in case aggregation is not applicable or the method does not produce a numeric result.") - value?: float32; - - @doc("Additional metadata about the aggregation calculation") - metadata?: Record; -} - @added(Versions.v2025_07_31_preview) @doc("Statistical summary of evaluation results including aggregated scores and pass/fail rates.") model EvaluationSummaryStatistics { @@ -336,32 +236,27 @@ model EvaluationSummaryStatistics { @doc("Proportion of evaluation results that failed to meet the specified criteria or threshold, expressed as a value between 0 and 1.") defectRate?: float32; + @doc("Proportion of evaluation results that passed the specified criteria or threshold, expressed as a value between 0 and 1.") + passRate?: float32; + @doc("Lowest score value observed across all evaluation results.") - minScore?: float32; + min?: float32; @doc("Highest score value observed across all evaluation results.") - maxScore?: float32; + max?: float32; @doc("Arithmetic mean of all score values in the evaluation results.") - averageScore?: float32; - - @doc("Standard deviation of score values, measuring the variability or spread of the evaluation results.") - standardDeviation?: float32; - - @doc("Lower bound of the 90% confidence interval for the evaluation scores, indicating statistical reliability of the results.") - confidenceIntervalLower?: float32; - - @doc("Upper bound of the 90% confidence interval for the evaluation scores, indicating statistical reliability of the results.") - confidenceIntervalUpper?: float32; + average?: float32; } -@doc("Aggregated summary of evaluation results for a specific evaluator across all data rows. This provides a high-level overview of the evaluator's performance.") +@doc("Status of evaluator across all data rows.") @added(Versions.v2025_07_31_preview) -model EvaluatorSummaryResult { - ...EvaluatorResultCommon; +model EvaluatorStatusResult { + @doc("State of the evaluation for this evaluator.") + state: OperationState; - @doc("Statistical summary of evaluation results where key is the metric id.") - statistics?: Record; + @doc("Usage statistics for the evaluation") + usage: Usage; } @doc("Specifies the type of external storage destination where evaluation results can be exported for further analysis or long-term retention.") @@ -413,9 +308,6 @@ model StorageAccountDestinationConfiguration @doc("Configuration settings that control how evaluation results are processed, stored, and exported. These settings affect data privacy, retention, and integration with external systems.") @added(Versions.v2025_07_31_preview) model EvaluationResultSettings { - @doc("Level of data redaction to apply to evaluation results. Defaults to 'sensitive' in production environments to protect confidential information.") - redactionLevel?: RedactionLevel; - @doc("List of external storage destinations where evaluation results should be exported in addition to the default AI Foundry storage") @added(Versions.v2025_07_31_preview) additionalDestinations?: Array; @@ -434,6 +326,32 @@ model OneEvaluation { properties?: Record; } +@doc("Aggregated metric that summarizes evaluation results across multiple data rows.") +@added(Versions.v2025_07_31_preview) +model AggregatedMetric { + @doc("Name of the metric") + name: string; + + @doc("Alias provided for the evaluator in the input request.") + evaluatorAlias: string; + + @doc("Statistical summary of evaluation results including aggregated scores and pass/fail rates.") + statistics: EvaluationSummaryStatistics; + + @doc("Additional metadata about the aggregated metric, such as processing details or custom fields") + additionalDetails?: Record; +} + +@added(Versions.v2025_07_31_preview) +@doc("Summary of evaluation results, including aggregated metrics and evaluator statuses.") +model EvaluationSummaryResult { + @doc("Status of each evaluator from the input request.") + evaluatorStatus: Record; + + @doc("Aggregated metrics that summarize evaluation results across multiple data rows. This provides a high-level overview of performance across all data rows.") + metrics?: AggregatedMetric[]; +} + @doc("Evaluation Definition") @resource("runs") @added(Versions.v2025_05_15_preview) @@ -485,10 +403,10 @@ model Evaluation { @removed(Versions.v2025_07_31_preview) target?: EvaluationTarget; - @doc("Aggregated summary of evaluation results for each configured evaluator. This provides a high-level overview of performance across all data rows and is available once the evaluation completes.") + @doc("Aggregated summary of evaluation results for each configured evaluator. This provides a high-level overview of performance across all data rows.") @visibility(Lifecycle.Read) @added(Versions.v2025_07_31_preview) - summary?: Record; + summary?: EvaluationSummaryResult; @doc(""" Unique identifier of the dataset containing detailed evaluation results. This dataset is created automatically upon evaluation completion and contains row-by-row results for analysis. @@ -651,15 +569,6 @@ union EvaluationResultOutcome { string, } -@doc("Detailed result of a specific evaluator's assessment on a single data row. Contains the evaluation outcome, score, reasoning, and additional metrics.") -@added(Versions.v2025_07_31_preview) -model EvaluatorResult { - ...EvaluatorResultCommon; - - @doc("Score for the evaluation result") - score?: Record; -} - @doc("Describes the desirable direction for the evaluation score. This indicates whether a higher or lower score is preferred for this evaluator.") @added(Versions.v2025_07_31_preview) union EvaluatorDesirableDirection { @@ -676,43 +585,50 @@ union EvaluatorDesirableDirection { } @added(Versions.v2025_07_31_preview) -@doc("Type of score assigned by the evaluator. This indicates whether the score is boolean, continuous, or ordinal.") -union EvaluatorScoreType { - @doc("Boolean score type, where the score is either true or false") - boolean: "boolean", +@doc("Usage statistics for the evaluation") +model Usage { + @doc("Total number of input tokens consumed during the evaluation process") + inputTokens: int64; - @doc("Continuous score type, where the score is a floating-point number representing a continuous value") - continuous: "continuous", - - @doc("Ordinal score type, where the score represents an ordered category or rank") - ordinal: "ordinal", - - string, + @doc("Total number of output tokens generated during the evaluation process") + outputTokens: int64; } @added(Versions.v2025_07_31_preview) -@doc("Score assigned by the evaluator to a specific data row") -model EvaluationScore { - #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." - @doc("Score of the evaluation.") - label?: boolean | string | float32; +@doc("Metadata about the evaluation result") +model EvaluationResultMetadata { + @doc("Unique identifier of the evaluator that produced this result") + evaluatorId: string; - @doc("Justification for the score, providing context on how it was derived or what it represents") - justification: string; + @doc("Desired direction for the evaluation score.") + desiredDirection: EvaluatorDesirableDirection; @doc("Threshold value that this score is compared against to determine the evaluation outcome") threshold?: float32; +} - @doc("Direction of the score that indicates whether a higher or lower score is desirable for this evaluator") - desirableDirection?: EvaluatorDesirableDirection; +@added(Versions.v2025_07_31_preview) +@doc("Metric representing the evaluation result for a specific evaluator.") +model EvaluationMetric { + @doc("Name of the evaluation metric") + name: string; - @doc("Type of the score that indicates whether it is a boolean, continuous, or ordinal score") - scoreType?: EvaluatorScoreType; + @doc("Alias provided for the evaluator in the input request.") + evaluatorAlias: string; - @doc("Outcome of the evaluation based on the score and threshold. Indicates whether the score meets, exceeds, or falls below expectations") - outcome?: EvaluationResultOutcome; + @doc("Score assigned by the evaluator to the specific data row") + score: float32; - @doc("Optional additional details that may include metadata or other relevant information about the score") + @doc("Outcome of the evaluation") + outcome: EvaluationResultOutcome; + + @doc("Reasoning or explanation provided by the evaluator for the score assigned to this data row") + reasoning?: string; + + @doc("Metadata about the evaluation result") + metadata: EvaluationResultMetadata; + + @doc("Additional metadata about the evaluation result, such as processing details or custom fields") additionalDetails?: Record; } @@ -735,6 +651,9 @@ model EvaluationResult { @added(Versions.v2025_07_31_preview) inputDataJson?: string; + @doc("Usage statistics for the evaluation") + usage?: Usage; + @doc("Evaluation results from all the configured evaluators for this data row.") - evaluatorResults: Record; + metrics?: EvaluationMetric[]; } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 080fb9744a16..76ed944d234f 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1692,21 +1692,33 @@ "parameters" ] }, - "AggregatedScore": { + "AggregatedMetric": { "type": "object", - "description": "Represents an aggregated score calculated using a specific statistical method across all rows of data.", + "description": "Aggregated metric that summarizes evaluation results across multiple data rows.", "properties": { - "value": { - "type": "number", - "format": "float", - "description": "The calculated aggregated value based on the specified method. This value is undefined in case aggregation is not applicable or the method does not produce a numeric result." + "name": { + "type": "string", + "description": "Name of the metric" }, - "metadata": { + "evaluatorAlias": { + "type": "string", + "description": "Alias provided for the evaluator in the input request." + }, + "statistics": { + "$ref": "#/definitions/EvaluationSummaryStatistics", + "description": "Statistical summary of evaluation results including aggregated scores and pass/fail rates." + }, + "additionalDetails": { "type": "object", - "description": "Additional metadata about the aggregation calculation", + "description": "Additional metadata about the aggregated metric, such as processing details or custom fields", "additionalProperties": {} } - } + }, + "required": [ + "name", + "evaluatorAlias", + "statistics" + ] }, "ApiKeyCredentials": { "type": "object", @@ -2147,7 +2159,7 @@ "$ref": "#/definitions/InlineData" } ], - "x-ms-discriminator-value": "chatMessages" + "x-ms-discriminator-value": "chatMessagesFormat" }, "ChatMessagesUpdate": { "type": "object", @@ -2180,7 +2192,7 @@ "$ref": "#/definitions/InlineDataUpdate" } ], - "x-ms-discriminator-value": "chatMessages" + "x-ms-discriminator-value": "chatMessagesFormat" }, "Connection": { "type": "object", @@ -2656,11 +2668,8 @@ } }, "summary": { - "type": "object", - "description": "Aggregated summary of evaluation results for each configured evaluator. This provides a high-level overview of performance across all data rows and is available once the evaluation completes.", - "additionalProperties": { - "$ref": "#/definitions/EvaluatorSummaryResult" - }, + "$ref": "#/definitions/EvaluationSummaryResult", + "description": "Aggregated summary of evaluation results for each configured evaluator. This provides a high-level overview of performance across all data rows.", "readOnly": true }, "resultDatasetId": { @@ -2796,6 +2805,49 @@ ] } }, + "EvaluationMetric": { + "type": "object", + "description": "Metric representing the evaluation result for a specific evaluator.", + "properties": { + "name": { + "type": "string", + "description": "Name of the evaluation metric" + }, + "evaluatorAlias": { + "type": "string", + "description": "Alias provided for the evaluator in the input request." + }, + "score": { + "type": "number", + "format": "float", + "description": "Score assigned by the evaluator to the specific data row" + }, + "outcome": { + "$ref": "#/definitions/EvaluationResultOutcome", + "description": "Outcome of the evaluation" + }, + "reasoning": { + "type": "string", + "description": "Reasoning or explanation provided by the evaluator for the score assigned to this data row" + }, + "metadata": { + "$ref": "#/definitions/EvaluationResultMetadata", + "description": "Metadata about the evaluation result" + }, + "additionalDetails": { + "type": "object", + "description": "Additional metadata about the evaluation result, such as processing details or custom fields", + "additionalProperties": {} + } + }, + "required": [ + "name", + "evaluatorAlias", + "score", + "outcome", + "metadata" + ] + }, "EvaluationModelConfiguration": { "type": "object", "description": "Comprehensive configuration for a model deployment used in evaluation scenarios. This defines how the model will process inputs and generate responses for evaluation.", @@ -2871,18 +2923,44 @@ "type": "string", "description": "Original input data for this row in JSON string format. This preserves the exact data that was evaluated for reference and debugging." }, - "evaluatorResults": { - "type": "object", + "usage": { + "$ref": "#/definitions/Usage", + "description": "Usage statistics for the evaluation" + }, + "metrics": { + "type": "array", "description": "Evaluation results from all the configured evaluators for this data row.", - "additionalProperties": { - "$ref": "#/definitions/EvaluatorResult" + "items": { + "$ref": "#/definitions/EvaluationMetric" } } }, "required": [ "id", - "state", - "evaluatorResults" + "state" + ] + }, + "EvaluationResultMetadata": { + "type": "object", + "description": "Metadata about the evaluation result", + "properties": { + "evaluatorId": { + "type": "string", + "description": "Unique identifier of the evaluator that produced this result" + }, + "desiredDirection": { + "$ref": "#/definitions/EvaluatorDesirableDirection", + "description": "Desired direction for the evaluation score." + }, + "threshold": { + "type": "number", + "format": "float", + "description": "Threshold value that this score is compared against to determine the evaluation outcome" + } + }, + "required": [ + "evaluatorId", + "desiredDirection" ] }, "EvaluationResultOutcome": { @@ -2925,10 +3003,6 @@ "type": "object", "description": "Configuration settings that control how evaluation results are processed, stored, and exported. These settings affect data privacy, retention, and integration with external systems.", "properties": { - "redactionLevel": { - "$ref": "#/definitions/RedactionLevel", - "description": "Level of data redaction to apply to evaluation results. Defaults to 'sensitive' in production environments to protect confidential information." - }, "additionalDestinations": { "type": "array", "description": "List of external storage destinations where evaluation results should be exported in addition to the default AI Foundry storage", @@ -2938,42 +3012,27 @@ } } }, - "EvaluationScore": { + "EvaluationSummaryResult": { "type": "object", - "description": "Score assigned by the evaluator to a specific data row", + "description": "Summary of evaluation results, including aggregated metrics and evaluator statuses.", "properties": { - "label": { - "description": "Score of the evaluation." - }, - "justification": { - "type": "string", - "description": "Justification for the score, providing context on how it was derived or what it represents" - }, - "threshold": { - "type": "number", - "format": "float", - "description": "Threshold value that this score is compared against to determine the evaluation outcome" - }, - "desirableDirection": { - "$ref": "#/definitions/EvaluatorDesirableDirection", - "description": "Direction of the score that indicates whether a higher or lower score is desirable for this evaluator" - }, - "scoreType": { - "$ref": "#/definitions/EvaluatorScoreType", - "description": "Type of the score that indicates whether it is a boolean, continuous, or ordinal score" - }, - "outcome": { - "$ref": "#/definitions/EvaluationResultOutcome", - "description": "Outcome of the evaluation based on the score and threshold. Indicates whether the score meets, exceeds, or falls below expectations" - }, - "additionalDetails": { + "evaluatorStatus": { "type": "object", - "description": "Optional additional details that may include metadata or other relevant information about the score", - "additionalProperties": {} + "description": "Status of each evaluator from the input request.", + "additionalProperties": { + "$ref": "#/definitions/EvaluatorStatusResult" + } + }, + "metrics": { + "type": "array", + "description": "Aggregated metrics that summarize evaluation results across multiple data rows. This provides a high-level overview of performance across all data rows.", + "items": { + "$ref": "#/definitions/AggregatedMetric" + } } }, "required": [ - "justification" + "evaluatorStatus" ] }, "EvaluationSummaryStatistics": { @@ -2990,35 +3049,25 @@ "format": "float", "description": "Proportion of evaluation results that failed to meet the specified criteria or threshold, expressed as a value between 0 and 1." }, - "minScore": { + "passRate": { "type": "number", "format": "float", - "description": "Lowest score value observed across all evaluation results." + "description": "Proportion of evaluation results that passed the specified criteria or threshold, expressed as a value between 0 and 1." }, - "maxScore": { + "min": { "type": "number", "format": "float", - "description": "Highest score value observed across all evaluation results." - }, - "averageScore": { - "type": "number", - "format": "float", - "description": "Arithmetic mean of all score values in the evaluation results." - }, - "standardDeviation": { - "type": "number", - "format": "float", - "description": "Standard deviation of score values, measuring the variability or spread of the evaluation results." + "description": "Lowest score value observed across all evaluation results." }, - "confidenceIntervalLower": { + "max": { "type": "number", "format": "float", - "description": "Lower bound of the 90% confidence interval for the evaluation scores, indicating statistical reliability of the results." + "description": "Highest score value observed across all evaluation results." }, - "confidenceIntervalUpper": { + "average": { "type": "number", "format": "float", - "description": "Upper bound of the 90% confidence interval for the evaluation scores, indicating statistical reliability of the results." + "description": "Arithmetic mean of all score values in the evaluation results." } }, "required": [ @@ -3145,146 +3194,22 @@ ] } }, - "EvaluatorResult": { + "EvaluatorStatusResult": { "type": "object", - "description": "Detailed result of a specific evaluator's assessment on a single data row. Contains the evaluation outcome, score, reasoning, and additional metrics.", + "description": "Status of evaluator across all data rows.", "properties": { - "evaluatorId": { - "type": "string", - "description": "Unique identifier of the evaluator that produced this result" - }, - "systemData": { - "$ref": "#/definitions/EvaluatorResultSystemData", - "description": "Metadata about the evaluation execution environment and resource usage." - }, "state": { "$ref": "#/definitions/Azure.Core.Foundations.OperationState", - "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." + "description": "State of the evaluation for this evaluator." }, - "score": { - "type": "object", - "description": "Score for the evaluation result", - "additionalProperties": { - "$ref": "#/definitions/EvaluationScore" - } - } - }, - "required": [ - "evaluatorId", - "state" - ] - }, - "EvaluatorResultCommon": { - "type": "object", - "description": "Common properties shared across all evaluator results, including metadata about the evaluation execution and resource consumption.", - "properties": { - "evaluatorId": { - "type": "string", - "description": "Unique identifier of the evaluator that produced this result" - }, - "systemData": { - "$ref": "#/definitions/EvaluatorResultSystemData", - "description": "Metadata about the evaluation execution environment and resource usage." - }, - "state": { - "$ref": "#/definitions/Azure.Core.Foundations.OperationState", - "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." - } - }, - "required": [ - "evaluatorId", - "state" - ] - }, - "EvaluatorResultSystemData": { - "type": "object", - "description": "Represents system Metadata about the evaluation execution environment and resource usage.", - "properties": { - "inputTokenCount": { - "type": "integer", - "format": "int64", - "description": "Total number of input tokens consumed during the evaluation process. This helps track resource usage and costs." - }, - "outputTokenCount": { - "type": "integer", - "format": "int64", - "description": "Total number of output tokens generated during the evaluation process. This helps track resource usage and costs." - }, - "duration": { - "type": "integer", - "format": "int64", - "description": "Total time taken to complete the evaluation, measured in milliseconds. This includes processing time and any network latency." - }, - "additionalDetails": { - "type": "object", - "description": "Additional details about the evaluator result", - "additionalProperties": {} - } - }, - "required": [ - "inputTokenCount", - "outputTokenCount", - "duration", - "additionalDetails" - ] - }, - "EvaluatorScoreType": { - "type": "string", - "description": "Type of score assigned by the evaluator. This indicates whether the score is boolean, continuous, or ordinal.", - "enum": [ - "boolean", - "continuous", - "ordinal" - ], - "x-ms-enum": { - "name": "EvaluatorScoreType", - "modelAsString": true, - "values": [ - { - "name": "boolean", - "value": "boolean", - "description": "Boolean score type, where the score is either true or false" - }, - { - "name": "continuous", - "value": "continuous", - "description": "Continuous score type, where the score is a floating-point number representing a continuous value" - }, - { - "name": "ordinal", - "value": "ordinal", - "description": "Ordinal score type, where the score represents an ordered category or rank" - } - ] - } - }, - "EvaluatorSummaryResult": { - "type": "object", - "description": "Aggregated summary of evaluation results for a specific evaluator across all data rows. This provides a high-level overview of the evaluator's performance.", - "properties": { - "evaluatorId": { - "type": "string", - "description": "Unique identifier of the evaluator that produced this result" - }, - "systemData": { - "$ref": "#/definitions/EvaluatorResultSystemData", - "description": "Metadata about the evaluation execution environment and resource usage." - }, - "state": { - "$ref": "#/definitions/Azure.Core.Foundations.OperationState", - "description": "Current state of the evaluation execution. Tracks progress from initiation to completion or failure." - }, - "statistics": { - "type": "object", - "description": "Statistical summary of evaluation results where key is the metric id.", - "additionalProperties": { - "$ref": "#/definitions/EvaluationSummaryStatistics" - } + "usage": { + "$ref": "#/definitions/Usage", + "description": "Usage statistics for the evaluation" } }, "required": [ - "evaluatorId", - "state" + "state", + "usage" ] }, "FieldMapping": { @@ -3651,27 +3576,27 @@ "type": "string", "description": "Available formats for structuring inline evaluation data.", "enum": [ - "queryResponseMessage", - "chatMessages", - "inlineJson" + "queryResponseMessageFormat", + "chatMessagesFormat", + "inlineJsonFormat" ], "x-ms-enum": { "name": "InlineDataFormat", "modelAsString": true, "values": [ { - "name": "queryResponseMessage", - "value": "queryResponseMessage", + "name": "queryResponseMessageFormat", + "value": "queryResponseMessageFormat", "description": "Query-response pairs for Q&A and chatbot evaluations" }, { - "name": "chatMessages", - "value": "chatMessages", + "name": "chatMessagesFormat", + "value": "chatMessagesFormat", "description": "Multi-turn conversations with role-based messages and tool definitions" }, { - "name": "inlineJson", - "value": "inlineJson", + "name": "inlineJsonFormat", + "value": "inlineJsonFormat", "description": "Flexible JSON format for custom data structures" } ] @@ -3754,7 +3679,7 @@ "$ref": "#/definitions/InlineData" } ], - "x-ms-discriminator-value": "inlineJson" + "x-ms-discriminator-value": "inlineJsonFormat" }, "InlineJsonUpdate": { "type": "object", @@ -3773,31 +3698,7 @@ "$ref": "#/definitions/InlineDataUpdate" } ], - "x-ms-discriminator-value": "inlineJson" - }, - "InlineMessagesType": { - "type": "string", - "description": "The type of inline messages used for evaluation. This union allows for different inline message formats to be specified.", - "enum": [ - "queryResponseMessage", - "roleContentType" - ], - "x-ms-enum": { - "name": "InlineMessagesType", - "modelAsString": true, - "values": [ - { - "name": "queryResponseMessage", - "value": "queryResponseMessage", - "description": "Inline messages that consist of query-response pairs for evaluation. Each pair includes a query string and the corresponding response generated by the model." - }, - { - "name": "roleContentType", - "value": "roleContentType", - "description": "role content type message" - } - ] - } + "x-ms-discriminator-value": "inlineJsonFormat" }, "InputData": { "type": "object", @@ -4240,7 +4141,7 @@ "$ref": "#/definitions/InlineData" } ], - "x-ms-discriminator-value": "queryResponseMessage" + "x-ms-discriminator-value": "queryResponseMessageFormat" }, "QueryResponseInlineMessagesUpdate": { "type": "object", @@ -4259,7 +4160,7 @@ "$ref": "#/definitions/InlineDataUpdate" } ], - "x-ms-discriminator-value": "queryResponseMessage" + "x-ms-discriminator-value": "queryResponseMessageFormat" }, "QueryResponseMessage": { "type": "object", @@ -4357,30 +4258,6 @@ "target" ] }, - "RedactionLevel": { - "type": "string", - "description": "Controls the level of data redaction applied to evaluation results to protect sensitive information and ensure compliance with privacy requirements.", - "enum": [ - "sensitive", - "none" - ], - "x-ms-enum": { - "name": "RedactionLevel", - "modelAsString": true, - "values": [ - { - "name": "Sensitive", - "value": "sensitive", - "description": "Apply redaction to sensitive data in evaluation results. This is the recommended setting for production environments to ensure data privacy." - }, - { - "name": "None", - "value": "none", - "description": "Do not apply any data redaction to evaluation results. Use this setting only in development or testing environments where data privacy is not a concern." - } - ] - } - }, "RiskCategory": { "type": "string", "description": "Risk category for the attack objective.", @@ -4607,6 +4484,26 @@ ], "x-ms-discriminator-value": "tool_result" }, + "Usage": { + "type": "object", + "description": "Usage statistics for the evaluation", + "properties": { + "inputTokens": { + "type": "integer", + "format": "int64", + "description": "Total number of input tokens consumed during the evaluation process" + }, + "outputTokens": { + "type": "integer", + "format": "int64", + "description": "Total number of output tokens generated during the evaluation process" + } + }, + "required": [ + "inputTokens", + "outputTokens" + ] + }, "UserMessage": { "type": "object", "description": "A message authored by the end user as input to the model.", From a3e5d0563688feb4321c39a6cf8a4bb455ae444c Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 28 Jul 2025 14:53:06 -0700 Subject: [PATCH 11/50] Fix example --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 2 +- .../preview/2025-07-31-preview/azure-ai-projects.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 4b251f7503ef..285290eb1e3b 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -223,7 +223,7 @@ model FoundryModelDatasetSource extends EvaluationDataSource { @doc("Unique identifier of the dataset containing prompts that will be processed by the model deployment") datasetId: DatasetId; - @doc("Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '{{item.messages}}'") + @doc("Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '\${data.query}'") queryField: string; } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 76ed944d234f..3e6864298250 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3348,7 +3348,7 @@ }, "queryField": { "type": "string", - "description": "Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '{{item.messages}}'" + "description": "Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '${data.query}'" } }, "required": [ @@ -3385,7 +3385,7 @@ }, "queryField": { "type": "string", - "description": "Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '{{item.messages}}'" + "description": "Name of the column in the dataset that contains the queries to be processed by the model deployment. This allows specifying which column should be used as input for evaluation. eg.. queryField: '${data.query}'" } }, "allOf": [ From 20d66875f5650fbe0da9f5853b4f71403619ceb1 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Tue, 29 Jul 2025 09:54:30 -0700 Subject: [PATCH 12/50] Adding standard deviation --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 3 +++ .../preview/2025-07-31-preview/azure-ai-projects.json | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 285290eb1e3b..9170eca03c7b 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -247,6 +247,9 @@ model EvaluationSummaryStatistics { @doc("Arithmetic mean of all score values in the evaluation results.") average?: float32; + + @doc("Standard deviation of score values, indicating the variability or spread of scores around the average.") + standardDeviation?: float32; } @doc("Status of evaluator across all data rows.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 3e6864298250..bdaba9cb558b 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3068,6 +3068,11 @@ "type": "number", "format": "float", "description": "Arithmetic mean of all score values in the evaluation results." + }, + "standardDeviation": { + "type": "number", + "format": "float", + "description": "Standard deviation of score values, indicating the variability or spread of scores around the average." } }, "required": [ From e21c2677f10345019e3485f466b99276e368ec4c Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Tue, 29 Jul 2025 10:09:25 -0700 Subject: [PATCH 13/50] confidence interval --- .../ai/Azure.AI.Projects/evaluations/models.tsp | 6 ++++++ .../preview/2025-07-31-preview/azure-ai-projects.json | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 9170eca03c7b..954e1ae780c0 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -250,6 +250,12 @@ model EvaluationSummaryStatistics { @doc("Standard deviation of score values, indicating the variability or spread of scores around the average.") standardDeviation?: float32; + + @doc("Lower bound of the confidence interval for the average score, providing a range within which the true average is likely to fall (90th percentile).") + confidenceIntervalLowerBound?: float32; + + @doc("Upper bound of the confidence interval for the average score, providing a range within which the true average is likely to fall (90th percentile).") + confidenceIntervalUpperBound?: float32; } @doc("Status of evaluator across all data rows.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index bdaba9cb558b..697603d3c41b 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3073,6 +3073,16 @@ "type": "number", "format": "float", "description": "Standard deviation of score values, indicating the variability or spread of scores around the average." + }, + "confidenceIntervalLowerBound": { + "type": "number", + "format": "float", + "description": "Lower bound of the confidence interval for the average score, providing a range within which the true average is likely to fall (90th percentile)." + }, + "confidenceIntervalUpperBound": { + "type": "number", + "format": "float", + "description": "Upper bound of the confidence interval for the average score, providing a range within which the true average is likely to fall (90th percentile)." } }, "required": [ From dac0d1217181284651d2dd817824c38207421887 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 30 Jul 2025 17:29:42 -0700 Subject: [PATCH 14/50] Addressing feedback from API review --- .../ai/Azure.AI.Projects/common/models.tsp | 22 + .../Azure.AI.Projects/evaluations/models.tsp | 99 +++-- .../Azure.AI.Projects/evaluations/routes.tsp | 2 +- .../ai/Azure.AI.Projects/red-teams/models.tsp | 18 - .../2025-05-15-preview/azure-ai-projects.json | 5 + .../2025-07-31-preview/azure-ai-projects.json | 404 ++++++++++-------- 6 files changed, 330 insertions(+), 220 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/common/models.tsp b/specification/ai/Azure.AI.Projects/common/models.tsp index 4d615ca7fdf6..70bb8a68a985 100644 --- a/specification/ai/Azure.AI.Projects/common/models.tsp +++ b/specification/ai/Azure.AI.Projects/common/models.tsp @@ -4,6 +4,7 @@ import "@azure-tools/typespec-azure-core"; import "@azure-tools/typespec-azure-resource-manager"; namespace Azure.AI.Projects; +using TypeSpec.Versioning; alias AssetBase = { @doc("Asset ID, a unique identifier for the asset") @@ -115,3 +116,24 @@ model AssetCredentialResponse { @doc("Credential info to access the storage account.") blobReference: BlobReference; } + +@doc("Azure OpenAI model configuration. The API version would be selected by the service for querying the model.") +@added(Versions.v2025_05_15_preview) +model AzureOpenAIModelConfiguration extends TargetConfig { + @visibility(Lifecycle.Read) + type: "AzureOpenAIModel"; + + @doc("Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`.") + modelDeploymentName: string; + + @doc("Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.") + modelParameters?: Record; +} + +@doc("Abstract class for target configuration.") +@added(Versions.v2025_05_15_preview) +@discriminator("type") +model TargetConfig { + @doc("Type of the model configuration.") + type: string; +} diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 954e1ae780c0..1c0a4958d62d 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -20,14 +20,18 @@ model EvaluatorConfiguration { @doc("Identifier of the evaluator.") id: string; + @doc("Name of the evaluator. It can be used for identify the evaluator in the evaluation results.") + @added(Versions.v2025_07_31_preview) + name?: string; + @doc("Initialization parameters of the evaluator.") @removed(Versions.v2025_07_31_preview) initParams?: Record; - @doc("Initial parameters of the evaluator.") + @doc("Initialization parameters of the evaluator.") @added(Versions.v2025_07_31_preview) @renamedFrom(Versions.v2025_07_31_preview, "initParams") - initialParameters?: Record; + initializationParameters?: Record; @doc("Data parameters of the evaluator.") dataMapping?: Record; @@ -105,7 +109,7 @@ union InlineDataFormat { queryResponseMessageFormat: "queryResponseMessageFormat", @doc("Multi-turn conversations with role-based messages and tool definitions") - chatMessagesFormat: "chatMessagesFormat", + evaluationMessagesFormat: "evaluationMessagesFormat", @doc("Flexible JSON format for custom data structures") inlineJsonFormat: "inlineJsonFormat", @@ -133,9 +137,9 @@ model QueryResponseInlineMessages extends InlineData { @added(Versions.v2025_07_31_preview) @doc("Multi-turn conversations for evaluating dialogue systems and context awareness.") -model ChatMessages extends InlineData { - @doc("Specifies chat messages format") - dataFormat: InlineDataFormat.chatMessagesFormat; +model EvaluationMessages extends InlineData { + @doc("Specifies evaluation messages format") + dataFormat: InlineDataFormat.evaluationMessagesFormat; @doc("Array of messages representing representing queries") query: Message[]; @@ -180,21 +184,11 @@ model FoundryDatasetDataSource extends EvaluationDataSource { datasetId: DatasetId; } -@doc("Comprehensive configuration for a model deployment used in evaluation scenarios. This defines how the model will process inputs and generate responses for evaluation.") -@added(Versions.v2025_07_31_preview) -model EvaluationModelConfiguration { - @doc("The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'.") - modelDeploymentName: string; - - @doc("Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.") - modelParameters?: Record; -} - @doc("Common properties shared across model deployment configurations used in evaluations.") @added(Versions.v2025_07_31_preview) model EvaluationModelSourceCommon { @doc("Configuration for the model deployment used in evaluation.") - modelConfiguration: EvaluationModelConfiguration; + modelTarget: TargetConfig; @doc("A list of messages comprising the conversation so far. Each message can be a json string with role and content to specify the conversation context.") baseMessages: Message[]; @@ -227,15 +221,22 @@ model FoundryModelDatasetSource extends EvaluationDataSource { queryField: string; } +@added(Versions.v2025_07_31_preview) +@doc("Confidence interval for the metric.") +model ConfidenceInterval { + @doc("Lower bound of the confidence interval for the average score, providing a range within which the true average is likely to fall.") + lowerBound: float32; + + @doc("Upper bound of the confidence interval for the average score, providing a range within which the true average is likely to fall.") + upperBound: float32; +} + @added(Versions.v2025_07_31_preview) @doc("Statistical summary of evaluation results including aggregated scores and pass/fail rates.") model EvaluationSummaryStatistics { @doc("Total number of rows that were evaluated.") sampleCount: int64; - @doc("Proportion of evaluation results that failed to meet the specified criteria or threshold, expressed as a value between 0 and 1.") - defectRate?: float32; - @doc("Proportion of evaluation results that passed the specified criteria or threshold, expressed as a value between 0 and 1.") passRate?: float32; @@ -251,16 +252,22 @@ model EvaluationSummaryStatistics { @doc("Standard deviation of score values, indicating the variability or spread of scores around the average.") standardDeviation?: float32; - @doc("Lower bound of the confidence interval for the average score, providing a range within which the true average is likely to fall (90th percentile).") - confidenceIntervalLowerBound?: float32; + @doc("Confidence interval for the average score, providing a range within which the pass average is likely to fall (95th percentile).") + confidenceInterval95th?: ConfidenceInterval; - @doc("Upper bound of the confidence interval for the average score, providing a range within which the true average is likely to fall (90th percentile).") - confidenceIntervalUpperBound?: float32; + @doc("Label frequency distribution, showing how often each label appears in the evaluation results. This can help identify common themes or issues across evaluated data rows.") + labelDistribution?: Record; } @doc("Status of evaluator across all data rows.") @added(Versions.v2025_07_31_preview) model EvaluatorStatusResult { + @doc("Name of the evaluator from the input request.") + name: string; + + @doc("Identifier of the evaluator from the input request.") + id: string; + @doc("State of the evaluation for this evaluator.") state: OperationState; @@ -329,7 +336,7 @@ model OneEvaluation { dataSource: InlineDataSource; @doc("Evaluators to be used for the evaluation.") - evaluators: Record; + evaluators: EvaluatorConfiguration[]; @doc("Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.") properties?: Record; @@ -341,12 +348,15 @@ model AggregatedMetric { @doc("Name of the metric") name: string; - @doc("Alias provided for the evaluator in the input request.") - evaluatorAlias: string; + @doc("Name provided for the evaluator in the input request.") + evaluatorName: string; @doc("Statistical summary of evaluation results including aggregated scores and pass/fail rates.") statistics: EvaluationSummaryStatistics; + @doc("Metadata about the metric") + metadata: EvaluationResultMetadata; + @doc("Additional metadata about the aggregated metric, such as processing details or custom fields") additionalDetails?: Record; } @@ -355,7 +365,7 @@ model AggregatedMetric { @doc("Summary of evaluation results, including aggregated metrics and evaluator statuses.") model EvaluationSummaryResult { @doc("Status of each evaluator from the input request.") - evaluatorStatus: Record; + evaluatorStatus: EvaluatorStatusResult[]; @doc("Aggregated metrics that summarize evaluation results across multiple data rows. This provides a high-level overview of performance across all data rows.") metrics?: AggregatedMetric[]; @@ -406,7 +416,8 @@ model Evaluation { properties?: Record; @doc("Evaluators to be used for the evaluation.") - evaluators: Record; + @typeChangedFrom(Versions.v2025_07_31_preview, Record) + evaluators: EvaluatorConfiguration[]; @doc("Specifies the type and configuration of the entity used for this evaluation.") @removed(Versions.v2025_07_31_preview) @@ -614,6 +625,24 @@ model EvaluationResultMetadata { @doc("Threshold value that this score is compared against to determine the evaluation outcome") threshold?: float32; + + @doc("Type of metric generated") + type: MetricType; +} + +@added(Versions.v2025_07_31_preview) +@doc("Enumeration of the different types of metrics that can be used in evaluations") +union MetricType { + @doc("Continuous metric type, representing a numerical score") + continuous: "continuous", + + @doc("Categorical metric type, representing discrete categories or labels") + categorical: "categorical", + + @doc("Ordinal metric type, representing ordered categories") + ordinal: "ordinal", + + string, } @added(Versions.v2025_07_31_preview) @@ -622,11 +651,14 @@ model EvaluationMetric { @doc("Name of the evaluation metric") name: string; - @doc("Alias provided for the evaluator in the input request.") - evaluatorAlias: string; + @doc("Name provided for the evaluator in the input request.") + evaluatorName: string; @doc("Score assigned by the evaluator to the specific data row") - score: float32; + score?: float32; + + @doc("Optional labels associated with the evaluation result.") + labels?: string[]; @doc("Outcome of the evaluation") outcome: EvaluationResultOutcome; @@ -634,6 +666,9 @@ model EvaluationMetric { @doc("Reasoning or explanation provided by the evaluator for the score assigned to this data row") reasoning?: string; + @doc("Error message describing why the evaluation failed for this data row, if applicable") + error?: string; + @doc("Metadata about the evaluation result") metadata: EvaluationResultMetadata; diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index bb87451a89ce..c007c6ccaece 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -76,7 +76,7 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates a new evaluation with the specified configuration.") @added(Versions.v2025_07_31_preview) - @route(":runs") + @route(":run") @post createEvaluation is Azure.Core.Foundations.Operation< { diff --git a/specification/ai/Azure.AI.Projects/red-teams/models.tsp b/specification/ai/Azure.AI.Projects/red-teams/models.tsp index f43edf233dbc..e837cfb16e04 100644 --- a/specification/ai/Azure.AI.Projects/red-teams/models.tsp +++ b/specification/ai/Azure.AI.Projects/red-teams/models.tsp @@ -107,24 +107,6 @@ union RiskCategory { SelfHarm: "SelfHarm", } -@doc("Azure OpenAI model configuration. The API version would be selected by the service for querying the model.") -@added(Versions.v2025_05_15_preview) -model AzureOpenAIModelConfiguration extends TargetConfig { - @visibility(Lifecycle.Read) - type: "AzureOpenAIModel"; - - @doc("Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`.") - modelDeploymentName: string; -} - -@doc("Abstract class for target configuration.") -@added(Versions.v2025_05_15_preview) -@discriminator("type") -model TargetConfig { - @doc("Type of the model configuration.") - type: string; -} - @doc("Red team details.") @resource("runs") @added(Versions.v2025_05_15_preview) diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index 6563d5e464d7..dc1f87e0d805 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1965,6 +1965,11 @@ "modelDeploymentName": { "type": "string", "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`." + }, + "modelParameters": { + "type": "object", + "description": "Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.", + "additionalProperties": {} } }, "required": [ diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 697603d3c41b..f4cf9b005ee2 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1075,7 +1075,7 @@ } } }, - "/evaluations:runs": { + "/evaluations:run": { "post": { "operationId": "Evaluations_CreateEvaluation", "description": "Creates a new evaluation with the specified configuration.", @@ -1700,14 +1700,18 @@ "type": "string", "description": "Name of the metric" }, - "evaluatorAlias": { + "evaluatorName": { "type": "string", - "description": "Alias provided for the evaluator in the input request." + "description": "Name provided for the evaluator in the input request." }, "statistics": { "$ref": "#/definitions/EvaluationSummaryStatistics", "description": "Statistical summary of evaluation results including aggregated scores and pass/fail rates." }, + "metadata": { + "$ref": "#/definitions/EvaluationResultMetadata", + "description": "Metadata about the metric" + }, "additionalDetails": { "type": "object", "description": "Additional metadata about the aggregated metric, such as processing details or custom fields", @@ -1716,8 +1720,9 @@ }, "required": [ "name", - "evaluatorAlias", - "statistics" + "evaluatorName", + "statistics", + "metadata" ] }, "ApiKeyCredentials": { @@ -2073,6 +2078,11 @@ "modelDeploymentName": { "type": "string", "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`." + }, + "modelParameters": { + "type": "object", + "description": "Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.", + "additionalProperties": {} } }, "required": [ @@ -2085,6 +2095,27 @@ ], "x-ms-discriminator-value": "AzureOpenAIModel" }, + "AzureOpenAIModelConfigurationUpdate": { + "type": "object", + "description": "Azure OpenAI model configuration. The API version would be selected by the service for querying the model.", + "properties": { + "modelDeploymentName": { + "type": "string", + "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`." + }, + "modelParameters": { + "type": "object", + "description": "Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.", + "additionalProperties": {} + } + }, + "allOf": [ + { + "$ref": "#/definitions/TargetConfigUpdate" + } + ], + "x-ms-discriminator-value": "AzureOpenAIModel" + }, "BaseCredentials": { "type": "object", "description": "A base class for connection credentials", @@ -2123,76 +2154,25 @@ "credential" ] }, - "ChatMessages": { + "ConfidenceInterval": { "type": "object", - "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", + "description": "Confidence interval for the metric.", "properties": { - "query": { - "type": "array", - "description": "Array of messages representing representing queries", - "items": { - "$ref": "#/definitions/Message" - } - }, - "response": { - "type": "array", - "description": "Array of messages representing responses", - "items": { - "$ref": "#/definitions/Message" - } + "lowerBound": { + "type": "number", + "format": "float", + "description": "Lower bound of the confidence interval for the average score, providing a range within which the true average is likely to fall." }, - "toolDefinitions": { - "type": "array", - "description": "Array of tool definitions that are used in the conversation", - "items": { - "$ref": "#/definitions/AgentToolDefinition" - } + "upperBound": { + "type": "number", + "format": "float", + "description": "Upper bound of the confidence interval for the average score, providing a range within which the true average is likely to fall." } }, "required": [ - "query", - "response", - "toolDefinitions" - ], - "allOf": [ - { - "$ref": "#/definitions/InlineData" - } - ], - "x-ms-discriminator-value": "chatMessagesFormat" - }, - "ChatMessagesUpdate": { - "type": "object", - "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", - "properties": { - "query": { - "type": "array", - "description": "Array of messages representing representing queries", - "items": { - "$ref": "#/definitions/Message" - } - }, - "response": { - "type": "array", - "description": "Array of messages representing responses", - "items": { - "$ref": "#/definitions/Message" - } - }, - "toolDefinitions": { - "type": "array", - "description": "Array of tool definitions that are used in the conversation", - "items": { - "$ref": "#/definitions/AgentToolDefinition" - } - } - }, - "allOf": [ - { - "$ref": "#/definitions/InlineDataUpdate" - } - ], - "x-ms-discriminator-value": "chatMessagesFormat" + "lowerBound", + "upperBound" + ] }, "Connection": { "type": "object", @@ -2661,9 +2641,9 @@ } }, "evaluators": { - "type": "object", + "type": "array", "description": "Evaluators to be used for the evaluation.", - "additionalProperties": { + "items": { "$ref": "#/definitions/EvaluatorConfiguration" } }, @@ -2805,6 +2785,77 @@ ] } }, + "EvaluationMessages": { + "type": "object", + "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", + "properties": { + "query": { + "type": "array", + "description": "Array of messages representing representing queries", + "items": { + "$ref": "#/definitions/Message" + } + }, + "response": { + "type": "array", + "description": "Array of messages representing responses", + "items": { + "$ref": "#/definitions/Message" + } + }, + "toolDefinitions": { + "type": "array", + "description": "Array of tool definitions that are used in the conversation", + "items": { + "$ref": "#/definitions/AgentToolDefinition" + } + } + }, + "required": [ + "query", + "response", + "toolDefinitions" + ], + "allOf": [ + { + "$ref": "#/definitions/InlineData" + } + ], + "x-ms-discriminator-value": "evaluationMessagesFormat" + }, + "EvaluationMessagesUpdate": { + "type": "object", + "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", + "properties": { + "query": { + "type": "array", + "description": "Array of messages representing representing queries", + "items": { + "$ref": "#/definitions/Message" + } + }, + "response": { + "type": "array", + "description": "Array of messages representing responses", + "items": { + "$ref": "#/definitions/Message" + } + }, + "toolDefinitions": { + "type": "array", + "description": "Array of tool definitions that are used in the conversation", + "items": { + "$ref": "#/definitions/AgentToolDefinition" + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/InlineDataUpdate" + } + ], + "x-ms-discriminator-value": "evaluationMessagesFormat" + }, "EvaluationMetric": { "type": "object", "description": "Metric representing the evaluation result for a specific evaluator.", @@ -2813,15 +2864,22 @@ "type": "string", "description": "Name of the evaluation metric" }, - "evaluatorAlias": { + "evaluatorName": { "type": "string", - "description": "Alias provided for the evaluator in the input request." + "description": "Name provided for the evaluator in the input request." }, "score": { "type": "number", "format": "float", "description": "Score assigned by the evaluator to the specific data row" }, + "labels": { + "type": "array", + "description": "Optional labels associated with the evaluation result.", + "items": { + "type": "string" + } + }, "outcome": { "$ref": "#/definitions/EvaluationResultOutcome", "description": "Outcome of the evaluation" @@ -2830,6 +2888,10 @@ "type": "string", "description": "Reasoning or explanation provided by the evaluator for the score assigned to this data row" }, + "error": { + "type": "string", + "description": "Error message describing why the evaluation failed for this data row, if applicable" + }, "metadata": { "$ref": "#/definitions/EvaluationResultMetadata", "description": "Metadata about the evaluation result" @@ -2842,51 +2904,17 @@ }, "required": [ "name", - "evaluatorAlias", - "score", + "evaluatorName", "outcome", "metadata" ] }, - "EvaluationModelConfiguration": { - "type": "object", - "description": "Comprehensive configuration for a model deployment used in evaluation scenarios. This defines how the model will process inputs and generate responses for evaluation.", - "properties": { - "modelDeploymentName": { - "type": "string", - "description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'." - }, - "modelParameters": { - "type": "object", - "description": "Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.", - "additionalProperties": {} - } - }, - "required": [ - "modelDeploymentName" - ] - }, - "EvaluationModelConfigurationUpdate": { - "type": "object", - "description": "Comprehensive configuration for a model deployment used in evaluation scenarios. This defines how the model will process inputs and generate responses for evaluation.", - "properties": { - "modelDeploymentName": { - "type": "string", - "description": "The model deployment to be evaluated. Accepts either the deployment name alone or with the connection name as '{connectionName}/modelDeploymentName'." - }, - "modelParameters": { - "type": "object", - "description": "Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.", - "additionalProperties": {} - } - } - }, "EvaluationModelSourceCommon": { "type": "object", "description": "Common properties shared across model deployment configurations used in evaluations.", "properties": { - "modelConfiguration": { - "$ref": "#/definitions/EvaluationModelConfiguration", + "modelTarget": { + "$ref": "#/definitions/TargetConfig", "description": "Configuration for the model deployment used in evaluation." }, "baseMessages": { @@ -2898,7 +2926,7 @@ } }, "required": [ - "modelConfiguration", + "modelTarget", "baseMessages" ] }, @@ -2956,11 +2984,16 @@ "type": "number", "format": "float", "description": "Threshold value that this score is compared against to determine the evaluation outcome" + }, + "type": { + "$ref": "#/definitions/MetricType", + "description": "Type of metric generated" } }, "required": [ "evaluatorId", - "desiredDirection" + "desiredDirection", + "type" ] }, "EvaluationResultOutcome": { @@ -3017,9 +3050,9 @@ "description": "Summary of evaluation results, including aggregated metrics and evaluator statuses.", "properties": { "evaluatorStatus": { - "type": "object", + "type": "array", "description": "Status of each evaluator from the input request.", - "additionalProperties": { + "items": { "$ref": "#/definitions/EvaluatorStatusResult" } }, @@ -3044,11 +3077,6 @@ "format": "int64", "description": "Total number of rows that were evaluated." }, - "defectRate": { - "type": "number", - "format": "float", - "description": "Proportion of evaluation results that failed to meet the specified criteria or threshold, expressed as a value between 0 and 1." - }, "passRate": { "type": "number", "format": "float", @@ -3074,15 +3102,17 @@ "format": "float", "description": "Standard deviation of score values, indicating the variability or spread of scores around the average." }, - "confidenceIntervalLowerBound": { - "type": "number", - "format": "float", - "description": "Lower bound of the confidence interval for the average score, providing a range within which the true average is likely to fall (90th percentile)." + "confidenceInterval95th": { + "$ref": "#/definitions/ConfidenceInterval", + "description": "Confidence interval for the average score, providing a range within which the pass average is likely to fall (95th percentile)." }, - "confidenceIntervalUpperBound": { - "type": "number", - "format": "float", - "description": "Upper bound of the confidence interval for the average score, providing a range within which the true average is likely to fall (90th percentile)." + "labelDistribution": { + "type": "object", + "description": "Label frequency distribution, showing how often each label appears in the evaluation results. This can help identify common themes or issues across evaluated data rows.", + "additionalProperties": { + "format": "int64", + "type": "integer" + } } }, "required": [ @@ -3124,10 +3154,10 @@ } }, "evaluators": { - "type": "object", + "type": "array", "description": "Evaluators to be used for the evaluation.", - "additionalProperties": { - "$ref": "#/definitions/EvaluatorConfigurationUpdate" + "items": { + "$ref": "#/definitions/EvaluatorConfiguration" } } } @@ -3140,9 +3170,13 @@ "type": "string", "description": "Identifier of the evaluator." }, - "initialParameters": { + "name": { + "type": "string", + "description": "Name of the evaluator. It can be used for identify the evaluator in the evaluation results." + }, + "initializationParameters": { "type": "object", - "description": "Initial parameters of the evaluator.", + "description": "Initialization parameters of the evaluator.", "additionalProperties": {} }, "dataMapping": { @@ -3157,28 +3191,6 @@ "id" ] }, - "EvaluatorConfigurationUpdate": { - "type": "object", - "description": "Evaluator Configuration", - "properties": { - "id": { - "type": "string", - "description": "Identifier of the evaluator." - }, - "initialParameters": { - "type": "object", - "description": "Initial parameters of the evaluator.", - "additionalProperties": {} - }, - "dataMapping": { - "type": "object", - "description": "Data parameters of the evaluator.", - "additionalProperties": { - "type": "string" - } - } - } - }, "EvaluatorDesirableDirection": { "type": "string", "description": "Describes the desirable direction for the evaluation score. This indicates whether a higher or lower score is preferred for this evaluator.", @@ -3213,6 +3225,14 @@ "type": "object", "description": "Status of evaluator across all data rows.", "properties": { + "name": { + "type": "string", + "description": "Name of the evaluator from the input request." + }, + "id": { + "type": "string", + "description": "Identifier of the evaluator from the input request." + }, "state": { "$ref": "#/definitions/Azure.Core.Foundations.OperationState", "description": "State of the evaluation for this evaluator." @@ -3223,6 +3243,8 @@ } }, "required": [ + "name", + "id", "state", "usage" ] @@ -3346,8 +3368,8 @@ "type": "object", "description": "Data source that uses a model deployment with a dataset containing prompts. The model processes each prompt from the dataset to generate responses, which are then evaluated against the configured evaluators.", "properties": { - "modelConfiguration": { - "$ref": "#/definitions/EvaluationModelConfiguration", + "modelTarget": { + "$ref": "#/definitions/TargetConfig", "description": "Configuration for the model deployment used in evaluation." }, "baseMessages": { @@ -3367,7 +3389,7 @@ } }, "required": [ - "modelConfiguration", + "modelTarget", "baseMessages", "datasetId", "queryField" @@ -3383,8 +3405,8 @@ "type": "object", "description": "Data source that uses a model deployment with a dataset containing prompts. The model processes each prompt from the dataset to generate responses, which are then evaluated against the configured evaluators.", "properties": { - "modelConfiguration": { - "$ref": "#/definitions/EvaluationModelConfigurationUpdate", + "modelTarget": { + "$ref": "#/definitions/TargetConfigUpdate", "description": "Configuration for the model deployment used in evaluation." }, "baseMessages": { @@ -3414,8 +3436,8 @@ "type": "object", "description": "Data source that uses a model deployment with inline queries. The specified model processes each query to generate responses, which are then evaluated against the configured evaluators.", "properties": { - "modelConfiguration": { - "$ref": "#/definitions/EvaluationModelConfiguration", + "modelTarget": { + "$ref": "#/definitions/TargetConfig", "description": "Configuration for the model deployment used in evaluation." }, "baseMessages": { @@ -3434,7 +3456,7 @@ } }, "required": [ - "modelConfiguration", + "modelTarget", "baseMessages", "queries" ], @@ -3449,8 +3471,8 @@ "type": "object", "description": "Data source that uses a model deployment with inline queries. The specified model processes each query to generate responses, which are then evaluated against the configured evaluators.", "properties": { - "modelConfiguration": { - "$ref": "#/definitions/EvaluationModelConfigurationUpdate", + "modelTarget": { + "$ref": "#/definitions/TargetConfigUpdate", "description": "Configuration for the model deployment used in evaluation." }, "baseMessages": { @@ -3592,7 +3614,7 @@ "description": "Available formats for structuring inline evaluation data.", "enum": [ "queryResponseMessageFormat", - "chatMessagesFormat", + "evaluationMessagesFormat", "inlineJsonFormat" ], "x-ms-enum": { @@ -3605,8 +3627,8 @@ "description": "Query-response pairs for Q&A and chatbot evaluations" }, { - "name": "chatMessagesFormat", - "value": "chatMessagesFormat", + "name": "evaluationMessagesFormat", + "value": "evaluationMessagesFormat", "description": "Multi-turn conversations with role-based messages and tool definitions" }, { @@ -3797,6 +3819,36 @@ "content" ] }, + "MetricType": { + "type": "string", + "description": "Enumeration of the different types of metrics that can be used in evaluations", + "enum": [ + "continuous", + "categorical", + "ordinal" + ], + "x-ms-enum": { + "name": "MetricType", + "modelAsString": true, + "values": [ + { + "name": "continuous", + "value": "continuous", + "description": "Continuous metric type, representing a numerical score" + }, + { + "name": "categorical", + "value": "categorical", + "description": "Categorical metric type, representing discrete categories or labels" + }, + { + "name": "ordinal", + "value": "ordinal", + "description": "Ordinal metric type, representing ordered categories" + } + ] + } + }, "ModelDeployment": { "type": "object", "description": "Model Deployment Definition", @@ -3868,9 +3920,9 @@ "description": "Input data to evaluate. Single row support only." }, "evaluators": { - "type": "object", + "type": "array", "description": "Evaluators to be used for the evaluation.", - "additionalProperties": { + "items": { "$ref": "#/definitions/EvaluatorConfiguration" } }, @@ -4430,6 +4482,20 @@ "type" ] }, + "TargetConfigUpdate": { + "type": "object", + "description": "Abstract class for target configuration.", + "properties": { + "type": { + "type": "string", + "description": "Type of the model configuration." + } + }, + "discriminator": "type", + "required": [ + "type" + ] + }, "TextContent": { "type": "object", "description": "Content for text messages in AI conversations.", From dcbe5fdb9f998a36b2f20d6846429dca24e1718b Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Thu, 31 Jul 2025 12:45:29 -0700 Subject: [PATCH 15/50] nit fixing names --- .../Azure.AI.Projects/evaluations/models.tsp | 2 +- .../Azure.AI.Projects/evaluations/routes.tsp | 14 ++--- .../2025-07-31-preview/azure-ai-projects.json | 51 +++++++------------ 3 files changed, 21 insertions(+), 46 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 1c0a4958d62d..a60fd57d4e80 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -256,7 +256,7 @@ model EvaluationSummaryStatistics { confidenceInterval95th?: ConfidenceInterval; @doc("Label frequency distribution, showing how often each label appears in the evaluation results. This can help identify common themes or issues across evaluated data rows.") - labelDistribution?: Record; + labelFrequency?: Record; } @doc("Status of evaluator across all data rows.") diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index c007c6ccaece..5d637f2c55c7 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -76,9 +76,9 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates a new evaluation with the specified configuration.") @added(Versions.v2025_07_31_preview) - @route(":run") + @route(":batchRun") @post - createEvaluation is Azure.Core.Foundations.Operation< + createBatchEvaluation is Azure.Core.Foundations.Operation< { @header("Repeatability-Request-ID") @doc("Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.") @@ -98,18 +98,10 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Independent API operation to perform a single evaluation and immediately get a result.") @added(Versions.v2025_07_31_preview) - @route(":evaluateOne") + @route(":run") @post createOneEvaluation is Azure.Core.Foundations.Operation< { - @header("Repeatability-Request-ID") - @doc("Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.") - repeatabilityRequestId?: string; - - @doc("Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.") - @header("Repeatability-First-Sent") - repeatabilityFirstSent?: utcDateTime; - @doc("Complete evaluation configuration including data source, evaluators, and result settings") @body oneEvaluation: OneEvaluation; diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index f4cf9b005ee2..90262d714ea8 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -19,7 +19,7 @@ { "name": "endpoint", "in": "path", - "description": "Project endpoint. In the form \"https://.services.ai.azure.com/api/projects/_project\"\nif your Foundry Hub has only one Project, or to use the default Project in your Hub. Or in the form \n\"https://.services.ai.azure.com/api/projects/\" if you want to explicitly\nspecify the Foundry Project name.", + "description": "Project endpoint. In the form \"https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project\"\nif your Foundry Hub has only one Project, or to use the default Project in your Hub. Or in the form \n\"https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name\" if you want to explicitly\nspecify the Foundry Project name.", "required": true, "type": "string", "format": "uri", @@ -346,7 +346,7 @@ "/datasets/{name}/versions/{version}": { "get": { "operationId": "Datasets_GetVersion", - "description": "Get the specific version of the DatasetVersion", + "description": "Get the specific version of the DatasetVersion. The service returns 404 Not Found error if the DatasetVersion does not exist.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" @@ -450,7 +450,7 @@ }, "delete": { "operationId": "Datasets_DeleteVersion", - "description": "Delete the specific version of the DatasetVersion", + "description": "Delete the specific version of the DatasetVersion. The service returns 204 No Content if the DatasetVersion was deleted successfully or if the DatasetVersion does not exist.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" @@ -1018,10 +1018,10 @@ } } }, - "/evaluations:evaluateOne": { + "/evaluations:batchRun": { "post": { - "operationId": "Evaluations_CreateOneEvaluation", - "description": "Independent API operation to perform a single evaluation and immediately get a result.", + "operationId": "Evaluations_CreateBatchEvaluation", + "description": "Creates a new evaluation with the specified configuration.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" @@ -1044,12 +1044,12 @@ "x-ms-client-name": "repeatabilityFirstSent" }, { - "name": "oneEvaluation", + "name": "evaluation", "in": "body", "description": "Complete evaluation configuration including data source, evaluators, and result settings", "required": true, "schema": { - "$ref": "#/definitions/OneEvaluation" + "$ref": "#/definitions/Evaluation" } } ], @@ -1057,7 +1057,7 @@ "201": { "description": "The request has succeeded and a new resource has been created as a result.", "schema": { - "$ref": "#/definitions/EvaluationResult" + "$ref": "#/definitions/Evaluation" } }, "default": { @@ -1077,36 +1077,19 @@ }, "/evaluations:run": { "post": { - "operationId": "Evaluations_CreateEvaluation", - "description": "Creates a new evaluation with the specified configuration.", + "operationId": "Evaluations_CreateOneEvaluation", + "description": "Independent API operation to perform a single evaluation and immediately get a result.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "Repeatability-Request-ID", - "in": "header", - "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", - "required": false, - "type": "string", - "x-ms-client-name": "repeatabilityRequestId" - }, - { - "name": "Repeatability-First-Sent", - "in": "header", - "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", - "required": false, - "type": "string", - "format": "date-time", - "x-ms-client-name": "repeatabilityFirstSent" - }, - { - "name": "evaluation", + "name": "oneEvaluation", "in": "body", "description": "Complete evaluation configuration including data source, evaluators, and result settings", "required": true, "schema": { - "$ref": "#/definitions/Evaluation" + "$ref": "#/definitions/OneEvaluation" } } ], @@ -1114,7 +1097,7 @@ "201": { "description": "The request has succeeded and a new resource has been created as a result.", "schema": { - "$ref": "#/definitions/Evaluation" + "$ref": "#/definitions/EvaluationResult" } }, "default": { @@ -1210,7 +1193,7 @@ "/indexes/{name}/versions/{version}": { "get": { "operationId": "Indexes_GetVersion", - "description": "Get the specific version of the Index", + "description": "Get the specific version of the Index. The service returns 404 Not Found error if the Index does not exist.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" @@ -1314,7 +1297,7 @@ }, "delete": { "operationId": "Indexes_DeleteVersion", - "description": "Delete the specific version of the Index", + "description": "Delete the specific version of the Index. The service returns 204 No Content if the Index was deleted successfully or if the Index does not exist.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" @@ -3106,7 +3089,7 @@ "$ref": "#/definitions/ConfidenceInterval", "description": "Confidence interval for the average score, providing a range within which the pass average is likely to fall (95th percentile)." }, - "labelDistribution": { + "labelFrequency": { "type": "object", "description": "Label frequency distribution, showing how often each label appears in the evaluation results. This can help identify common themes or issues across evaluated data rows.", "additionalProperties": { From ffdc523dbddf0501ebe55f712d5818d4bb7ba37a Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Thu, 31 Jul 2025 15:30:25 -0700 Subject: [PATCH 16/50] Update to runBatch --- .../Azure.AI.Projects/evaluations/routes.tsp | 2 +- .../2025-07-31-preview/azure-ai-projects.json | 58 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 5d637f2c55c7..27624a4b1cd9 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -76,7 +76,7 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates a new evaluation with the specified configuration.") @added(Versions.v2025_07_31_preview) - @route(":batchRun") + @route(":runBatch") @post createBatchEvaluation is Azure.Core.Foundations.Operation< { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 90262d714ea8..420162e4cbde 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1018,38 +1018,21 @@ } } }, - "/evaluations:batchRun": { + "/evaluations:run": { "post": { - "operationId": "Evaluations_CreateBatchEvaluation", - "description": "Creates a new evaluation with the specified configuration.", + "operationId": "Evaluations_CreateOneEvaluation", + "description": "Independent API operation to perform a single evaluation and immediately get a result.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "Repeatability-Request-ID", - "in": "header", - "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", - "required": false, - "type": "string", - "x-ms-client-name": "repeatabilityRequestId" - }, - { - "name": "Repeatability-First-Sent", - "in": "header", - "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", - "required": false, - "type": "string", - "format": "date-time", - "x-ms-client-name": "repeatabilityFirstSent" - }, - { - "name": "evaluation", + "name": "oneEvaluation", "in": "body", "description": "Complete evaluation configuration including data source, evaluators, and result settings", "required": true, "schema": { - "$ref": "#/definitions/Evaluation" + "$ref": "#/definitions/OneEvaluation" } } ], @@ -1057,7 +1040,7 @@ "201": { "description": "The request has succeeded and a new resource has been created as a result.", "schema": { - "$ref": "#/definitions/Evaluation" + "$ref": "#/definitions/EvaluationResult" } }, "default": { @@ -1075,21 +1058,38 @@ } } }, - "/evaluations:run": { + "/evaluations:runBatch": { "post": { - "operationId": "Evaluations_CreateOneEvaluation", - "description": "Independent API operation to perform a single evaluation and immediately get a result.", + "operationId": "Evaluations_CreateBatchEvaluation", + "description": "Creates a new evaluation with the specified configuration.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "oneEvaluation", + "name": "Repeatability-Request-ID", + "in": "header", + "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", + "required": false, + "type": "string", + "x-ms-client-name": "repeatabilityRequestId" + }, + { + "name": "Repeatability-First-Sent", + "in": "header", + "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", + "required": false, + "type": "string", + "format": "date-time", + "x-ms-client-name": "repeatabilityFirstSent" + }, + { + "name": "evaluation", "in": "body", "description": "Complete evaluation configuration including data source, evaluators, and result settings", "required": true, "schema": { - "$ref": "#/definitions/OneEvaluation" + "$ref": "#/definitions/Evaluation" } } ], @@ -1097,7 +1097,7 @@ "201": { "description": "The request has succeeded and a new resource has been created as a result.", "schema": { - "$ref": "#/definitions/EvaluationResult" + "$ref": "#/definitions/Evaluation" } }, "default": { From e643fabbd7ec629a205599c9352d91d9bf2b119f Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Thu, 31 Jul 2025 17:50:00 -0700 Subject: [PATCH 17/50] Readme update --- specification/ai/data-plane/Azure.AI.Projects/readme.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/ai/data-plane/Azure.AI.Projects/readme.md b/specification/ai/data-plane/Azure.AI.Projects/readme.md index 1f090a5ed13f..2b60b4c8a00d 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/readme.md +++ b/specification/ai/data-plane/Azure.AI.Projects/readme.md @@ -36,6 +36,13 @@ input-file: - preview/2025-05-15-preview/azure-ai-projects.json ``` +### Release v2025-07-31-preview +These settings apply only when `--tag=2025-07-31-preview` is specified on the command line. +``` yaml $(tag) == '2025-07-31-preview' +input-file: + - preview/2025-07-31-preview/azure-ai-projects.json +``` + # Suppressions ``` yaml suppressions: From 49877f476c1bc7bb415b7f306dd5a81d9b08d199 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Thu, 31 Jul 2025 20:38:52 -0700 Subject: [PATCH 18/50] Example Added --- .../Evaluations_Cancel_MaximumSet_Gen.json | 12 ++ ..._CreateBatchEvaluation_MaximumSet_Gen.json | 121 ++++++++++++++++++ ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 62 +++++++++ .../Evaluations_Delete_MaximumSet_Gen.json | 12 ++ ...s_GetEvaluationResults_MaximumSet_Gen.json | 47 +++++++ .../Evaluations_Get_MaximumSet_Gen.json | 94 ++++++++++++++ .../Evaluations_List_MaximumSet_Gen.json | 99 ++++++++++++++ .../Evaluations_Update_MaximumSet_Gen.json | 117 +++++++++++++++++ .../2025-07-31-preview/azure-ai-projects.json | 40 ++++++ .../Evaluations_Cancel_MaximumSet_Gen.json | 12 ++ ..._CreateBatchEvaluation_MaximumSet_Gen.json | 121 ++++++++++++++++++ ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 62 +++++++++ .../Evaluations_Delete_MaximumSet_Gen.json | 12 ++ ...s_GetEvaluationResults_MaximumSet_Gen.json | 47 +++++++ .../Evaluations_Get_MaximumSet_Gen.json | 94 ++++++++++++++ .../Evaluations_List_MaximumSet_Gen.json | 99 ++++++++++++++ .../Evaluations_Update_MaximumSet_Gen.json | 117 +++++++++++++++++ 17 files changed, 1168 insertions(+) create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json new file mode 100644 index 000000000000..a58506b8977c --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Evaluations_Cancel", + "operationId": "Evaluations_Cancel", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "haegrdtolkdxedjsvaw", + "x-ms-client-request-id": "gpyrswfdmrmcesrhplohwoypfn" + }, + "responses": { + "204": {} + } +} \ No newline at end of file diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json new file mode 100644 index 000000000000..d77bae51ac97 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json @@ -0,0 +1,121 @@ +{ + "title": "Evaluations_CreateBatchEvaluation", + "operationId": "Evaluations_CreateBatchEvaluation", + "parameters": { + "api-version": "2025-07-31-preview", + "Repeatability-Request-ID": "mvxmtptnhf", + "Repeatability-First-Sent": "2025-08-01T03:11:41.785Z", + "evaluation": { + "displayName": "Customer Satisfaction Evaluation", + "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ] + } + }, + "responses": { + "201": { + "body": { + "id": "eqwigaovprcqmoli", + "displayName": "Customer Satisfaction Evaluation", + "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ], + "resultDatasetId": "dataset://some-dataset-id", + "summary": { + "evaluatorStatus": [ + { + "state": "succeeded", + "name": "relevance", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } + } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "passRate": 0.9, + "sampleCount": 100, + "average": 4.0, + "min": 4.0, + "max": 4.0, + "standardDeviation": 1.0, + "confidenceInterval95th": { + "lowerBound": 3.5, + "upperBound": 4.5 + }, + "labelFrequency": {} + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + }, + "systemData": { + "key8144": "momiykonfordaz" + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json new file mode 100644 index 000000000000..5a440703d526 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -0,0 +1,62 @@ +{ + "title": "Evaluations_CreateOneEvaluation", + "operationId": "Evaluations_CreateOneEvaluation", + "parameters": { + "api-version": "2025-07-31-preview", + "oneEvaluation": { + "dataSource": { + "data": { + "query": "What is the capital of France?", + "response": "The capital of France is Paris.", + "context": "France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille." + }, + "type": "inlineData" + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.8, + "model": "gpt-4o" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}", + "context": "${data.context}" + } + } + ] + } + }, + "responses": { + "201": { + "body": { + "id": "eval_12345", + "state": "Succeeded", + "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", + "usage": { + "inputTokens": 45, + "outputTokens": 12 + }, + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "score": 4.8, + "labels": [ + "highly_relevant" + ], + "outcome": "pass", + "reasoning": "The response directly and accurately answers the question about France's capital. The answer is factually correct and highly relevant to the query.", + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json new file mode 100644 index 000000000000..71861a1e08bd --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Evaluations_Delete", + "operationId": "Evaluations_Delete", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "todlqljalqbtuoxkqszutbyy", + "x-ms-client-request-id": "gpyrswfdmrmcesrhplohwoypfn" + }, + "responses": { + "204": {} + } +} \ No newline at end of file diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json new file mode 100644 index 000000000000..2914f005cb33 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -0,0 +1,47 @@ +{ + "title": "Evaluations_GetEvaluationResults", + "operationId": "Evaluations_GetEvaluationResults", + "parameters": { + "name": "customer-satisfaction-eval-001", + "api-version": "2025-07-31-preview", + "top": 10, + "skip": 0, + "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "id": "result_001", + "state": "Succeeded", + "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", + "usage": { + "inputTokens": 45, + "outputTokens": 12 + }, + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "score": 4.8, + "labels": [ + "highly_relevant" + ], + "outcome": "pass", + "reasoning": "The response directly and accurately answers the question about France's capital. The answer is factually correct and highly relevant to the query.", + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + } + ], + "nextLink": "/evaluations/customer-satisfaction-eval-001/results?api-version=2025-07-31-preview&top=10&skip=10" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json new file mode 100644 index 000000000000..e69f690da989 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json @@ -0,0 +1,94 @@ +{ + "title": "Evaluations_Get", + "operationId": "Evaluations_Get", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "customer-satisfaction-eval-001", + "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc" + }, + "responses": { + "200": { + "body": { + "id": "customer-satisfaction-eval-001", + "displayName": "Customer Satisfaction Evaluation", + "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", + "state": "Succeeded", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ], + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } + } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2.0, + "max": 4.0, + "average": 3.0, + "standardDeviation": 1.0, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": { + "relevant": 1, + "partially_relevant": 1 + } + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + }, + "systemData": { + "createdAt": "2025-07-31T10:00:00Z", + "createdBy": "user@example.com", + "lastModifiedAt": "2025-07-31T10:30:00Z" + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json new file mode 100644 index 000000000000..908a23d4fcf1 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json @@ -0,0 +1,99 @@ +{ + "title": "Evaluations_List", + "operationId": "Evaluations_List", + "parameters": { + "api-version": "2025-07-31-preview", + "filter": "state eq 'Succeeded'", + "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "id": "customer-satisfaction-eval-001", + "displayName": "Customer Satisfaction Evaluation", + "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", + "state": "Succeeded", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ], + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } + } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2.0, + "max": 4.0, + "average": 3.0, + "standardDeviation": 1.0, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": { + "relevant": 1, + "partially_relevant": 1 + } + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + }, + "systemData": { + "createdAt": "2025-07-31T10:00:00Z", + "createdBy": "user@example.com", + "lastModifiedAt": "2025-07-31T10:30:00Z" + } + } + ], + "nextLink": "https://endpoint/evaluations?api-version=2025-07-31-preview&filter=state%20eq%20%27Succeeded%27&$skiptoken=next_page_token" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json new file mode 100644 index 000000000000..d898de7a7054 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json @@ -0,0 +1,117 @@ +{ + "title": "Evaluations_Update", + "operationId": "Evaluations_Update", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "customer-satisfaction-eval-001", + "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc", + "resource": { + "displayName": "Updated Customer Satisfaction Evaluation", + "description": "Updated comprehensive evaluation of customer satisfaction - now includes additional test scenarios", + "tags": { + "environment": "production", + "version": "2.0", + "team": "ai-quality" + }, + "properties": { + "lastUpdatedBy": "user@example.com", + "updateReason": "Added new test scenarios for better coverage" + } + } + }, + "responses": { + "200": { + "body": { + "id": "customer-satisfaction-eval-001", + "displayName": "Updated Customer Satisfaction Evaluation", + "description": "Updated comprehensive evaluation of customer satisfaction - now includes additional test scenarios", + "state": "Succeeded", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ], + "tags": { + "environment": "production", + "version": "2.0", + "team": "ai-quality" + }, + "properties": { + "lastUpdatedBy": "user@example.com", + "updateReason": "Added new test scenarios for better coverage" + }, + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } + } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2.0, + "max": 4.0, + "average": 3.0, + "standardDeviation": 1.0, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": { + "relevant": 1, + "partially_relevant": 1 + } + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + }, + "systemData": { + "createdAt": "2025-07-31T10:00:00Z", + "createdBy": "user@example.com", + "lastModifiedAt": "2025-07-31T12:00:00Z", + "lastModifiedBy": "user@example.com" + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 420162e4cbde..cdd090190a80 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -761,6 +761,11 @@ } } }, + "x-ms-examples": { + "Evaluations_List": { + "$ref": "./examples/Evaluations_List_MaximumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -815,6 +820,11 @@ } } } + }, + "x-ms-examples": { + "Evaluations_Get": { + "$ref": "./examples/Evaluations_Get_MaximumSet_Gen.json" + } } }, "patch": { @@ -873,6 +883,11 @@ } } } + }, + "x-ms-examples": { + "Evaluations_Update": { + "$ref": "./examples/Evaluations_Update_MaximumSet_Gen.json" + } } }, "delete": { @@ -916,6 +931,11 @@ } } } + }, + "x-ms-examples": { + "Evaluations_Delete": { + "$ref": "./examples/Evaluations_Delete_MaximumSet_Gen.json" + } } } }, @@ -961,6 +981,11 @@ } } } + }, + "x-ms-examples": { + "Evaluations_Cancel": { + "$ref": "./examples/Evaluations_Cancel_MaximumSet_Gen.json" + } } } }, @@ -1013,6 +1038,11 @@ } } }, + "x-ms-examples": { + "Evaluations_GetEvaluationResults": { + "$ref": "./examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -1055,6 +1085,11 @@ } } } + }, + "x-ms-examples": { + "Evaluations_CreateOneEvaluation": { + "$ref": "./examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json" + } } } }, @@ -1112,6 +1147,11 @@ } } } + }, + "x-ms-examples": { + "Evaluations_CreateBatchEvaluation": { + "$ref": "./examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json" + } } } }, diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json new file mode 100644 index 000000000000..a58506b8977c --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Evaluations_Cancel", + "operationId": "Evaluations_Cancel", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "haegrdtolkdxedjsvaw", + "x-ms-client-request-id": "gpyrswfdmrmcesrhplohwoypfn" + }, + "responses": { + "204": {} + } +} \ No newline at end of file diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json new file mode 100644 index 000000000000..d77bae51ac97 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json @@ -0,0 +1,121 @@ +{ + "title": "Evaluations_CreateBatchEvaluation", + "operationId": "Evaluations_CreateBatchEvaluation", + "parameters": { + "api-version": "2025-07-31-preview", + "Repeatability-Request-ID": "mvxmtptnhf", + "Repeatability-First-Sent": "2025-08-01T03:11:41.785Z", + "evaluation": { + "displayName": "Customer Satisfaction Evaluation", + "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ] + } + }, + "responses": { + "201": { + "body": { + "id": "eqwigaovprcqmoli", + "displayName": "Customer Satisfaction Evaluation", + "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ], + "resultDatasetId": "dataset://some-dataset-id", + "summary": { + "evaluatorStatus": [ + { + "state": "succeeded", + "name": "relevance", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } + } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "passRate": 0.9, + "sampleCount": 100, + "average": 4.0, + "min": 4.0, + "max": 4.0, + "standardDeviation": 1.0, + "confidenceInterval95th": { + "lowerBound": 3.5, + "upperBound": 4.5 + }, + "labelFrequency": {} + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + }, + "systemData": { + "key8144": "momiykonfordaz" + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json new file mode 100644 index 000000000000..5a440703d526 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -0,0 +1,62 @@ +{ + "title": "Evaluations_CreateOneEvaluation", + "operationId": "Evaluations_CreateOneEvaluation", + "parameters": { + "api-version": "2025-07-31-preview", + "oneEvaluation": { + "dataSource": { + "data": { + "query": "What is the capital of France?", + "response": "The capital of France is Paris.", + "context": "France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille." + }, + "type": "inlineData" + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.8, + "model": "gpt-4o" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}", + "context": "${data.context}" + } + } + ] + } + }, + "responses": { + "201": { + "body": { + "id": "eval_12345", + "state": "Succeeded", + "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", + "usage": { + "inputTokens": 45, + "outputTokens": 12 + }, + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "score": 4.8, + "labels": [ + "highly_relevant" + ], + "outcome": "pass", + "reasoning": "The response directly and accurately answers the question about France's capital. The answer is factually correct and highly relevant to the query.", + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json new file mode 100644 index 000000000000..71861a1e08bd --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Evaluations_Delete", + "operationId": "Evaluations_Delete", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "todlqljalqbtuoxkqszutbyy", + "x-ms-client-request-id": "gpyrswfdmrmcesrhplohwoypfn" + }, + "responses": { + "204": {} + } +} \ No newline at end of file diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json new file mode 100644 index 000000000000..2914f005cb33 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -0,0 +1,47 @@ +{ + "title": "Evaluations_GetEvaluationResults", + "operationId": "Evaluations_GetEvaluationResults", + "parameters": { + "name": "customer-satisfaction-eval-001", + "api-version": "2025-07-31-preview", + "top": 10, + "skip": 0, + "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "id": "result_001", + "state": "Succeeded", + "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", + "usage": { + "inputTokens": 45, + "outputTokens": 12 + }, + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "score": 4.8, + "labels": [ + "highly_relevant" + ], + "outcome": "pass", + "reasoning": "The response directly and accurately answers the question about France's capital. The answer is factually correct and highly relevant to the query.", + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + } + ], + "nextLink": "/evaluations/customer-satisfaction-eval-001/results?api-version=2025-07-31-preview&top=10&skip=10" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json new file mode 100644 index 000000000000..e69f690da989 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json @@ -0,0 +1,94 @@ +{ + "title": "Evaluations_Get", + "operationId": "Evaluations_Get", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "customer-satisfaction-eval-001", + "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc" + }, + "responses": { + "200": { + "body": { + "id": "customer-satisfaction-eval-001", + "displayName": "Customer Satisfaction Evaluation", + "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", + "state": "Succeeded", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ], + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } + } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2.0, + "max": 4.0, + "average": 3.0, + "standardDeviation": 1.0, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": { + "relevant": 1, + "partially_relevant": 1 + } + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + }, + "systemData": { + "createdAt": "2025-07-31T10:00:00Z", + "createdBy": "user@example.com", + "lastModifiedAt": "2025-07-31T10:30:00Z" + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json new file mode 100644 index 000000000000..908a23d4fcf1 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json @@ -0,0 +1,99 @@ +{ + "title": "Evaluations_List", + "operationId": "Evaluations_List", + "parameters": { + "api-version": "2025-07-31-preview", + "filter": "state eq 'Succeeded'", + "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "id": "customer-satisfaction-eval-001", + "displayName": "Customer Satisfaction Evaluation", + "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", + "state": "Succeeded", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ], + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } + } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2.0, + "max": 4.0, + "average": 3.0, + "standardDeviation": 1.0, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": { + "relevant": 1, + "partially_relevant": 1 + } + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + }, + "systemData": { + "createdAt": "2025-07-31T10:00:00Z", + "createdBy": "user@example.com", + "lastModifiedAt": "2025-07-31T10:30:00Z" + } + } + ], + "nextLink": "https://endpoint/evaluations?api-version=2025-07-31-preview&filter=state%20eq%20%27Succeeded%27&$skiptoken=next_page_token" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json new file mode 100644 index 000000000000..d898de7a7054 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json @@ -0,0 +1,117 @@ +{ + "title": "Evaluations_Update", + "operationId": "Evaluations_Update", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "customer-satisfaction-eval-001", + "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc", + "resource": { + "displayName": "Updated Customer Satisfaction Evaluation", + "description": "Updated comprehensive evaluation of customer satisfaction - now includes additional test scenarios", + "tags": { + "environment": "production", + "version": "2.0", + "team": "ai-quality" + }, + "properties": { + "lastUpdatedBy": "user@example.com", + "updateReason": "Added new test scenarios for better coverage" + } + } + }, + "responses": { + "200": { + "body": { + "id": "customer-satisfaction-eval-001", + "displayName": "Updated Customer Satisfaction Evaluation", + "description": "Updated comprehensive evaluation of customer satisfaction - now includes additional test scenarios", + "state": "Succeeded", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "items": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.7, + "model": "gpt-4" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}" + } + } + ], + "tags": { + "environment": "production", + "version": "2.0", + "team": "ai-quality" + }, + "properties": { + "lastUpdatedBy": "user@example.com", + "updateReason": "Added new test scenarios for better coverage" + }, + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } + } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2.0, + "max": 4.0, + "average": 3.0, + "standardDeviation": 1.0, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": { + "relevant": 1, + "partially_relevant": 1 + } + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3.0, + "type": "continuous" + } + } + ] + }, + "systemData": { + "createdAt": "2025-07-31T10:00:00Z", + "createdBy": "user@example.com", + "lastModifiedAt": "2025-07-31T12:00:00Z", + "lastModifiedBy": "user@example.com" + } + } + } + } +} From 0fb4139dd9a8abac5e1f2b1a99ec66ce58db8da3 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Thu, 31 Jul 2025 20:58:24 -0700 Subject: [PATCH 19/50] Fix examples --- .../Evaluations_Cancel_MaximumSet_Gen.json | 4 ++-- ...s_CreateBatchEvaluation_MaximumSet_Gen.json | 8 +++++--- ...ons_CreateOneEvaluation_MaximumSet_Gen.json | 18 +++++++++++++----- .../Evaluations_Delete_MaximumSet_Gen.json | 4 ++-- ...ns_GetEvaluationResults_MaximumSet_Gen.json | 2 +- .../Evaluations_Get_MaximumSet_Gen.json | 2 +- .../Evaluations_List_MaximumSet_Gen.json | 2 +- .../Evaluations_Update_MaximumSet_Gen.json | 2 +- .../Evaluations_Cancel_MaximumSet_Gen.json | 4 ++-- ...s_CreateBatchEvaluation_MaximumSet_Gen.json | 8 +++++--- ...ons_CreateOneEvaluation_MaximumSet_Gen.json | 18 +++++++++++++----- .../Evaluations_Delete_MaximumSet_Gen.json | 4 ++-- ...ns_GetEvaluationResults_MaximumSet_Gen.json | 2 +- .../Evaluations_Get_MaximumSet_Gen.json | 2 +- .../Evaluations_List_MaximumSet_Gen.json | 2 +- .../Evaluations_Update_MaximumSet_Gen.json | 2 +- 16 files changed, 52 insertions(+), 32 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json index a58506b8977c..fb25b0dbbd03 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json @@ -4,9 +4,9 @@ "parameters": { "api-version": "2025-07-31-preview", "name": "haegrdtolkdxedjsvaw", - "x-ms-client-request-id": "gpyrswfdmrmcesrhplohwoypfn" + "x-ms-client-request-id": "7946ee3f-e534-40e1-a9f5-a7afc0cc4484" }, "responses": { "204": {} } -} \ No newline at end of file +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json index d77bae51ac97..794dc22153bd 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json @@ -12,7 +12,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" @@ -49,7 +49,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" @@ -75,11 +75,13 @@ } ], "resultDatasetId": "dataset://some-dataset-id", + "state": "Succeeded", "summary": { "evaluatorStatus": [ { - "state": "succeeded", + "state": "Succeeded", "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", "usage": { "inputTokens": 102, "outputTokens": 50 diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 5a440703d526..6efe81a4b7b0 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -5,12 +5,20 @@ "api-version": "2025-07-31-preview", "oneEvaluation": { "dataSource": { + "type": "inlineData", "data": { - "query": "What is the capital of France?", - "response": "The capital of France is Paris.", - "context": "France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille." - }, - "type": "inlineData" + "dataFormat": "queryResponseMessageFormat", + "messages": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } }, "evaluators": [ { diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json index 71861a1e08bd..1ca6d7cd77a4 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json @@ -4,9 +4,9 @@ "parameters": { "api-version": "2025-07-31-preview", "name": "todlqljalqbtuoxkqszutbyy", - "x-ms-client-request-id": "gpyrswfdmrmcesrhplohwoypfn" + "x-ms-client-request-id": "7946ee3f-e534-40e1-a9f5-a7afc0cc4484" }, "responses": { "204": {} } -} \ No newline at end of file +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json index 2914f005cb33..311ad63cf223 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -40,7 +40,7 @@ ] } ], - "nextLink": "/evaluations/customer-satisfaction-eval-001/results?api-version=2025-07-31-preview&top=10&skip=10" + "nextLink": "https://microsoft.com/a" } } } diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json index e69f690da989..d33cf064f6f3 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json @@ -17,7 +17,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json index 908a23d4fcf1..62503866f288 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json @@ -19,7 +19,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json index d898de7a7054..13d52f1372c9 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json @@ -30,7 +30,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json index a58506b8977c..fb25b0dbbd03 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json @@ -4,9 +4,9 @@ "parameters": { "api-version": "2025-07-31-preview", "name": "haegrdtolkdxedjsvaw", - "x-ms-client-request-id": "gpyrswfdmrmcesrhplohwoypfn" + "x-ms-client-request-id": "7946ee3f-e534-40e1-a9f5-a7afc0cc4484" }, "responses": { "204": {} } -} \ No newline at end of file +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json index d77bae51ac97..794dc22153bd 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json @@ -12,7 +12,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" @@ -49,7 +49,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" @@ -75,11 +75,13 @@ } ], "resultDatasetId": "dataset://some-dataset-id", + "state": "Succeeded", "summary": { "evaluatorStatus": [ { - "state": "succeeded", + "state": "Succeeded", "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", "usage": { "inputTokens": 102, "outputTokens": 50 diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 5a440703d526..6efe81a4b7b0 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -5,12 +5,20 @@ "api-version": "2025-07-31-preview", "oneEvaluation": { "dataSource": { + "type": "inlineData", "data": { - "query": "What is the capital of France?", - "response": "The capital of France is Paris.", - "context": "France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille." - }, - "type": "inlineData" + "dataFormat": "queryResponseMessageFormat", + "messages": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } }, "evaluators": [ { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json index 71861a1e08bd..1ca6d7cd77a4 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json @@ -4,9 +4,9 @@ "parameters": { "api-version": "2025-07-31-preview", "name": "todlqljalqbtuoxkqszutbyy", - "x-ms-client-request-id": "gpyrswfdmrmcesrhplohwoypfn" + "x-ms-client-request-id": "7946ee3f-e534-40e1-a9f5-a7afc0cc4484" }, "responses": { "204": {} } -} \ No newline at end of file +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json index 2914f005cb33..311ad63cf223 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -40,7 +40,7 @@ ] } ], - "nextLink": "/evaluations/customer-satisfaction-eval-001/results?api-version=2025-07-31-preview&top=10&skip=10" + "nextLink": "https://microsoft.com/a" } } } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json index e69f690da989..d33cf064f6f3 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json @@ -17,7 +17,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json index 908a23d4fcf1..62503866f288 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json @@ -19,7 +19,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json index d898de7a7054..13d52f1372c9 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json @@ -30,7 +30,7 @@ "type": "inlineData", "data": { "dataFormat": "queryResponseMessageFormat", - "items": [ + "messages": [ { "query": "Hi", "response": "hi" From ec477910e494f871c5c4f98ed3063a8567139fcf Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 1 Aug 2025 12:59:22 -0700 Subject: [PATCH 20/50] Fix paths --- .../Azure.AI.Projects/evaluations/routes.tsp | 4 +- .../2025-07-31-preview/azure-ai-projects.json | 66 +++++++++---------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 27624a4b1cd9..20837d3bb30c 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -76,7 +76,7 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates a new evaluation with the specified configuration.") @added(Versions.v2025_07_31_preview) - @route(":runBatch") + @route("runs:runBatch") @post createBatchEvaluation is Azure.Core.Foundations.Operation< { @@ -98,7 +98,7 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Independent API operation to perform a single evaluation and immediately get a result.") @added(Versions.v2025_07_31_preview) - @route(":run") + @route("runs:runSingle") @post createOneEvaluation is Azure.Core.Foundations.Operation< { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index cdd090190a80..3b41659f68d8 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1048,21 +1048,38 @@ } } }, - "/evaluations:run": { + "/evaluations/runs:runBatch": { "post": { - "operationId": "Evaluations_CreateOneEvaluation", - "description": "Independent API operation to perform a single evaluation and immediately get a result.", + "operationId": "Evaluations_CreateBatchEvaluation", + "description": "Creates a new evaluation with the specified configuration.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "oneEvaluation", + "name": "Repeatability-Request-ID", + "in": "header", + "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", + "required": false, + "type": "string", + "x-ms-client-name": "repeatabilityRequestId" + }, + { + "name": "Repeatability-First-Sent", + "in": "header", + "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", + "required": false, + "type": "string", + "format": "date-time", + "x-ms-client-name": "repeatabilityFirstSent" + }, + { + "name": "evaluation", "in": "body", "description": "Complete evaluation configuration including data source, evaluators, and result settings", "required": true, "schema": { - "$ref": "#/definitions/OneEvaluation" + "$ref": "#/definitions/Evaluation" } } ], @@ -1070,7 +1087,7 @@ "201": { "description": "The request has succeeded and a new resource has been created as a result.", "schema": { - "$ref": "#/definitions/EvaluationResult" + "$ref": "#/definitions/Evaluation" } }, "default": { @@ -1087,44 +1104,27 @@ } }, "x-ms-examples": { - "Evaluations_CreateOneEvaluation": { - "$ref": "./examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json" + "Evaluations_CreateBatchEvaluation": { + "$ref": "./examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json" } } } }, - "/evaluations:runBatch": { + "/evaluations/runs:runSingle": { "post": { - "operationId": "Evaluations_CreateBatchEvaluation", - "description": "Creates a new evaluation with the specified configuration.", + "operationId": "Evaluations_CreateOneEvaluation", + "description": "Independent API operation to perform a single evaluation and immediately get a result.", "parameters": [ { "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "Repeatability-Request-ID", - "in": "header", - "description": "Unique, client-generated identifier for ensuring request idempotency. Use the same ID for retries to prevent duplicate evaluations.", - "required": false, - "type": "string", - "x-ms-client-name": "repeatabilityRequestId" - }, - { - "name": "Repeatability-First-Sent", - "in": "header", - "description": "Timestamp indicating when this request was first initiated. Used in conjunction with repeatability-request-id for idempotency control.", - "required": false, - "type": "string", - "format": "date-time", - "x-ms-client-name": "repeatabilityFirstSent" - }, - { - "name": "evaluation", + "name": "oneEvaluation", "in": "body", "description": "Complete evaluation configuration including data source, evaluators, and result settings", "required": true, "schema": { - "$ref": "#/definitions/Evaluation" + "$ref": "#/definitions/OneEvaluation" } } ], @@ -1132,7 +1132,7 @@ "201": { "description": "The request has succeeded and a new resource has been created as a result.", "schema": { - "$ref": "#/definitions/Evaluation" + "$ref": "#/definitions/EvaluationResult" } }, "default": { @@ -1149,8 +1149,8 @@ } }, "x-ms-examples": { - "Evaluations_CreateBatchEvaluation": { - "$ref": "./examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json" + "Evaluations_CreateOneEvaluation": { + "$ref": "./examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json" } } } From be8973bf67fb011bd655f6999fbcb61db88f2111 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 1 Aug 2025 13:00:58 -0700 Subject: [PATCH 21/50] Revert older version changes --- .../evaluations/chat_messages.tsp | 25 ++++++++--- .../2025-05-15-preview/azure-ai-projects.json | 43 ++++++++++++++++--- .../2025-07-31-preview/azure-ai-projects.json | 38 +++++++++++++--- 3 files changed, 90 insertions(+), 16 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp index 53fd391ed2d0..8240a712e5ce 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp @@ -68,11 +68,6 @@ model ToolResultContent extends AIContent { model Message { @doc("The role of the message author. Known values: 'system', 'assistant', 'developer', 'user'.") role: "system" | "assistant" | "developer" | "user" | string; - - #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." - @doc("The content of the message.") - @typeChangedFrom(Versions.v2025_07_31_preview, string) - content: AIContent[] | string; } @doc("A message authored by the system to guide model behavior.") @@ -80,6 +75,11 @@ model Message { model SystemMessage extends Message { @doc("Indicates this is a system message.") role: "system"; + + #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." + @doc("The content of the message.") + @typeChangedFrom(Versions.v2025_07_31_preview, string) + content: AIContent[] | string; } @doc("A message authored by a developer to guide the model during evaluation.") @@ -87,6 +87,11 @@ model SystemMessage extends Message { model DeveloperMessage extends Message { @doc("Indicates this is a developer message.") role: "developer"; + + #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." + @doc("The content of the message.") + @typeChangedFrom(Versions.v2025_07_31_preview, string) + content: AIContent[] | string; } @doc("A message authored by the end user as input to the model.") @@ -94,6 +99,11 @@ model DeveloperMessage extends Message { model UserMessage extends Message { @doc("Indicates this is a user message.") role: "user"; + + #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." + @doc("The content of the message.") + @typeChangedFrom(Versions.v2025_07_31_preview, string) + content: AIContent[] | string; } @doc("A message generated by the assistant in response to previous messages.") @@ -101,6 +111,11 @@ model UserMessage extends Message { model AssistantMessage extends Message { @doc("Indicates this is an assistant message.") role: "assistant"; + + #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." + @doc("The content of the message.") + @typeChangedFrom(Versions.v2025_07_31_preview, string) + content: AIContent[] | string; } @doc("Definition of a tool that can be used by the agent.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index b1d4db9870bc..ae43dfbb42bd 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1698,6 +1698,15 @@ "AssistantMessage": { "type": "object", "description": "A message generated by the assistant in response to previous messages.", + "properties": { + "content": { + "type": "string", + "description": "The content of the message." + } + }, + "required": [ + "content" + ], "allOf": [ { "$ref": "#/definitions/Message" @@ -2396,6 +2405,15 @@ "DeveloperMessage": { "type": "object", "description": "A message authored by a developer to guide the model during evaluation.", + "properties": { + "content": { + "type": "string", + "description": "The content of the message." + } + }, + "required": [ + "content" + ], "allOf": [ { "$ref": "#/definitions/Message" @@ -2783,16 +2801,11 @@ "x-ms-enum": { "modelAsString": true } - }, - "content": { - "type": "string", - "description": "The content of the message." } }, "discriminator": "role", "required": [ - "role", - "content" + "role" ] }, "ModelDeployment": { @@ -3273,6 +3286,15 @@ "SystemMessage": { "type": "object", "description": "A message authored by the system to guide model behavior.", + "properties": { + "content": { + "type": "string", + "description": "The content of the message." + } + }, + "required": [ + "content" + ], "allOf": [ { "$ref": "#/definitions/Message" @@ -3297,6 +3319,15 @@ "UserMessage": { "type": "object", "description": "A message authored by the end user as input to the model.", + "properties": { + "content": { + "type": "string", + "description": "The content of the message." + } + }, + "required": [ + "content" + ], "allOf": [ { "$ref": "#/definitions/Message" diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 3b41659f68d8..8d1d36936eb3 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1792,6 +1792,14 @@ "AssistantMessage": { "type": "object", "description": "A message generated by the assistant in response to previous messages.", + "properties": { + "content": { + "description": "The content of the message." + } + }, + "required": [ + "content" + ], "allOf": [ { "$ref": "#/definitions/Message" @@ -2577,6 +2585,14 @@ "DeveloperMessage": { "type": "object", "description": "A message authored by a developer to guide the model during evaluation.", + "properties": { + "content": { + "description": "The content of the message." + } + }, + "required": [ + "content" + ], "allOf": [ { "$ref": "#/definitions/Message" @@ -3831,15 +3847,11 @@ "x-ms-enum": { "modelAsString": true } - }, - "content": { - "description": "The content of the message." } }, "discriminator": "role", "required": [ - "role", - "content" + "role" ] }, "MetricType": { @@ -4484,6 +4496,14 @@ "SystemMessage": { "type": "object", "description": "A message authored by the system to guide model behavior.", + "properties": { + "content": { + "description": "The content of the message." + } + }, + "required": [ + "content" + ], "allOf": [ { "$ref": "#/definitions/Message" @@ -4611,6 +4631,14 @@ "UserMessage": { "type": "object", "description": "A message authored by the end user as input to the model.", + "properties": { + "content": { + "description": "The content of the message." + } + }, + "required": [ + "content" + ], "allOf": [ { "$ref": "#/definitions/Message" From a3e225ea760a329a9073d64c031f305544db9c20 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 1 Aug 2025 13:09:39 -0700 Subject: [PATCH 22/50] Revert changes to older version --- .../ai/Azure.AI.Projects/common/models.tsp | 1 + .../Azure.AI.Projects/evaluations/chat_messages.tsp | 8 ++++---- .../2025-05-15-preview/azure-ai-projects.json | 13 ++++--------- .../2025-07-31-preview/azure-ai-projects.json | 8 ++++---- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/common/models.tsp b/specification/ai/Azure.AI.Projects/common/models.tsp index 70bb8a68a985..064eba6ec942 100644 --- a/specification/ai/Azure.AI.Projects/common/models.tsp +++ b/specification/ai/Azure.AI.Projects/common/models.tsp @@ -127,6 +127,7 @@ model AzureOpenAIModelConfiguration extends TargetConfig { modelDeploymentName: string; @doc("Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.") + @added(Versions.v2025_07_31_preview) modelParameters?: Record; } diff --git a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp index 8240a712e5ce..255006b6dfcc 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp @@ -77,7 +77,7 @@ model SystemMessage extends Message { role: "system"; #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." - @doc("The content of the message.") + @doc("Plain text instructions provided by the system to steer model behavior.") @typeChangedFrom(Versions.v2025_07_31_preview, string) content: AIContent[] | string; } @@ -89,7 +89,7 @@ model DeveloperMessage extends Message { role: "developer"; #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." - @doc("The content of the message.") + @doc("Content provided by a developer to guide model behavior in an evaluation context.") @typeChangedFrom(Versions.v2025_07_31_preview, string) content: AIContent[] | string; } @@ -101,7 +101,7 @@ model UserMessage extends Message { role: "user"; #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." - @doc("The content of the message.") + @doc("Input content or question provided by the end user.") @typeChangedFrom(Versions.v2025_07_31_preview, string) content: AIContent[] | string; } @@ -113,7 +113,7 @@ model AssistantMessage extends Message { role: "assistant"; #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." - @doc("The content of the message.") + @doc("Response content generated by the assistant.") @typeChangedFrom(Versions.v2025_07_31_preview, string) content: AIContent[] | string; } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index ae43dfbb42bd..580f2d2f9ded 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1701,7 +1701,7 @@ "properties": { "content": { "type": "string", - "description": "The content of the message." + "description": "Response content generated by the assistant." } }, "required": [ @@ -1974,11 +1974,6 @@ "modelDeploymentName": { "type": "string", "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`." - }, - "modelParameters": { - "type": "object", - "description": "Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.", - "additionalProperties": {} } }, "required": [ @@ -2408,7 +2403,7 @@ "properties": { "content": { "type": "string", - "description": "The content of the message." + "description": "Content provided by a developer to guide model behavior in an evaluation context." } }, "required": [ @@ -3289,7 +3284,7 @@ "properties": { "content": { "type": "string", - "description": "The content of the message." + "description": "Plain text instructions provided by the system to steer model behavior." } }, "required": [ @@ -3322,7 +3317,7 @@ "properties": { "content": { "type": "string", - "description": "The content of the message." + "description": "Input content or question provided by the end user." } }, "required": [ diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 8d1d36936eb3..164a646904a3 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1794,7 +1794,7 @@ "description": "A message generated by the assistant in response to previous messages.", "properties": { "content": { - "description": "The content of the message." + "description": "Response content generated by the assistant." } }, "required": [ @@ -2587,7 +2587,7 @@ "description": "A message authored by a developer to guide the model during evaluation.", "properties": { "content": { - "description": "The content of the message." + "description": "Content provided by a developer to guide model behavior in an evaluation context." } }, "required": [ @@ -4498,7 +4498,7 @@ "description": "A message authored by the system to guide model behavior.", "properties": { "content": { - "description": "The content of the message." + "description": "Plain text instructions provided by the system to steer model behavior." } }, "required": [ @@ -4633,7 +4633,7 @@ "description": "A message authored by the end user as input to the model.", "properties": { "content": { - "description": "The content of the message." + "description": "Input content or question provided by the end user." } }, "required": [ From c0b953eb9bfab83d9871e45a852c2528337893b6 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 1 Aug 2025 13:12:25 -0700 Subject: [PATCH 23/50] Reverting more changes --- specification/ai/Azure.AI.Projects/common/models.tsp | 2 +- .../preview/2025-05-15-preview/azure-ai-projects.json | 2 +- .../preview/2025-07-31-preview/azure-ai-projects.json | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/common/models.tsp b/specification/ai/Azure.AI.Projects/common/models.tsp index 064eba6ec942..ae7a9458abe4 100644 --- a/specification/ai/Azure.AI.Projects/common/models.tsp +++ b/specification/ai/Azure.AI.Projects/common/models.tsp @@ -123,7 +123,7 @@ model AzureOpenAIModelConfiguration extends TargetConfig { @visibility(Lifecycle.Read) type: "AzureOpenAIModel"; - @doc("Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`.") + @doc("Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (e.g. `my-aoai-connection/gpt-4o`).") modelDeploymentName: string; @doc("Optional model-specific parameters to fine-tune behavior during evaluation. These may include temperature, max tokens, top-p, frequency penalty, and other model configuration options supported by the deployment.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index 580f2d2f9ded..cb475d40c91b 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1973,7 +1973,7 @@ "properties": { "modelDeploymentName": { "type": "string", - "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`." + "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (e.g. `my-aoai-connection/gpt-4o`)." } }, "required": [ diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 164a646904a3..c111db076708 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -2108,7 +2108,7 @@ "properties": { "modelDeploymentName": { "type": "string", - "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`." + "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (e.g. `my-aoai-connection/gpt-4o`)." }, "modelParameters": { "type": "object", @@ -2132,7 +2132,7 @@ "properties": { "modelDeploymentName": { "type": "string", - "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (i.e. `my-aoai-connection/gpt-4o`." + "description": "Deployment name for AOAI model. Example: gpt-4o if in AIServices or connection based `connection_name/deployment_name` (e.g. `my-aoai-connection/gpt-4o`)." }, "modelParameters": { "type": "object", From 38c0b47f87588a08bf10e1954b03a4f3e9aa0289 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 1 Aug 2025 15:10:55 -0700 Subject: [PATCH 24/50] Adding Examples --- ...ons_GetWithCredentials_MaximumSet_Gen.json | 26 ++++ .../Connections_Get_MaximumSet_Gen.json | 26 ++++ .../Connections_List_MaximumSet_Gen.json | 35 +++++ ..._CreateOrUpdateVersion_MaximumSet_Gen.json | 39 +++++ ..._CreateOrUpdateVersion_MinimumSet_Gen.json | 31 ++++ ...Datasets_DeleteVersion_MaximumSet_Gen.json | 12 ++ ...Datasets_DeleteVersion_MinimumSet_Gen.json | 12 ++ ...atasets_GetCredentials_MaximumSet_Gen.json | 24 +++ ...atasets_GetCredentials_MinimumSet_Gen.json | 24 +++ .../Datasets_GetVersion_MaximumSet_Gen.json | 21 +++ .../Datasets_GetVersion_MinimumSet_Gen.json | 19 +++ .../Datasets_ListLatest_MaximumSet_Gen.json | 28 ++++ .../Datasets_ListLatest_MinimumSet_Gen.json | 21 +++ .../Datasets_ListVersions_MaximumSet_Gen.json | 29 ++++ .../Datasets_ListVersions_MinimumSet_Gen.json | 22 +++ ...rtPendingUploadVersion_MaximumSet_Gen.json | 31 ++++ ...rtPendingUploadVersion_MinimumSet_Gen.json | 28 ++++ .../Deployments_Get_MaximumSet_Gen.json | 17 +++ .../Deployments_List_MaximumSet_Gen.json | 26 ++++ ..._CreateOrUpdateVersion_MaximumSet_Gen.json | 34 +++++ ..._CreateOrUpdateVersion_MinimumSet_Gen.json | 28 ++++ .../Indexes_DeleteVersion_MaximumSet_Gen.json | 12 ++ .../Indexes_DeleteVersion_MinimumSet_Gen.json | 12 ++ .../Indexes_GetVersion_MaximumSet_Gen.json | 19 +++ .../Indexes_GetVersion_MinimumSet_Gen.json | 18 +++ .../Indexes_ListLatest_MaximumSet_Gen.json | 26 ++++ .../Indexes_ListLatest_MinimumSet_Gen.json | 20 +++ .../Indexes_ListVersions_MaximumSet_Gen.json | 27 ++++ .../Indexes_ListVersions_MinimumSet_Gen.json | 21 +++ .../RedTeams_Create_MaximumSet_Gen.json | 53 +++++++ .../RedTeams_Create_MinimumSet_Gen.json | 38 +++++ .../RedTeams_Get_MaximumSet_Gen.json | 35 +++++ .../RedTeams_List_MaximumSet_Gen.json | 42 ++++++ .../2025-07-31-preview/azure-ai-projects.json | 139 ++++++++++++++++++ ...ons_GetWithCredentials_MaximumSet_Gen.json | 26 ++++ .../Connections_Get_MaximumSet_Gen.json | 26 ++++ .../Connections_List_MaximumSet_Gen.json | 35 +++++ ..._CreateOrUpdateVersion_MaximumSet_Gen.json | 39 +++++ ..._CreateOrUpdateVersion_MinimumSet_Gen.json | 31 ++++ ...Datasets_DeleteVersion_MaximumSet_Gen.json | 12 ++ ...Datasets_DeleteVersion_MinimumSet_Gen.json | 12 ++ ...atasets_GetCredentials_MaximumSet_Gen.json | 24 +++ ...atasets_GetCredentials_MinimumSet_Gen.json | 24 +++ .../Datasets_GetVersion_MaximumSet_Gen.json | 21 +++ .../Datasets_GetVersion_MinimumSet_Gen.json | 19 +++ .../Datasets_ListLatest_MaximumSet_Gen.json | 28 ++++ .../Datasets_ListLatest_MinimumSet_Gen.json | 21 +++ .../Datasets_ListVersions_MaximumSet_Gen.json | 29 ++++ .../Datasets_ListVersions_MinimumSet_Gen.json | 22 +++ ...rtPendingUploadVersion_MaximumSet_Gen.json | 31 ++++ ...rtPendingUploadVersion_MinimumSet_Gen.json | 28 ++++ .../Deployments_Get_MaximumSet_Gen.json | 17 +++ .../Deployments_List_MaximumSet_Gen.json | 26 ++++ ..._CreateOrUpdateVersion_MaximumSet_Gen.json | 34 +++++ ..._CreateOrUpdateVersion_MinimumSet_Gen.json | 28 ++++ .../Indexes_DeleteVersion_MaximumSet_Gen.json | 12 ++ .../Indexes_DeleteVersion_MinimumSet_Gen.json | 12 ++ .../Indexes_GetVersion_MaximumSet_Gen.json | 19 +++ .../Indexes_GetVersion_MinimumSet_Gen.json | 18 +++ .../Indexes_ListLatest_MaximumSet_Gen.json | 26 ++++ .../Indexes_ListLatest_MinimumSet_Gen.json | 20 +++ .../Indexes_ListVersions_MaximumSet_Gen.json | 27 ++++ .../Indexes_ListVersions_MinimumSet_Gen.json | 21 +++ .../RedTeams_Create_MaximumSet_Gen.json | 53 +++++++ .../RedTeams_Create_MinimumSet_Gen.json | 38 +++++ .../examples/RedTeams_Get_MaximumSet_Gen.json | 35 +++++ .../RedTeams_List_MaximumSet_Gen.json | 42 ++++++ 67 files changed, 1851 insertions(+) create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_GetWithCredentials_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_Get_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_List_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_DeleteVersion_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_DeleteVersion_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetCredentials_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetCredentials_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetVersion_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetVersion_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListVersions_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListVersions_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Deployments_Get_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Deployments_List_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_DeleteVersion_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_DeleteVersion_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_GetVersion_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_GetVersion_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListVersions_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListVersions_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Create_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Create_MinimumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Get_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_List_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_GetWithCredentials_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_Get_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_List_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_DeleteVersion_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_DeleteVersion_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetCredentials_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetCredentials_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetVersion_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetVersion_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListVersions_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListVersions_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Deployments_Get_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Deployments_List_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_DeleteVersion_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_DeleteVersion_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_GetVersion_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_GetVersion_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListVersions_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListVersions_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Create_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Create_MinimumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Get_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_List_MaximumSet_Gen.json diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_GetWithCredentials_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_GetWithCredentials_MaximumSet_Gen.json new file mode 100644 index 000000000000..95d082931fc4 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_GetWithCredentials_MaximumSet_Gen.json @@ -0,0 +1,26 @@ +{ + "title": "Connections_GetWithCredentials_MaximumSet", + "operationId": "Connections_GetWithCredentials", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "bblabqcnrjcbxlyjsejqiyixfoil", + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "name": "cpezssfnwhl", + "id": "unique-id", + "type": "AzureOpenAI", + "target": "gxrgfvtt", + "isDefault": true, + "credentials": { + "type": "BaseCredentials" + }, + "metadata": { + "key2930": "uovt" + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_Get_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_Get_MaximumSet_Gen.json new file mode 100644 index 000000000000..f6a576e46422 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_Get_MaximumSet_Gen.json @@ -0,0 +1,26 @@ +{ + "title": "Connections_Get_MaximumSet", + "operationId": "Connections_Get", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "ijxjmv", + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "name": "cpezssfnwhl", + "id": "unique-id", + "type": "AzureOpenAI", + "target": "gxrgfvtt", + "isDefault": true, + "credentials": { + "type": "BaseCredentials" + }, + "metadata": { + "key2930": "uovt" + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_List_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_List_MaximumSet_Gen.json new file mode 100644 index 000000000000..569c7ed6aeca --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Connections_List_MaximumSet_Gen.json @@ -0,0 +1,35 @@ +{ + "title": "Connections_List_MaximumSet", + "operationId": "Connections_List", + "parameters": { + "api-version": "2025-07-31-preview", + "connectionType": "AzureOpenAI", + "defaultConnection": true, + "top": 28, + "skip": 8, + "maxpagesize": 21, + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "name": "cpezssfnwhl", + "id": "unique-id", + "type": "AzureOpenAI", + "target": "gxrgfvtt", + "isDefault": true, + "credentials": { + "type": "BaseCredentials" + }, + "metadata": { + "key2930": "uovt" + } + } + ], + "nextLink": "https://microsoft.com/a" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..1ce2e6a2be68 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json @@ -0,0 +1,39 @@ +{ + "title": "Datasets_CreateOrUpdateVersion_MaximumSet", + "operationId": "Datasets_CreateOrUpdateVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "vfupg", + "version": "ecacrnqtmggfqsocagocpf", + "datasetVersion": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "description": "u", + "tags": { + "key7559": "ybrhnrxopsvmxqxibvysedlsy" + } + } + }, + "responses": { + "200": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + }, + "201": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..d65d8b8ff57b --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json @@ -0,0 +1,31 @@ +{ + "title": "Datasets_CreateOrUpdateVersion_MinimumSet", + "operationId": "Datasets_CreateOrUpdateVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "ryqozexodzklgkffokixpftfkcy", + "version": "vldbiveryfmysxuzw", + "datasetVersion": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion" + } + }, + "responses": { + "200": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + }, + "201": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_DeleteVersion_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_DeleteVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..3d0ef8268feb --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_DeleteVersion_MaximumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Datasets_DeleteVersion_MaximumSet", + "operationId": "Datasets_DeleteVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "alpqwogwzatcyot", + "version": "zrkxodsgcd" + }, + "responses": { + "204": {} + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_DeleteVersion_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_DeleteVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..325c223486e7 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_DeleteVersion_MinimumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Datasets_DeleteVersion_MinimumSet", + "operationId": "Datasets_DeleteVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "wllolxqbymlz", + "version": "ch" + }, + "responses": { + "204": {} + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetCredentials_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetCredentials_MaximumSet_Gen.json new file mode 100644 index 000000000000..f92c62537c18 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetCredentials_MaximumSet_Gen.json @@ -0,0 +1,24 @@ +{ + "title": "Datasets_GetCredentials_MaximumSet", + "operationId": "Datasets_GetCredentials", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "vzqmzhfdzfqegdk", + "version": "cuiflnznlzybcrc", + "body": {} + }, + "responses": { + "200": { + "body": { + "blobReference": { + "blobUri": "azqrjzvwamspuuvtqvvzfaxiasf", + "storageAccountArmId": "deqizonopmijpxsmmzugiwnyy", + "credential": { + "sasUri": "agojbnipwrhupvervcdiffxes", + "type": "SAS" + } + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetCredentials_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetCredentials_MinimumSet_Gen.json new file mode 100644 index 000000000000..974f38f9ee50 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetCredentials_MinimumSet_Gen.json @@ -0,0 +1,24 @@ +{ + "title": "Datasets_GetCredentials_MinimumSet", + "operationId": "Datasets_GetCredentials", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "sbblxkyqnfvdiqacezyygvts", + "version": "ikak", + "body": {} + }, + "responses": { + "200": { + "body": { + "blobReference": { + "blobUri": "azqrjzvwamspuuvtqvvzfaxiasf", + "storageAccountArmId": "deqizonopmijpxsmmzugiwnyy", + "credential": { + "sasUri": "agojbnipwrhupvervcdiffxes", + "type": "SAS" + } + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetVersion_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..bda83997a81d --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetVersion_MaximumSet_Gen.json @@ -0,0 +1,21 @@ +{ + "title": "Datasets_GetVersion_MaximumSet", + "operationId": "Datasets_GetVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "temeurpoaslnnu", + "version": "icrcjdmwkggtdogifuoolnrg" + }, + "responses": { + "200": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetVersion_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..f343f1cd231d --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_GetVersion_MinimumSet_Gen.json @@ -0,0 +1,19 @@ +{ + "title": "Datasets_GetVersion_MinimumSet", + "operationId": "Datasets_GetVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "xikgeqethuxytuseiawhklaapj", + "version": "hxksxrkgmtucfrbbnxyzwl" + }, + "responses": { + "200": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MaximumSet_Gen.json new file mode 100644 index 000000000000..15cf921f26ed --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MaximumSet_Gen.json @@ -0,0 +1,28 @@ +{ + "title": "Datasets_ListLatest_MaximumSet", + "operationId": "Datasets_ListLatest", + "parameters": { + "api-version": "2025-07-31-preview", + "top": 26, + "skip": "lbra", + "tags": "pp", + "listViewType": "ActiveOnly" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + ], + "nextLink": "https://microsoft.com/a" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json new file mode 100644 index 000000000000..9ec0c6d4ca45 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json @@ -0,0 +1,21 @@ +{ + "title": "Datasets_ListLatest_MinimumSet", + "operationId": "Datasets_ListLatest", + "parameters": { + "api-version": "2025-05-15-preview" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + ] + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListVersions_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListVersions_MaximumSet_Gen.json new file mode 100644 index 000000000000..006b2e70fc03 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListVersions_MaximumSet_Gen.json @@ -0,0 +1,29 @@ +{ + "title": "Datasets_ListVersions_MaximumSet", + "operationId": "Datasets_ListVersions", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "xogklfj", + "top": 18, + "skip": "qtdfikprjdlsosjaa", + "tags": "amyob", + "listViewType": "ActiveOnly" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + ], + "nextLink": "https://microsoft.com/a" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListVersions_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListVersions_MinimumSet_Gen.json new file mode 100644 index 000000000000..5529b3880929 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListVersions_MinimumSet_Gen.json @@ -0,0 +1,22 @@ +{ + "title": "Datasets_ListVersions_MinimumSet", + "operationId": "Datasets_ListVersions", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "hxemlflfomh" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + ] + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..503729b97f6f --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json @@ -0,0 +1,31 @@ +{ + "title": "Datasets_StartPendingUploadVersion_MaximumSet", + "operationId": "Datasets_StartPendingUploadVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "sgrzyrpltz", + "version": "ozdqisviavgqgfbtnzk", + "pendingUploadRequest": { + "pendingUploadId": "mxjahcghabuplfwtlopiqgxtcyw", + "connectionName": "bknvpmlisrqxaphkf", + "pendingUploadType": "BlobReference" + } + }, + "responses": { + "200": { + "body": { + "blobReference": { + "blobUri": "azqrjzvwamspuuvtqvvzfaxiasf", + "storageAccountArmId": "deqizonopmijpxsmmzugiwnyy", + "credential": { + "sasUri": "agojbnipwrhupvervcdiffxes", + "type": "SAS" + } + }, + "pendingUploadId": "qgizwomhljzn", + "version": "tegswajurquatmuhfde", + "pendingUploadType": "BlobReference" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..ab04125c7c3d --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json @@ -0,0 +1,28 @@ +{ + "title": "Datasets_StartPendingUploadVersion_MinimumSet", + "operationId": "Datasets_StartPendingUploadVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "duxoiaywfik", + "version": "yzzzt", + "pendingUploadRequest": { + "pendingUploadType": "BlobReference" + } + }, + "responses": { + "200": { + "body": { + "blobReference": { + "blobUri": "azqrjzvwamspuuvtqvvzfaxiasf", + "storageAccountArmId": "deqizonopmijpxsmmzugiwnyy", + "credential": { + "sasUri": "agojbnipwrhupvervcdiffxes", + "type": "SAS" + } + }, + "pendingUploadId": "qgizwomhljzn", + "pendingUploadType": "BlobReference" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Deployments_Get_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Deployments_Get_MaximumSet_Gen.json new file mode 100644 index 000000000000..06e70059d82d --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Deployments_Get_MaximumSet_Gen.json @@ -0,0 +1,17 @@ +{ + "title": "Deployments_Get_MaximumSet", + "operationId": "Deployments_Get", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "qqvacpzjfk", + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "type": "Deployment", + "name": "pjnkbctjpshoeunqedinagotrj" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Deployments_List_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Deployments_List_MaximumSet_Gen.json new file mode 100644 index 000000000000..501858012749 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Deployments_List_MaximumSet_Gen.json @@ -0,0 +1,26 @@ +{ + "title": "Deployments_List_MaximumSet", + "operationId": "Deployments_List", + "parameters": { + "api-version": "2025-07-31-preview", + "modelPublisher": "kznreojrvlwdq", + "modelName": "gplerszsuyisseeksnyvkaqperxox", + "top": 28, + "skip": 8, + "maxpagesize": 21, + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Deployment", + "name": "pjnkbctjpshoeunqedinagotrj" + } + ], + "nextLink": "https://microsoft.com/ahpgda" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..787f37bfc1a0 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json @@ -0,0 +1,34 @@ +{ + "title": "Indexes_CreateOrUpdateVersion_MaximumSet", + "operationId": "Indexes_CreateOrUpdateVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "tv", + "version": "emphgqdq", + "index": { + "type": "Index", + "description": "vdwtgaudopguto", + "tags": { + "key8846": "serbpbugykgbwuwvh" + } + } + }, + "responses": { + "200": { + "body": { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + }, + "201": { + "body": { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..d76ce7aa17aa --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json @@ -0,0 +1,28 @@ +{ + "title": "Indexes_CreateOrUpdateVersion_MinimumSet", + "operationId": "Indexes_CreateOrUpdateVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "lvlygvkgvkbzutknlnretfvivdgkb", + "version": "ct", + "index": { + "type": "Index" + } + }, + "responses": { + "200": { + "body": { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + }, + "201": { + "body": { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_DeleteVersion_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_DeleteVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..5b216e99e5f2 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_DeleteVersion_MaximumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Indexes_DeleteVersion_MaximumSet", + "operationId": "Indexes_DeleteVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "uxzepyytxjtpjkdrpxicavot", + "version": "vyihcshjrfglzhj" + }, + "responses": { + "204": {} + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_DeleteVersion_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_DeleteVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..eccb427076e4 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_DeleteVersion_MinimumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Indexes_DeleteVersion_MinimumSet", + "operationId": "Indexes_DeleteVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "mzshhabnrpuocgtyxkzftkr", + "version": "orbdovkdebzfwluronkmsoty" + }, + "responses": { + "204": {} + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_GetVersion_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_GetVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..a90856ba6f34 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_GetVersion_MaximumSet_Gen.json @@ -0,0 +1,19 @@ +{ + "title": "Indexes_GetVersion_MaximumSet", + "operationId": "Indexes_GetVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "nnhahjajcarruliib", + "version": "njynfwrxvujbklgxnibzafvcm" + }, + "responses": { + "200": { + "body": { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_GetVersion_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_GetVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..dac693c2d3c7 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_GetVersion_MinimumSet_Gen.json @@ -0,0 +1,18 @@ +{ + "title": "Indexes_GetVersion_MinimumSet", + "operationId": "Indexes_GetVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "kbfueisvpmwlmmgqrylbgcwabum", + "version": "qtzhdugby" + }, + "responses": { + "200": { + "body": { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MaximumSet_Gen.json new file mode 100644 index 000000000000..2dfc328caf79 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MaximumSet_Gen.json @@ -0,0 +1,26 @@ +{ + "title": "Indexes_ListLatest_MaximumSet", + "operationId": "Indexes_ListLatest", + "parameters": { + "api-version": "2025-07-31-preview", + "top": 11, + "skip": "qijkobbksrqhgvwzmujd", + "tags": "lmpvvr", + "listViewType": "ActiveOnly" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + ], + "nextLink": "https://microsoft.com/ayyakes" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json new file mode 100644 index 000000000000..2c147c3ed85e --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json @@ -0,0 +1,20 @@ +{ + "title": "Indexes_ListLatest_MinimumSet", + "operationId": "Indexes_ListLatest", + "parameters": { + "api-version": "2025-05-15-preview" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + ] + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListVersions_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListVersions_MaximumSet_Gen.json new file mode 100644 index 000000000000..96ba1cade2c7 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListVersions_MaximumSet_Gen.json @@ -0,0 +1,27 @@ +{ + "title": "Indexes_ListVersions_MaximumSet", + "operationId": "Indexes_ListVersions", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "tvabzvonlomkdvglaubvvqmzwdaiz", + "top": 2, + "skip": "fgwrnkksjofetmtfyizikbyv", + "tags": "hymlvxullwef", + "listViewType": "ActiveOnly" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + ], + "nextLink": "https://microsoft.com/ayyakes" + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListVersions_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListVersions_MinimumSet_Gen.json new file mode 100644 index 000000000000..0e1f0dc76b42 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListVersions_MinimumSet_Gen.json @@ -0,0 +1,21 @@ +{ + "title": "Indexes_ListVersions_MinimumSet", + "operationId": "Indexes_ListVersions", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "opimayfinqlkarudm" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + ] + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Create_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Create_MaximumSet_Gen.json new file mode 100644 index 000000000000..ae4e40cb606a --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Create_MaximumSet_Gen.json @@ -0,0 +1,53 @@ +{ + "title": "RedTeams_Create_MaximumSet", + "operationId": "RedTeams_Create", + "parameters": { + "api-version": "2025-07-31-preview", + "RedTeam": { + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "applicationScenario": "qaxxxhjp", + "tags": { + "key1287": "gbklekkgmxkfbhehgh" + }, + "properties": { + "key9280": "fwzjtipl" + }, + "target": { + "type": "TargetConfig" + } + } + }, + "responses": { + "201": { + "body": { + "id": "tztegmf", + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "applicationScenario": "qaxxxhjp", + "tags": { + "key1287": "gbklekkgmxkfbhehgh" + }, + "properties": { + "key9280": "fwzjtipl" + }, + "status": "owgxaiudnkkeqwlnhtmihvhdkbgd", + "target": { + "type": "TargetConfig" + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Create_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Create_MinimumSet_Gen.json new file mode 100644 index 000000000000..9afd67c038ee --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Create_MinimumSet_Gen.json @@ -0,0 +1,38 @@ +{ + "title": "RedTeams_Create_MinimumSet", + "operationId": "RedTeams_Create", + "parameters": { + "api-version": "2025-07-31-preview", + "RedTeam": { + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "target": { + "type": "TargetConfig" + } + } + }, + "responses": { + "201": { + "body": { + "id": "tztegmf", + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "target": { + "type": "TargetConfig" + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Get_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Get_MaximumSet_Gen.json new file mode 100644 index 000000000000..b95e1186caa1 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_Get_MaximumSet_Gen.json @@ -0,0 +1,35 @@ +{ + "title": "RedTeams_Get_MaximumSet", + "operationId": "RedTeams_Get", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "apwpcf", + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "id": "tztegmf", + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "applicationScenario": "qaxxxhjp", + "tags": { + "key1287": "gbklekkgmxkfbhehgh" + }, + "properties": { + "key9280": "fwzjtipl" + }, + "status": "owgxaiudnkkeqwlnhtmihvhdkbgd", + "target": { + "type": "TargetConfig" + } + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_List_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_List_MaximumSet_Gen.json new file mode 100644 index 000000000000..08ddce9e679b --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/RedTeams_List_MaximumSet_Gen.json @@ -0,0 +1,42 @@ +{ + "title": "RedTeams_List_MaximumSet", + "operationId": "RedTeams_List", + "parameters": { + "api-version": "2025-07-31-preview", + "top": 28, + "skip": 8, + "maxpagesize": 21, + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "id": "tztegmf", + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "applicationScenario": "qaxxxhjp", + "tags": { + "key1287": "gbklekkgmxkfbhehgh" + }, + "properties": { + "key9280": "fwzjtipl" + }, + "status": "owgxaiudnkkeqwlnhtmihvhdkbgd", + "target": { + "type": "TargetConfig" + } + } + ], + "nextLink": "https://microsoft.com/a" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index c111db076708..0e52e2beec2b 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -167,6 +167,11 @@ } } }, + "x-ms-examples": { + "Connections_List_MaximumSet": { + "$ref": "./examples/Connections_List_MaximumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -217,6 +222,11 @@ } } } + }, + "x-ms-examples": { + "Connections_Get_MaximumSet": { + "$ref": "./examples/Connections_Get_MaximumSet_Gen.json" + } } } }, @@ -265,6 +275,11 @@ } } } + }, + "x-ms-examples": { + "Connections_GetWithCredentials_MaximumSet": { + "$ref": "./examples/Connections_GetWithCredentials_MaximumSet_Gen.json" + } } } }, @@ -297,6 +312,14 @@ } } }, + "x-ms-examples": { + "Datasets_ListLatest_MaximumSet": { + "$ref": "./examples/Datasets_ListLatest_MaximumSet_Gen.json" + }, + "Datasets_ListLatest_MinimumSet": { + "$ref": "./examples/Datasets_ListLatest_MinimumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -338,6 +361,14 @@ } } }, + "x-ms-examples": { + "Datasets_ListVersions_MaximumSet": { + "$ref": "./examples/Datasets_ListVersions_MaximumSet_Gen.json" + }, + "Datasets_ListVersions_MinimumSet": { + "$ref": "./examples/Datasets_ListVersions_MinimumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -385,6 +416,14 @@ } } } + }, + "x-ms-examples": { + "Datasets_GetVersion_MaximumSet": { + "$ref": "./examples/Datasets_GetVersion_MaximumSet_Gen.json" + }, + "Datasets_GetVersion_MinimumSet": { + "$ref": "./examples/Datasets_GetVersion_MinimumSet_Gen.json" + } } }, "patch": { @@ -446,6 +485,14 @@ } } } + }, + "x-ms-examples": { + "Datasets_CreateOrUpdateVersion_MaximumSet": { + "$ref": "./examples/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json" + }, + "Datasets_CreateOrUpdateVersion_MinimumSet": { + "$ref": "./examples/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json" + } } }, "delete": { @@ -486,6 +533,14 @@ } } } + }, + "x-ms-examples": { + "Datasets_DeleteVersion_MaximumSet": { + "$ref": "./examples/Datasets_DeleteVersion_MaximumSet_Gen.json" + }, + "Datasets_DeleteVersion_MinimumSet": { + "$ref": "./examples/Datasets_DeleteVersion_MinimumSet_Gen.json" + } } } }, @@ -531,6 +586,14 @@ } } } + }, + "x-ms-examples": { + "Datasets_GetCredentials_MaximumSet": { + "$ref": "./examples/Datasets_GetCredentials_MaximumSet_Gen.json" + }, + "Datasets_GetCredentials_MinimumSet": { + "$ref": "./examples/Datasets_GetCredentials_MinimumSet_Gen.json" + } } } }, @@ -585,6 +648,14 @@ } } } + }, + "x-ms-examples": { + "Datasets_StartPendingUploadVersion_MaximumSet": { + "$ref": "./examples/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json" + }, + "Datasets_StartPendingUploadVersion_MinimumSet": { + "$ref": "./examples/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json" + } } } }, @@ -662,6 +733,11 @@ } } }, + "x-ms-examples": { + "Deployments_List_MaximumSet": { + "$ref": "./examples/Deployments_List_MaximumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -712,6 +788,11 @@ } } } + }, + "x-ms-examples": { + "Deployments_Get_MaximumSet": { + "$ref": "./examples/Deployments_Get_MaximumSet_Gen.json" + } } } }, @@ -1184,6 +1265,14 @@ } } }, + "x-ms-examples": { + "Indexes_ListLatest_MaximumSet": { + "$ref": "./examples/Indexes_ListLatest_MaximumSet_Gen.json" + }, + "Indexes_ListLatest_MinimumSet": { + "$ref": "./examples/Indexes_ListLatest_MinimumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -1225,6 +1314,14 @@ } } }, + "x-ms-examples": { + "Indexes_ListVersions_MaximumSet": { + "$ref": "./examples/Indexes_ListVersions_MaximumSet_Gen.json" + }, + "Indexes_ListVersions_MinimumSet": { + "$ref": "./examples/Indexes_ListVersions_MinimumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -1272,6 +1369,14 @@ } } } + }, + "x-ms-examples": { + "Indexes_GetVersion_MaximumSet": { + "$ref": "./examples/Indexes_GetVersion_MaximumSet_Gen.json" + }, + "Indexes_GetVersion_MinimumSet": { + "$ref": "./examples/Indexes_GetVersion_MinimumSet_Gen.json" + } } }, "patch": { @@ -1333,6 +1438,14 @@ } } } + }, + "x-ms-examples": { + "Indexes_CreateOrUpdateVersion_MaximumSet": { + "$ref": "./examples/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json" + }, + "Indexes_CreateOrUpdateVersion_MinimumSet": { + "$ref": "./examples/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json" + } } }, "delete": { @@ -1373,6 +1486,14 @@ } } } + }, + "x-ms-examples": { + "Indexes_DeleteVersion_MaximumSet": { + "$ref": "./examples/Indexes_DeleteVersion_MaximumSet_Gen.json" + }, + "Indexes_DeleteVersion_MinimumSet": { + "$ref": "./examples/Indexes_DeleteVersion_MinimumSet_Gen.json" + } } } }, @@ -1415,6 +1536,11 @@ } } }, + "x-ms-examples": { + "RedTeams_List_MaximumSet": { + "$ref": "./examples/RedTeams_List_MaximumSet_Gen.json" + } + }, "x-ms-pageable": { "nextLinkName": "nextLink" } @@ -1465,6 +1591,11 @@ } } } + }, + "x-ms-examples": { + "RedTeams_Get_MaximumSet": { + "$ref": "./examples/RedTeams_Get_MaximumSet_Gen.json" + } } } }, @@ -1505,6 +1636,14 @@ } } } + }, + "x-ms-examples": { + "RedTeams_Create_MaximumSet": { + "$ref": "./examples/RedTeams_Create_MaximumSet_Gen.json" + }, + "RedTeams_Create_MinimumSet": { + "$ref": "./examples/RedTeams_Create_MinimumSet_Gen.json" + } } } } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_GetWithCredentials_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_GetWithCredentials_MaximumSet_Gen.json new file mode 100644 index 000000000000..95d082931fc4 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_GetWithCredentials_MaximumSet_Gen.json @@ -0,0 +1,26 @@ +{ + "title": "Connections_GetWithCredentials_MaximumSet", + "operationId": "Connections_GetWithCredentials", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "bblabqcnrjcbxlyjsejqiyixfoil", + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "name": "cpezssfnwhl", + "id": "unique-id", + "type": "AzureOpenAI", + "target": "gxrgfvtt", + "isDefault": true, + "credentials": { + "type": "BaseCredentials" + }, + "metadata": { + "key2930": "uovt" + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_Get_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_Get_MaximumSet_Gen.json new file mode 100644 index 000000000000..f6a576e46422 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_Get_MaximumSet_Gen.json @@ -0,0 +1,26 @@ +{ + "title": "Connections_Get_MaximumSet", + "operationId": "Connections_Get", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "ijxjmv", + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "name": "cpezssfnwhl", + "id": "unique-id", + "type": "AzureOpenAI", + "target": "gxrgfvtt", + "isDefault": true, + "credentials": { + "type": "BaseCredentials" + }, + "metadata": { + "key2930": "uovt" + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_List_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_List_MaximumSet_Gen.json new file mode 100644 index 000000000000..569c7ed6aeca --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Connections_List_MaximumSet_Gen.json @@ -0,0 +1,35 @@ +{ + "title": "Connections_List_MaximumSet", + "operationId": "Connections_List", + "parameters": { + "api-version": "2025-07-31-preview", + "connectionType": "AzureOpenAI", + "defaultConnection": true, + "top": 28, + "skip": 8, + "maxpagesize": 21, + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "name": "cpezssfnwhl", + "id": "unique-id", + "type": "AzureOpenAI", + "target": "gxrgfvtt", + "isDefault": true, + "credentials": { + "type": "BaseCredentials" + }, + "metadata": { + "key2930": "uovt" + } + } + ], + "nextLink": "https://microsoft.com/a" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..1ce2e6a2be68 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_CreateOrUpdateVersion_MaximumSet_Gen.json @@ -0,0 +1,39 @@ +{ + "title": "Datasets_CreateOrUpdateVersion_MaximumSet", + "operationId": "Datasets_CreateOrUpdateVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "vfupg", + "version": "ecacrnqtmggfqsocagocpf", + "datasetVersion": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "description": "u", + "tags": { + "key7559": "ybrhnrxopsvmxqxibvysedlsy" + } + } + }, + "responses": { + "200": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + }, + "201": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..d65d8b8ff57b --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_CreateOrUpdateVersion_MinimumSet_Gen.json @@ -0,0 +1,31 @@ +{ + "title": "Datasets_CreateOrUpdateVersion_MinimumSet", + "operationId": "Datasets_CreateOrUpdateVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "ryqozexodzklgkffokixpftfkcy", + "version": "vldbiveryfmysxuzw", + "datasetVersion": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion" + } + }, + "responses": { + "200": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + }, + "201": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_DeleteVersion_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_DeleteVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..3d0ef8268feb --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_DeleteVersion_MaximumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Datasets_DeleteVersion_MaximumSet", + "operationId": "Datasets_DeleteVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "alpqwogwzatcyot", + "version": "zrkxodsgcd" + }, + "responses": { + "204": {} + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_DeleteVersion_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_DeleteVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..325c223486e7 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_DeleteVersion_MinimumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Datasets_DeleteVersion_MinimumSet", + "operationId": "Datasets_DeleteVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "wllolxqbymlz", + "version": "ch" + }, + "responses": { + "204": {} + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetCredentials_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetCredentials_MaximumSet_Gen.json new file mode 100644 index 000000000000..f92c62537c18 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetCredentials_MaximumSet_Gen.json @@ -0,0 +1,24 @@ +{ + "title": "Datasets_GetCredentials_MaximumSet", + "operationId": "Datasets_GetCredentials", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "vzqmzhfdzfqegdk", + "version": "cuiflnznlzybcrc", + "body": {} + }, + "responses": { + "200": { + "body": { + "blobReference": { + "blobUri": "azqrjzvwamspuuvtqvvzfaxiasf", + "storageAccountArmId": "deqizonopmijpxsmmzugiwnyy", + "credential": { + "sasUri": "agojbnipwrhupvervcdiffxes", + "type": "SAS" + } + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetCredentials_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetCredentials_MinimumSet_Gen.json new file mode 100644 index 000000000000..974f38f9ee50 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetCredentials_MinimumSet_Gen.json @@ -0,0 +1,24 @@ +{ + "title": "Datasets_GetCredentials_MinimumSet", + "operationId": "Datasets_GetCredentials", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "sbblxkyqnfvdiqacezyygvts", + "version": "ikak", + "body": {} + }, + "responses": { + "200": { + "body": { + "blobReference": { + "blobUri": "azqrjzvwamspuuvtqvvzfaxiasf", + "storageAccountArmId": "deqizonopmijpxsmmzugiwnyy", + "credential": { + "sasUri": "agojbnipwrhupvervcdiffxes", + "type": "SAS" + } + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetVersion_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..bda83997a81d --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetVersion_MaximumSet_Gen.json @@ -0,0 +1,21 @@ +{ + "title": "Datasets_GetVersion_MaximumSet", + "operationId": "Datasets_GetVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "temeurpoaslnnu", + "version": "icrcjdmwkggtdogifuoolnrg" + }, + "responses": { + "200": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetVersion_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..f343f1cd231d --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_GetVersion_MinimumSet_Gen.json @@ -0,0 +1,19 @@ +{ + "title": "Datasets_GetVersion_MinimumSet", + "operationId": "Datasets_GetVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "xikgeqethuxytuseiawhklaapj", + "version": "hxksxrkgmtucfrbbnxyzwl" + }, + "responses": { + "200": { + "body": { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MaximumSet_Gen.json new file mode 100644 index 000000000000..15cf921f26ed --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MaximumSet_Gen.json @@ -0,0 +1,28 @@ +{ + "title": "Datasets_ListLatest_MaximumSet", + "operationId": "Datasets_ListLatest", + "parameters": { + "api-version": "2025-07-31-preview", + "top": 26, + "skip": "lbra", + "tags": "pp", + "listViewType": "ActiveOnly" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + ], + "nextLink": "https://microsoft.com/a" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MinimumSet_Gen.json new file mode 100644 index 000000000000..9ec0c6d4ca45 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MinimumSet_Gen.json @@ -0,0 +1,21 @@ +{ + "title": "Datasets_ListLatest_MinimumSet", + "operationId": "Datasets_ListLatest", + "parameters": { + "api-version": "2025-05-15-preview" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + ] + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListVersions_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListVersions_MaximumSet_Gen.json new file mode 100644 index 000000000000..006b2e70fc03 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListVersions_MaximumSet_Gen.json @@ -0,0 +1,29 @@ +{ + "title": "Datasets_ListVersions_MaximumSet", + "operationId": "Datasets_ListVersions", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "xogklfj", + "top": 18, + "skip": "qtdfikprjdlsosjaa", + "tags": "amyob", + "listViewType": "ActiveOnly" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "isReference": true, + "id": "hvpdyfpeoqcl", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + ], + "nextLink": "https://microsoft.com/a" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListVersions_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListVersions_MinimumSet_Gen.json new file mode 100644 index 000000000000..5529b3880929 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListVersions_MinimumSet_Gen.json @@ -0,0 +1,22 @@ +{ + "title": "Datasets_ListVersions_MinimumSet", + "operationId": "Datasets_ListVersions", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "hxemlflfomh" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "dataUri": "Replace this value with a string matching RegExp [a-zA-Z0-9_]", + "type": "DatasetVersion", + "name": "xtdmvwhhobloqqsovgpynsnow", + "version": "eaixzft" + } + ] + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..503729b97f6f --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_StartPendingUploadVersion_MaximumSet_Gen.json @@ -0,0 +1,31 @@ +{ + "title": "Datasets_StartPendingUploadVersion_MaximumSet", + "operationId": "Datasets_StartPendingUploadVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "sgrzyrpltz", + "version": "ozdqisviavgqgfbtnzk", + "pendingUploadRequest": { + "pendingUploadId": "mxjahcghabuplfwtlopiqgxtcyw", + "connectionName": "bknvpmlisrqxaphkf", + "pendingUploadType": "BlobReference" + } + }, + "responses": { + "200": { + "body": { + "blobReference": { + "blobUri": "azqrjzvwamspuuvtqvvzfaxiasf", + "storageAccountArmId": "deqizonopmijpxsmmzugiwnyy", + "credential": { + "sasUri": "agojbnipwrhupvervcdiffxes", + "type": "SAS" + } + }, + "pendingUploadId": "qgizwomhljzn", + "version": "tegswajurquatmuhfde", + "pendingUploadType": "BlobReference" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..ab04125c7c3d --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_StartPendingUploadVersion_MinimumSet_Gen.json @@ -0,0 +1,28 @@ +{ + "title": "Datasets_StartPendingUploadVersion_MinimumSet", + "operationId": "Datasets_StartPendingUploadVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "duxoiaywfik", + "version": "yzzzt", + "pendingUploadRequest": { + "pendingUploadType": "BlobReference" + } + }, + "responses": { + "200": { + "body": { + "blobReference": { + "blobUri": "azqrjzvwamspuuvtqvvzfaxiasf", + "storageAccountArmId": "deqizonopmijpxsmmzugiwnyy", + "credential": { + "sasUri": "agojbnipwrhupvervcdiffxes", + "type": "SAS" + } + }, + "pendingUploadId": "qgizwomhljzn", + "pendingUploadType": "BlobReference" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Deployments_Get_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Deployments_Get_MaximumSet_Gen.json new file mode 100644 index 000000000000..06e70059d82d --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Deployments_Get_MaximumSet_Gen.json @@ -0,0 +1,17 @@ +{ + "title": "Deployments_Get_MaximumSet", + "operationId": "Deployments_Get", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "qqvacpzjfk", + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "type": "Deployment", + "name": "pjnkbctjpshoeunqedinagotrj" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Deployments_List_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Deployments_List_MaximumSet_Gen.json new file mode 100644 index 000000000000..501858012749 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Deployments_List_MaximumSet_Gen.json @@ -0,0 +1,26 @@ +{ + "title": "Deployments_List_MaximumSet", + "operationId": "Deployments_List", + "parameters": { + "api-version": "2025-07-31-preview", + "modelPublisher": "kznreojrvlwdq", + "modelName": "gplerszsuyisseeksnyvkaqperxox", + "top": 28, + "skip": 8, + "maxpagesize": 21, + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Deployment", + "name": "pjnkbctjpshoeunqedinagotrj" + } + ], + "nextLink": "https://microsoft.com/ahpgda" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..787f37bfc1a0 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_CreateOrUpdateVersion_MaximumSet_Gen.json @@ -0,0 +1,34 @@ +{ + "title": "Indexes_CreateOrUpdateVersion_MaximumSet", + "operationId": "Indexes_CreateOrUpdateVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "tv", + "version": "emphgqdq", + "index": { + "type": "Index", + "description": "vdwtgaudopguto", + "tags": { + "key8846": "serbpbugykgbwuwvh" + } + } + }, + "responses": { + "200": { + "body": { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + }, + "201": { + "body": { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..d76ce7aa17aa --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_CreateOrUpdateVersion_MinimumSet_Gen.json @@ -0,0 +1,28 @@ +{ + "title": "Indexes_CreateOrUpdateVersion_MinimumSet", + "operationId": "Indexes_CreateOrUpdateVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "lvlygvkgvkbzutknlnretfvivdgkb", + "version": "ct", + "index": { + "type": "Index" + } + }, + "responses": { + "200": { + "body": { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + }, + "201": { + "body": { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_DeleteVersion_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_DeleteVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..5b216e99e5f2 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_DeleteVersion_MaximumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Indexes_DeleteVersion_MaximumSet", + "operationId": "Indexes_DeleteVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "uxzepyytxjtpjkdrpxicavot", + "version": "vyihcshjrfglzhj" + }, + "responses": { + "204": {} + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_DeleteVersion_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_DeleteVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..eccb427076e4 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_DeleteVersion_MinimumSet_Gen.json @@ -0,0 +1,12 @@ +{ + "title": "Indexes_DeleteVersion_MinimumSet", + "operationId": "Indexes_DeleteVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "mzshhabnrpuocgtyxkzftkr", + "version": "orbdovkdebzfwluronkmsoty" + }, + "responses": { + "204": {} + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_GetVersion_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_GetVersion_MaximumSet_Gen.json new file mode 100644 index 000000000000..a90856ba6f34 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_GetVersion_MaximumSet_Gen.json @@ -0,0 +1,19 @@ +{ + "title": "Indexes_GetVersion_MaximumSet", + "operationId": "Indexes_GetVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "nnhahjajcarruliib", + "version": "njynfwrxvujbklgxnibzafvcm" + }, + "responses": { + "200": { + "body": { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_GetVersion_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_GetVersion_MinimumSet_Gen.json new file mode 100644 index 000000000000..dac693c2d3c7 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_GetVersion_MinimumSet_Gen.json @@ -0,0 +1,18 @@ +{ + "title": "Indexes_GetVersion_MinimumSet", + "operationId": "Indexes_GetVersion", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "kbfueisvpmwlmmgqrylbgcwabum", + "version": "qtzhdugby" + }, + "responses": { + "200": { + "body": { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MaximumSet_Gen.json new file mode 100644 index 000000000000..2dfc328caf79 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MaximumSet_Gen.json @@ -0,0 +1,26 @@ +{ + "title": "Indexes_ListLatest_MaximumSet", + "operationId": "Indexes_ListLatest", + "parameters": { + "api-version": "2025-07-31-preview", + "top": 11, + "skip": "qijkobbksrqhgvwzmujd", + "tags": "lmpvvr", + "listViewType": "ActiveOnly" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + ], + "nextLink": "https://microsoft.com/ayyakes" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MinimumSet_Gen.json new file mode 100644 index 000000000000..2c147c3ed85e --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MinimumSet_Gen.json @@ -0,0 +1,20 @@ +{ + "title": "Indexes_ListLatest_MinimumSet", + "operationId": "Indexes_ListLatest", + "parameters": { + "api-version": "2025-05-15-preview" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + ] + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListVersions_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListVersions_MaximumSet_Gen.json new file mode 100644 index 000000000000..96ba1cade2c7 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListVersions_MaximumSet_Gen.json @@ -0,0 +1,27 @@ +{ + "title": "Indexes_ListVersions_MaximumSet", + "operationId": "Indexes_ListVersions", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "tvabzvonlomkdvglaubvvqmzwdaiz", + "top": 2, + "skip": "fgwrnkksjofetmtfyizikbyv", + "tags": "hymlvxullwef", + "listViewType": "ActiveOnly" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Index", + "id": "fxdrmhqrfmclzkjmfldnszfnztla", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + ], + "nextLink": "https://microsoft.com/ayyakes" + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListVersions_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListVersions_MinimumSet_Gen.json new file mode 100644 index 000000000000..0e1f0dc76b42 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListVersions_MinimumSet_Gen.json @@ -0,0 +1,21 @@ +{ + "title": "Indexes_ListVersions_MinimumSet", + "operationId": "Indexes_ListVersions", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "opimayfinqlkarudm" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "type": "Index", + "name": "fiwkavutgfan", + "version": "iqupugmfgctoxzdubrcdnzx" + } + ] + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Create_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Create_MaximumSet_Gen.json new file mode 100644 index 000000000000..ae4e40cb606a --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Create_MaximumSet_Gen.json @@ -0,0 +1,53 @@ +{ + "title": "RedTeams_Create_MaximumSet", + "operationId": "RedTeams_Create", + "parameters": { + "api-version": "2025-07-31-preview", + "RedTeam": { + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "applicationScenario": "qaxxxhjp", + "tags": { + "key1287": "gbklekkgmxkfbhehgh" + }, + "properties": { + "key9280": "fwzjtipl" + }, + "target": { + "type": "TargetConfig" + } + } + }, + "responses": { + "201": { + "body": { + "id": "tztegmf", + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "applicationScenario": "qaxxxhjp", + "tags": { + "key1287": "gbklekkgmxkfbhehgh" + }, + "properties": { + "key9280": "fwzjtipl" + }, + "status": "owgxaiudnkkeqwlnhtmihvhdkbgd", + "target": { + "type": "TargetConfig" + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Create_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Create_MinimumSet_Gen.json new file mode 100644 index 000000000000..9afd67c038ee --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Create_MinimumSet_Gen.json @@ -0,0 +1,38 @@ +{ + "title": "RedTeams_Create_MinimumSet", + "operationId": "RedTeams_Create", + "parameters": { + "api-version": "2025-07-31-preview", + "RedTeam": { + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "target": { + "type": "TargetConfig" + } + } + }, + "responses": { + "201": { + "body": { + "id": "tztegmf", + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "target": { + "type": "TargetConfig" + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Get_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Get_MaximumSet_Gen.json new file mode 100644 index 000000000000..b95e1186caa1 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_Get_MaximumSet_Gen.json @@ -0,0 +1,35 @@ +{ + "title": "RedTeams_Get_MaximumSet", + "operationId": "RedTeams_Get", + "parameters": { + "api-version": "2025-07-31-preview", + "name": "apwpcf", + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "id": "tztegmf", + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "applicationScenario": "qaxxxhjp", + "tags": { + "key1287": "gbklekkgmxkfbhehgh" + }, + "properties": { + "key9280": "fwzjtipl" + }, + "status": "owgxaiudnkkeqwlnhtmihvhdkbgd", + "target": { + "type": "TargetConfig" + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_List_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_List_MaximumSet_Gen.json new file mode 100644 index 000000000000..08ddce9e679b --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/RedTeams_List_MaximumSet_Gen.json @@ -0,0 +1,42 @@ +{ + "title": "RedTeams_List_MaximumSet", + "operationId": "RedTeams_List", + "parameters": { + "api-version": "2025-07-31-preview", + "top": 28, + "skip": 8, + "maxpagesize": 21, + "x-ms-client-request-id": "cf35b680-dc80-4815-ab83-9364acc3bce6" + }, + "responses": { + "200": { + "body": { + "value": [ + { + "id": "tztegmf", + "numTurns": 10, + "attackStrategies": [ + "easy" + ], + "simulationOnly": true, + "riskCategories": [ + "HateUnfairness" + ], + "applicationScenario": "qaxxxhjp", + "tags": { + "key1287": "gbklekkgmxkfbhehgh" + }, + "properties": { + "key9280": "fwzjtipl" + }, + "status": "owgxaiudnkkeqwlnhtmihvhdkbgd", + "target": { + "type": "TargetConfig" + } + } + ], + "nextLink": "https://microsoft.com/a" + } + } + } +} From b2203701a18b8fdae3187e3d844ed6474fca293d Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 1 Aug 2025 17:21:51 -0700 Subject: [PATCH 25/50] adding 2 MB limit --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 2 +- .../preview/2025-07-31-preview/azure-ai-projects.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index a60fd57d4e80..fe17f7e83c79 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -161,7 +161,7 @@ model InlineJson extends InlineData { messages: string[]; } -@doc("Data source using inline data provided directly in the request.") +@doc("Data source using inline data provided directly in the request. Size limit is 2 MB") @added(Versions.v2025_07_31_preview) model InlineDataSource extends EvaluationDataSource { @doc("Specifies inline JSON data source") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 0e52e2beec2b..1a7e2d7f8393 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3819,7 +3819,7 @@ }, "InlineDataSource": { "type": "object", - "description": "Data source using inline data provided directly in the request.", + "description": "Data source using inline data provided directly in the request. Size limit is 2 MB", "properties": { "id": { "type": "string", @@ -3842,7 +3842,7 @@ }, "InlineDataSourceUpdate": { "type": "object", - "description": "Data source using inline data provided directly in the request.", + "description": "Data source using inline data provided directly in the request. Size limit is 2 MB", "properties": { "id": { "type": "string", From 66912c1a8ac14befe67254965abd28f7fd433281 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 1 Aug 2025 17:41:44 -0700 Subject: [PATCH 26/50] Fix version in examples --- .../2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json | 2 +- .../2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json | 2 +- .../examples/Datasets_ListLatest_MinimumSet_Gen.json | 2 +- .../examples/Indexes_ListLatest_MinimumSet_Gen.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json index 9ec0c6d4ca45..56c3b4c2a204 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Datasets_ListLatest_MinimumSet_Gen.json @@ -2,7 +2,7 @@ "title": "Datasets_ListLatest_MinimumSet", "operationId": "Datasets_ListLatest", "parameters": { - "api-version": "2025-05-15-preview" + "api-version": "2025-07-31-preview" }, "responses": { "200": { diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json index 2c147c3ed85e..871ffb5bc5f3 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Indexes_ListLatest_MinimumSet_Gen.json @@ -2,7 +2,7 @@ "title": "Indexes_ListLatest_MinimumSet", "operationId": "Indexes_ListLatest", "parameters": { - "api-version": "2025-05-15-preview" + "api-version": "2025-07-31-preview" }, "responses": { "200": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MinimumSet_Gen.json index 9ec0c6d4ca45..56c3b4c2a204 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MinimumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Datasets_ListLatest_MinimumSet_Gen.json @@ -2,7 +2,7 @@ "title": "Datasets_ListLatest_MinimumSet", "operationId": "Datasets_ListLatest", "parameters": { - "api-version": "2025-05-15-preview" + "api-version": "2025-07-31-preview" }, "responses": { "200": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MinimumSet_Gen.json index 2c147c3ed85e..871ffb5bc5f3 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MinimumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Indexes_ListLatest_MinimumSet_Gen.json @@ -2,7 +2,7 @@ "title": "Indexes_ListLatest_MinimumSet", "operationId": "Indexes_ListLatest", "parameters": { - "api-version": "2025-05-15-preview" + "api-version": "2025-07-31-preview" }, "responses": { "200": { From 501b5f9523ff89ade8a61e1ac0257b82101dadac Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Sat, 2 Aug 2025 01:13:18 -0700 Subject: [PATCH 27/50] Added deprecated in route --- .../Azure.AI.Projects/evaluations/routes.tsp | 38 +++++++- .../2025-07-31-preview/azure-ai-projects.json | 86 ++++++++++++++++++- 2 files changed, 120 insertions(+), 4 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 20837d3bb30c..b1d33326051f 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -50,6 +50,7 @@ interface Evaluations { @route("runs:run") @removed(Versions.v2025_07_31_preview) @post + @sharedRoute create is Azure.Core.Foundations.Operation< { @doc("Evaluation to be run") @@ -59,11 +60,28 @@ interface Evaluations { ResourceCreatedResponse >; + #deprecated "Use createBatchEvaluation or createOneEvaluation" + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" + @doc("Creates an evaluation run.") + @route("runs:run") + @added(Versions.v2025_07_31_preview) + @post + @sharedRoute + createDeprecated is Azure.Core.Foundations.Operation< + { + @doc("Evaluation to be run") + @body + evaluation: Evaluation; + }, + ResourceCreatedResponse + >; + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates an agent evaluation run.") @route("runs:runAgent") - @removed(Versions.v2025_07_31_preview) // Use createOrReplace instead + @removed(Versions.v2025_07_31_preview) @post + @sharedRoute createAgentEvaluation is Azure.Core.Foundations.Operation< { @doc("Agent evaluation to be run") @@ -73,6 +91,22 @@ interface Evaluations { ResourceCreatedResponse >; + #deprecated "Use createBatchEvaluation or createOneEvaluation" + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" + @doc("Creates an agent evaluation run.") + @route("runs:runAgent") + @added(Versions.v2025_07_31_preview) // Use createOrReplace instead + @post + @sharedRoute + createAgentEvaluationDeprecated is Azure.Core.Foundations.Operation< + { + @doc("Agent evaluation to be run") + @body + evaluation: AgentEvaluationRequest; + }, + ResourceCreatedResponse + >; + #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates a new evaluation with the specified configuration.") @added(Versions.v2025_07_31_preview) @@ -106,7 +140,7 @@ interface Evaluations { @body oneEvaluation: OneEvaluation; }, - ResourceCreatedResponse + OkResponse >; @doc("Updates specific properties of an existing evaluation. Supports modification of metadata fields including description, display name, and tags. Note: Core evaluation configuration such as data sources and evaluators cannot be modified after creation.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 1a7e2d7f8393..decdaf9406b9 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1129,6 +1129,88 @@ } } }, + "/evaluations/runs:run": { + "post": { + "operationId": "Evaluations_CreateDeprecated", + "description": "Creates an evaluation run.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "evaluation", + "in": "body", + "description": "Evaluation to be run", + "required": true, + "schema": { + "$ref": "#/definitions/Evaluation" + } + } + ], + "responses": { + "201": { + "description": "The request has succeeded and a new resource has been created as a result.", + "schema": { + "$ref": "#/definitions/Evaluation" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "deprecated": true + } + }, + "/evaluations/runs:runAgent": { + "post": { + "operationId": "Evaluations_CreateAgentEvaluationDeprecated", + "description": "Creates an agent evaluation run.", + "parameters": [ + { + "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" + }, + { + "name": "evaluation", + "in": "body", + "description": "Agent evaluation to be run", + "required": true, + "schema": { + "$ref": "#/definitions/AgentEvaluationRequest" + } + } + ], + "responses": { + "201": { + "description": "The request has succeeded and a new resource has been created as a result.", + "schema": { + "$ref": "#/definitions/AgentEvaluation" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" + }, + "headers": { + "x-ms-error-code": { + "type": "string", + "description": "String error code indicating what went wrong." + } + } + } + }, + "deprecated": true + } + }, "/evaluations/runs:runBatch": { "post": { "operationId": "Evaluations_CreateBatchEvaluation", @@ -1210,8 +1292,8 @@ } ], "responses": { - "201": { - "description": "The request has succeeded and a new resource has been created as a result.", + "200": { + "description": "The request has succeeded.", "schema": { "$ref": "#/definitions/EvaluationResult" } From b9bb23df1905312d3acfa34fd513466ced23618c Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Sat, 2 Aug 2025 01:15:59 -0700 Subject: [PATCH 28/50] Adding comment --- specification/ai/Azure.AI.Projects/evaluations/routes.tsp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index b1d33326051f..4b86177b9456 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -45,6 +45,8 @@ interface Evaluations { ListQueryParametersTrait >; + // create is deprecated from version v2025_07_31_preview onwards. + // so it has 2 entries here (create and createDeprecated) #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates an evaluation run.") @route("runs:run") From 3c6ae41f7a811857ae8efe72be0291847e96caa1 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Sat, 2 Aug 2025 01:31:19 -0700 Subject: [PATCH 29/50] Update name to id in new version of API --- .../ai/Azure.AI.Projects/evaluations/models.tsp | 5 +++-- .../Evaluations_Cancel_MaximumSet_Gen.json | 2 +- .../Evaluations_Delete_MaximumSet_Gen.json | 2 +- ...tions_GetEvaluationResults_MaximumSet_Gen.json | 2 +- .../Evaluations_Get_MaximumSet_Gen.json | 2 +- .../Evaluations_Update_MaximumSet_Gen.json | 2 +- .../2025-07-31-preview/azure-ai-projects.json | 15 +++++++-------- .../Evaluations_Cancel_MaximumSet_Gen.json | 2 +- .../Evaluations_Delete_MaximumSet_Gen.json | 2 +- ...tions_GetEvaluationResults_MaximumSet_Gen.json | 2 +- .../examples/Evaluations_Get_MaximumSet_Gen.json | 2 +- .../Evaluations_Update_MaximumSet_Gen.json | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index fe17f7e83c79..1c9b292b8618 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -376,10 +376,11 @@ model EvaluationSummaryResult { @added(Versions.v2025_05_15_preview) model Evaluation { @doc("Identifier of the evaluation.") - @key("name") + @key("id") + @renamedFrom(Versions.v2025_07_31_preview, "name") @encodedName("application/json", "id") @visibility(Lifecycle.Read) - name: string; + id: string; @doc("Data for evaluation.") @removed(Versions.v2025_07_31_preview) // Replaced by `dataSource` diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json index fb25b0dbbd03..14d526da43b6 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Cancel_MaximumSet_Gen.json @@ -3,7 +3,7 @@ "operationId": "Evaluations_Cancel", "parameters": { "api-version": "2025-07-31-preview", - "name": "haegrdtolkdxedjsvaw", + "id": "haegrdtolkdxedjsvaw", "x-ms-client-request-id": "7946ee3f-e534-40e1-a9f5-a7afc0cc4484" }, "responses": { diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json index 1ca6d7cd77a4..0a6dfe828d55 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Delete_MaximumSet_Gen.json @@ -3,7 +3,7 @@ "operationId": "Evaluations_Delete", "parameters": { "api-version": "2025-07-31-preview", - "name": "todlqljalqbtuoxkqszutbyy", + "id": "todlqljalqbtuoxkqszutbyy", "x-ms-client-request-id": "7946ee3f-e534-40e1-a9f5-a7afc0cc4484" }, "responses": { diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json index 311ad63cf223..c9da3e883b34 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -2,7 +2,7 @@ "title": "Evaluations_GetEvaluationResults", "operationId": "Evaluations_GetEvaluationResults", "parameters": { - "name": "customer-satisfaction-eval-001", + "id": "customer-satisfaction-eval-001", "api-version": "2025-07-31-preview", "top": 10, "skip": 0, diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json index d33cf064f6f3..584ed7f0a07d 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Get_MaximumSet_Gen.json @@ -3,7 +3,7 @@ "operationId": "Evaluations_Get", "parameters": { "api-version": "2025-07-31-preview", - "name": "customer-satisfaction-eval-001", + "id": "customer-satisfaction-eval-001", "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc" }, "responses": { diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json index 13d52f1372c9..0542a9d2f8d7 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json @@ -3,7 +3,7 @@ "operationId": "Evaluations_Update", "parameters": { "api-version": "2025-07-31-preview", - "name": "customer-satisfaction-eval-001", + "id": "customer-satisfaction-eval-001", "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc", "resource": { "displayName": "Updated Customer Satisfaction Evaluation", diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index decdaf9406b9..bb7c161ae257 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -852,7 +852,7 @@ } } }, - "/evaluations/runs/{name}": { + "/evaluations/runs/{id}": { "get": { "operationId": "Evaluations_Get", "description": "Get an evaluation run by name.", @@ -861,7 +861,7 @@ "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "name", + "name": "id", "in": "path", "description": "Identifier of the evaluation.", "required": true, @@ -919,7 +919,7 @@ "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "name", + "name": "id", "in": "path", "description": "Identifier of the evaluation.", "required": true, @@ -979,7 +979,7 @@ "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "name", + "name": "id", "in": "path", "description": "Identifier of the evaluation.", "required": true, @@ -1020,7 +1020,7 @@ } } }, - "/evaluations/runs/{name}:cancel": { + "/evaluations/runs/{id}:cancel": { "post": { "operationId": "Evaluations_Cancel", "description": "Cancel an evaluation run by name", @@ -1029,7 +1029,7 @@ "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" }, { - "name": "name", + "name": "id", "in": "path", "description": "Identifier of the evaluation.", "required": true, @@ -2862,8 +2862,7 @@ "id": { "type": "string", "description": "Identifier of the evaluation.", - "readOnly": true, - "x-ms-client-name": "name" + "readOnly": true }, "dataSource": { "$ref": "#/definitions/EvaluationDataSource", diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json index fb25b0dbbd03..14d526da43b6 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Cancel_MaximumSet_Gen.json @@ -3,7 +3,7 @@ "operationId": "Evaluations_Cancel", "parameters": { "api-version": "2025-07-31-preview", - "name": "haegrdtolkdxedjsvaw", + "id": "haegrdtolkdxedjsvaw", "x-ms-client-request-id": "7946ee3f-e534-40e1-a9f5-a7afc0cc4484" }, "responses": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json index 1ca6d7cd77a4..0a6dfe828d55 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Delete_MaximumSet_Gen.json @@ -3,7 +3,7 @@ "operationId": "Evaluations_Delete", "parameters": { "api-version": "2025-07-31-preview", - "name": "todlqljalqbtuoxkqszutbyy", + "id": "todlqljalqbtuoxkqszutbyy", "x-ms-client-request-id": "7946ee3f-e534-40e1-a9f5-a7afc0cc4484" }, "responses": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json index 311ad63cf223..c9da3e883b34 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -2,7 +2,7 @@ "title": "Evaluations_GetEvaluationResults", "operationId": "Evaluations_GetEvaluationResults", "parameters": { - "name": "customer-satisfaction-eval-001", + "id": "customer-satisfaction-eval-001", "api-version": "2025-07-31-preview", "top": 10, "skip": 0, diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json index d33cf064f6f3..584ed7f0a07d 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Get_MaximumSet_Gen.json @@ -3,7 +3,7 @@ "operationId": "Evaluations_Get", "parameters": { "api-version": "2025-07-31-preview", - "name": "customer-satisfaction-eval-001", + "id": "customer-satisfaction-eval-001", "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc" }, "responses": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json index 13d52f1372c9..0542a9d2f8d7 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json @@ -3,7 +3,7 @@ "operationId": "Evaluations_Update", "parameters": { "api-version": "2025-07-31-preview", - "name": "customer-satisfaction-eval-001", + "id": "customer-satisfaction-eval-001", "x-ms-client-request-id": "12345678-1234-1234-1234-123456789abc", "resource": { "displayName": "Updated Customer Satisfaction Evaluation", From 2ed63eedd9ff0a0eb4ae6b19e7a676e029cf7b72 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 09:50:33 -0700 Subject: [PATCH 30/50] Fix --- .../ai/Azure.AI.Projects/evaluations/models.tsp | 6 ++++++ .../2025-07-31-preview/azure-ai-projects.json | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 1c9b292b8618..b125c3931717 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -587,6 +587,12 @@ union EvaluationResultOutcome { @doc("The evaluation criteria do not apply to this particular data row or context") notApplicable: "notApplicable", + @doc("The evaluation is skipped for this particular data row") + skipped: "skipped", + + @doc("The evaluation resulted in an error for this particular data row") + error: "error", + string, } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index bb7c161ae257..c3e0401c048d 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3262,7 +3262,9 @@ "pass", "fail", "notEvaluated", - "notApplicable" + "notApplicable", + "skipped", + "error" ], "x-ms-enum": { "name": "EvaluationResultOutcome", @@ -3287,6 +3289,16 @@ "name": "notApplicable", "value": "notApplicable", "description": "The evaluation criteria do not apply to this particular data row or context" + }, + { + "name": "skipped", + "value": "skipped", + "description": "The evaluation is skipped for this particular data row" + }, + { + "name": "error", + "value": "error", + "description": "The evaluation resulted in an error for this particular data row" } ] } From 0af086d724860d401158175f85676aa5d412ecbf Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 10:22:21 -0700 Subject: [PATCH 31/50] Model validation fixes --- .../Azure.AI.Projects/evaluations/routes.tsp | 4 +- ...ntEvaluationDeprecated_MaximumSet_Gen.json | 54 ++++++++++++++++++ ...tions_CreateDeprecated_MaximumSet_Gen.json | 57 +++++++++++++++++++ .../2025-07-31-preview/azure-ai-projects.json | 18 ++++-- ...ntEvaluationDeprecated_MaximumSet_Gen.json | 54 ++++++++++++++++++ ...tions_CreateDeprecated_MaximumSet_Gen.json | 57 +++++++++++++++++++ 6 files changed, 238 insertions(+), 6 deletions(-) create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 4b86177b9456..9e69cd40347a 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -151,13 +151,13 @@ interface Evaluations { #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Retrieves detailed row-by-row evaluation results with pagination support. This endpoint provides access to individual evaluation outcomes, scores, and reasoning for each data row processed.") - @route("runs/{name}/results") + @route("runs/{id}/results") @get @added(Versions.v2025_07_31_preview) getEvaluationResults( @doc("Unique identifier of the evaluation whose results are being retrieved") @path - name: string, + id: string, @doc("API version specifier for this operation") @query("api-version") diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json new file mode 100644 index 000000000000..6a26a52df99c --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json @@ -0,0 +1,54 @@ +{ + "title": "Evaluations_CreateAgentEvaluationDeprecated_MaximumSet", + "operationId": "Evaluations_CreateAgentEvaluationDeprecated", + "parameters": { + "api-version": "2025-05-15-preview", + "evaluation": { + "runId": "ecfcffqsrdhpecocrufffiqz", + "threadId": "cxjjsyhkeezgnaqqyerrdrbbth", + "evaluators": { + "key2653": { + "id": "gujwtvhptykq", + "initParams": {}, + "dataMapping": { + "key7400": "ijkjfvoswni" + } + } + }, + "samplingConfiguration": { + "name": "tj", + "samplingPercent": 7, + "maxRequestRate": 8 + }, + "redactionConfiguration": { + "redactScoreProperties": true + }, + "appInsightsConnectionString": "dvcnrcwar" + } + }, + "responses": { + "201": { + "body": { + "id": "gji", + "status": "ozywrhiasll", + "error": "stcjbhxgmqvjqwzmbvaa", + "result": [ + { + "evaluator": "upnt", + "evaluatorId": "upnt", + "score": 13, + "status": "eoxw", + "reason": "kfrjetziuketgioobeydlugonzfxo", + "version": "lfqwtzuktnbmcxxwkghmlpdtuu", + "threadId": "sldwqklyuxeayfmxpbl", + "runId": "lauehlf", + "error": "lrutmshl", + "additionalDetails": { + "key3590": "vhhneubavtiklsjcdexij" + } + } + ] + } + } + } +} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json new file mode 100644 index 000000000000..e40f40708d81 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json @@ -0,0 +1,57 @@ +{ + "title": "Evaluations_CreateDeprecated_MaximumSet", + "operationId": "Evaluations_CreateDeprecated", + "parameters": { + "api-version": "2025-07-31-preview", + "evaluation": { + "data": { + "type": "InputData" + }, + "displayName": "ymjxkodggqrhjnmusta", + "description": "cuviaxytqckkybxwc", + "tags": { + "key30": "nqrdpedyhkjzgrbtqhxbmkdmlpdf" + }, + "properties": { + "key9192": "e" + }, + "evaluators": { + "key2735": { + "id": "gujwtvhptykq", + "initParams": {}, + "dataMapping": { + "key7400": "ijkjfvoswni" + } + } + } + } + }, + "responses": { + "201": { + "body": { + "id": "aarhggbojnh", + "data": { + "type": "InputData" + }, + "displayName": "ymjxkodggqrhjnmusta", + "description": "cuviaxytqckkybxwc", + "status": "uhejnohmiiogajkx", + "tags": { + "key30": "nqrdpedyhkjzgrbtqhxbmkdmlpdf" + }, + "properties": { + "key9192": "e" + }, + "evaluators": { + "key2735": { + "id": "gujwtvhptykq", + "initParams": {}, + "dataMapping": { + "key7400": "ijkjfvoswni" + } + } + } + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index c3e0401c048d..9deec71732a0 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1070,13 +1070,13 @@ } } }, - "/evaluations/runs/{name}/results": { + "/evaluations/runs/{id}/results": { "get": { "operationId": "Evaluations_GetEvaluationResults", "description": "Retrieves detailed row-by-row evaluation results with pagination support. This endpoint provides access to individual evaluation outcomes, scores, and reasoning for each data row processed.", "parameters": [ { - "name": "name", + "name": "id", "in": "path", "description": "Unique identifier of the evaluation whose results are being retrieved", "required": true, @@ -1167,7 +1167,12 @@ } } }, - "deprecated": true + "deprecated": true, + "x-ms-examples": { + "Evaluations_CreateDeprecated_MaximumSet": { + "$ref": "./examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json" + } + } } }, "/evaluations/runs:runAgent": { @@ -1208,7 +1213,12 @@ } } }, - "deprecated": true + "deprecated": true, + "x-ms-examples": { + "Evaluations_CreateAgentEvaluationDeprecated_MaximumSet": { + "$ref": "./examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json" + } + } } }, "/evaluations/runs:runBatch": { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json new file mode 100644 index 000000000000..6a26a52df99c --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json @@ -0,0 +1,54 @@ +{ + "title": "Evaluations_CreateAgentEvaluationDeprecated_MaximumSet", + "operationId": "Evaluations_CreateAgentEvaluationDeprecated", + "parameters": { + "api-version": "2025-05-15-preview", + "evaluation": { + "runId": "ecfcffqsrdhpecocrufffiqz", + "threadId": "cxjjsyhkeezgnaqqyerrdrbbth", + "evaluators": { + "key2653": { + "id": "gujwtvhptykq", + "initParams": {}, + "dataMapping": { + "key7400": "ijkjfvoswni" + } + } + }, + "samplingConfiguration": { + "name": "tj", + "samplingPercent": 7, + "maxRequestRate": 8 + }, + "redactionConfiguration": { + "redactScoreProperties": true + }, + "appInsightsConnectionString": "dvcnrcwar" + } + }, + "responses": { + "201": { + "body": { + "id": "gji", + "status": "ozywrhiasll", + "error": "stcjbhxgmqvjqwzmbvaa", + "result": [ + { + "evaluator": "upnt", + "evaluatorId": "upnt", + "score": 13, + "status": "eoxw", + "reason": "kfrjetziuketgioobeydlugonzfxo", + "version": "lfqwtzuktnbmcxxwkghmlpdtuu", + "threadId": "sldwqklyuxeayfmxpbl", + "runId": "lauehlf", + "error": "lrutmshl", + "additionalDetails": { + "key3590": "vhhneubavtiklsjcdexij" + } + } + ] + } + } + } +} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json new file mode 100644 index 000000000000..e40f40708d81 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json @@ -0,0 +1,57 @@ +{ + "title": "Evaluations_CreateDeprecated_MaximumSet", + "operationId": "Evaluations_CreateDeprecated", + "parameters": { + "api-version": "2025-07-31-preview", + "evaluation": { + "data": { + "type": "InputData" + }, + "displayName": "ymjxkodggqrhjnmusta", + "description": "cuviaxytqckkybxwc", + "tags": { + "key30": "nqrdpedyhkjzgrbtqhxbmkdmlpdf" + }, + "properties": { + "key9192": "e" + }, + "evaluators": { + "key2735": { + "id": "gujwtvhptykq", + "initParams": {}, + "dataMapping": { + "key7400": "ijkjfvoswni" + } + } + } + } + }, + "responses": { + "201": { + "body": { + "id": "aarhggbojnh", + "data": { + "type": "InputData" + }, + "displayName": "ymjxkodggqrhjnmusta", + "description": "cuviaxytqckkybxwc", + "status": "uhejnohmiiogajkx", + "tags": { + "key30": "nqrdpedyhkjzgrbtqhxbmkdmlpdf" + }, + "properties": { + "key9192": "e" + }, + "evaluators": { + "key2735": { + "id": "gujwtvhptykq", + "initParams": {}, + "dataMapping": { + "key7400": "ijkjfvoswni" + } + } + } + } + } + } +} From 6648074875191b1bd82ca653b5b82c6a2899ba5d Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 10:31:54 -0700 Subject: [PATCH 32/50] EvaluationMessages -> EvaluatorMessages --- .../Azure.AI.Projects/evaluations/models.tsp | 2 +- .../2025-07-31-preview/azure-ai-projects.json | 142 +++++++++--------- 2 files changed, 72 insertions(+), 72 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index b125c3931717..d94ac312b22f 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -137,7 +137,7 @@ model QueryResponseInlineMessages extends InlineData { @added(Versions.v2025_07_31_preview) @doc("Multi-turn conversations for evaluating dialogue systems and context awareness.") -model EvaluationMessages extends InlineData { +model EvaluatorMessages extends InlineData { @doc("Specifies evaluation messages format") dataFormat: InlineDataFormat.evaluationMessagesFormat; diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 9deec71732a0..1c9e804d3727 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3054,77 +3054,6 @@ ] } }, - "EvaluationMessages": { - "type": "object", - "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", - "properties": { - "query": { - "type": "array", - "description": "Array of messages representing representing queries", - "items": { - "$ref": "#/definitions/Message" - } - }, - "response": { - "type": "array", - "description": "Array of messages representing responses", - "items": { - "$ref": "#/definitions/Message" - } - }, - "toolDefinitions": { - "type": "array", - "description": "Array of tool definitions that are used in the conversation", - "items": { - "$ref": "#/definitions/AgentToolDefinition" - } - } - }, - "required": [ - "query", - "response", - "toolDefinitions" - ], - "allOf": [ - { - "$ref": "#/definitions/InlineData" - } - ], - "x-ms-discriminator-value": "evaluationMessagesFormat" - }, - "EvaluationMessagesUpdate": { - "type": "object", - "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", - "properties": { - "query": { - "type": "array", - "description": "Array of messages representing representing queries", - "items": { - "$ref": "#/definitions/Message" - } - }, - "response": { - "type": "array", - "description": "Array of messages representing responses", - "items": { - "$ref": "#/definitions/Message" - } - }, - "toolDefinitions": { - "type": "array", - "description": "Array of tool definitions that are used in the conversation", - "items": { - "$ref": "#/definitions/AgentToolDefinition" - } - } - }, - "allOf": [ - { - "$ref": "#/definitions/InlineDataUpdate" - } - ], - "x-ms-discriminator-value": "evaluationMessagesFormat" - }, "EvaluationMetric": { "type": "object", "description": "Metric representing the evaluation result for a specific evaluator.", @@ -3502,6 +3431,77 @@ ] } }, + "EvaluatorMessages": { + "type": "object", + "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", + "properties": { + "query": { + "type": "array", + "description": "Array of messages representing representing queries", + "items": { + "$ref": "#/definitions/Message" + } + }, + "response": { + "type": "array", + "description": "Array of messages representing responses", + "items": { + "$ref": "#/definitions/Message" + } + }, + "toolDefinitions": { + "type": "array", + "description": "Array of tool definitions that are used in the conversation", + "items": { + "$ref": "#/definitions/AgentToolDefinition" + } + } + }, + "required": [ + "query", + "response", + "toolDefinitions" + ], + "allOf": [ + { + "$ref": "#/definitions/InlineData" + } + ], + "x-ms-discriminator-value": "evaluationMessagesFormat" + }, + "EvaluatorMessagesUpdate": { + "type": "object", + "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", + "properties": { + "query": { + "type": "array", + "description": "Array of messages representing representing queries", + "items": { + "$ref": "#/definitions/Message" + } + }, + "response": { + "type": "array", + "description": "Array of messages representing responses", + "items": { + "$ref": "#/definitions/Message" + } + }, + "toolDefinitions": { + "type": "array", + "description": "Array of tool definitions that are used in the conversation", + "items": { + "$ref": "#/definitions/AgentToolDefinition" + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/InlineDataUpdate" + } + ], + "x-ms-discriminator-value": "evaluationMessagesFormat" + }, "EvaluatorStatusResult": { "type": "object", "description": "Status of evaluator across all data rows.", From 4db6668585d69d82b425f43241eaaa64bf20b8e2 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 10:54:47 -0700 Subject: [PATCH 33/50] Fix model for labels --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 6 +++--- .../preview/2025-07-31-preview/azure-ai-projects.json | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index d94ac312b22f..894800e94da6 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -628,7 +628,7 @@ model EvaluationResultMetadata { evaluatorId: string; @doc("Desired direction for the evaluation score.") - desiredDirection: EvaluatorDesirableDirection; + desiredDirection?: EvaluatorDesirableDirection; @doc("Threshold value that this score is compared against to determine the evaluation outcome") threshold?: float32; @@ -668,10 +668,10 @@ model EvaluationMetric { labels?: string[]; @doc("Outcome of the evaluation") - outcome: EvaluationResultOutcome; + outcome?: EvaluationResultOutcome; @doc("Reasoning or explanation provided by the evaluator for the score assigned to this data row") - reasoning?: string; + reasoning: string; @doc("Error message describing why the evaluation failed for this data row, if applicable") error?: string; diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 1c9e804d3727..aa271b90e16d 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3103,7 +3103,7 @@ "required": [ "name", "evaluatorName", - "outcome", + "reasoning", "metadata" ] }, @@ -3190,7 +3190,6 @@ }, "required": [ "evaluatorId", - "desiredDirection", "type" ] }, From c20ba2efd93af4b2c53db0b3659089f64317b527 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 12:06:11 -0700 Subject: [PATCH 34/50] Fixed examples for deprecated endpoints --- ...ntEvaluationDeprecated_MaximumSet_Gen.json | 54 ------ ...ntEvaluationDeprecated_MinimumSet_Gen.json | 20 +++ ...tions_CreateDeprecated_MaximumSet_Gen.json | 166 ++++++++++++++---- .../2025-07-31-preview/azure-ai-projects.json | 6 +- ...ntEvaluationDeprecated_MaximumSet_Gen.json | 54 ------ ...ntEvaluationDeprecated_MinimumSet_Gen.json | 20 +++ ...tions_CreateDeprecated_MaximumSet_Gen.json | 166 ++++++++++++++---- 7 files changed, 313 insertions(+), 173 deletions(-) delete mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json create mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json delete mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json create mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json deleted file mode 100644 index 6a26a52df99c..000000000000 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "title": "Evaluations_CreateAgentEvaluationDeprecated_MaximumSet", - "operationId": "Evaluations_CreateAgentEvaluationDeprecated", - "parameters": { - "api-version": "2025-05-15-preview", - "evaluation": { - "runId": "ecfcffqsrdhpecocrufffiqz", - "threadId": "cxjjsyhkeezgnaqqyerrdrbbth", - "evaluators": { - "key2653": { - "id": "gujwtvhptykq", - "initParams": {}, - "dataMapping": { - "key7400": "ijkjfvoswni" - } - } - }, - "samplingConfiguration": { - "name": "tj", - "samplingPercent": 7, - "maxRequestRate": 8 - }, - "redactionConfiguration": { - "redactScoreProperties": true - }, - "appInsightsConnectionString": "dvcnrcwar" - } - }, - "responses": { - "201": { - "body": { - "id": "gji", - "status": "ozywrhiasll", - "error": "stcjbhxgmqvjqwzmbvaa", - "result": [ - { - "evaluator": "upnt", - "evaluatorId": "upnt", - "score": 13, - "status": "eoxw", - "reason": "kfrjetziuketgioobeydlugonzfxo", - "version": "lfqwtzuktnbmcxxwkghmlpdtuu", - "threadId": "sldwqklyuxeayfmxpbl", - "runId": "lauehlf", - "error": "lrutmshl", - "additionalDetails": { - "key3590": "vhhneubavtiklsjcdexij" - } - } - ] - } - } - } -} diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json new file mode 100644 index 000000000000..659d92f73bf9 --- /dev/null +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json @@ -0,0 +1,20 @@ +{ + "title": "Evaluations_CreateAgentEvaluationDeprecated", + "operationId": "Evaluations_CreateAgentEvaluationDeprecated", + "parameters": { + "api-version": "2025-07-31-preview", + "evaluation": { + "runId": "rksfkyiajbqcijlxbuiztjsbf", + "evaluators": {}, + "appInsightsConnectionString": "cuskxsdv" + } + }, + "responses": { + "201": { + "body": { + "id": "giui", + "status": "eqyr" + } + } + } +} \ No newline at end of file diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json index e40f40708d81..2f884540cd68 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json @@ -1,57 +1,161 @@ { - "title": "Evaluations_CreateDeprecated_MaximumSet", + "title": "Evaluations_CreateDeprecated", "operationId": "Evaluations_CreateDeprecated", "parameters": { "api-version": "2025-07-31-preview", "evaluation": { - "data": { - "type": "InputData" + "displayName": "Customer Satisfaction Evaluation", + "dataSource": { + "type": "inlineData" }, - "displayName": "ymjxkodggqrhjnmusta", - "description": "cuviaxytqckkybxwc", + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": {}, + "dataMapping": {}, + "name": "tuqexrguotmvzroscysjlomwanpsd" + } + ], + "resultSettings": { + "additionalDestinations": [ + { + "type": "EvaluationDestinationConfiguration", + "connectionName": "zcplllhwtsnopjvutbawpxksdkd", + "state": "NotStarted" + } + ] + }, + "description": "psdgsqjr", + "state": "Succeeded", "tags": { - "key30": "nqrdpedyhkjzgrbtqhxbmkdmlpdf" + "key2067": "rmligdgmzhycryitns", + "key9064": "gerzgbsstbzpewwayymt" }, "properties": { - "key9192": "e" + "key704": "urj", + "key2302": "hoqrucqsjwjgmzramgvtstyvkfck" }, - "evaluators": { - "key2735": { - "id": "gujwtvhptykq", - "initParams": {}, - "dataMapping": { - "key7400": "ijkjfvoswni" + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } } - } - } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2, + "max": 4, + "average": 3, + "standardDeviation": 1, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": {} + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3, + "type": "continuous" + }, + "additionalDetails": {} + } + ] + }, + "resultDatasetId": "iuklrvrubdrb" } }, "responses": { "201": { "body": { - "id": "aarhggbojnh", - "data": { - "type": "InputData" + "displayName": "Customer Satisfaction Evaluation", + "dataSource": { + "type": "inlineData" + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": {}, + "dataMapping": {}, + "name": "tuqexrguotmvzroscysjlomwanpsd" + } + ], + "id": "customer-satisfaction-eval-001", + "resultSettings": { + "additionalDestinations": [ + { + "type": "EvaluationDestinationConfiguration", + "connectionName": "zcplllhwtsnopjvutbawpxksdkd", + "state": "NotStarted" + } + ] }, - "displayName": "ymjxkodggqrhjnmusta", - "description": "cuviaxytqckkybxwc", - "status": "uhejnohmiiogajkx", + "description": "psdgsqjr", + "state": "Succeeded", "tags": { - "key30": "nqrdpedyhkjzgrbtqhxbmkdmlpdf" + "key2067": "rmligdgmzhycryitns", + "key9064": "gerzgbsstbzpewwayymt" }, "properties": { - "key9192": "e" + "key704": "urj", + "key2302": "hoqrucqsjwjgmzramgvtstyvkfck" }, - "evaluators": { - "key2735": { - "id": "gujwtvhptykq", - "initParams": {}, - "dataMapping": { - "key7400": "ijkjfvoswni" + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } } - } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2, + "max": 4, + "average": 3, + "standardDeviation": 1, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": {} + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3, + "type": "continuous" + }, + "additionalDetails": {} + } + ] + }, + "resultDatasetId": "iuklrvrubdrb", + "systemData": { + "key8219": "rwxceksngfwvmsbpkcdap" } } } } -} +} \ No newline at end of file diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index aa271b90e16d..f638797a3a89 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1169,7 +1169,7 @@ }, "deprecated": true, "x-ms-examples": { - "Evaluations_CreateDeprecated_MaximumSet": { + "Evaluations_CreateDeprecated": { "$ref": "./examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json" } } @@ -1215,8 +1215,8 @@ }, "deprecated": true, "x-ms-examples": { - "Evaluations_CreateAgentEvaluationDeprecated_MaximumSet": { - "$ref": "./examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json" + "Evaluations_CreateAgentEvaluationDeprecated": { + "$ref": "./examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json" } } } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json deleted file mode 100644 index 6a26a52df99c..000000000000 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MaximumSet_Gen.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "title": "Evaluations_CreateAgentEvaluationDeprecated_MaximumSet", - "operationId": "Evaluations_CreateAgentEvaluationDeprecated", - "parameters": { - "api-version": "2025-05-15-preview", - "evaluation": { - "runId": "ecfcffqsrdhpecocrufffiqz", - "threadId": "cxjjsyhkeezgnaqqyerrdrbbth", - "evaluators": { - "key2653": { - "id": "gujwtvhptykq", - "initParams": {}, - "dataMapping": { - "key7400": "ijkjfvoswni" - } - } - }, - "samplingConfiguration": { - "name": "tj", - "samplingPercent": 7, - "maxRequestRate": 8 - }, - "redactionConfiguration": { - "redactScoreProperties": true - }, - "appInsightsConnectionString": "dvcnrcwar" - } - }, - "responses": { - "201": { - "body": { - "id": "gji", - "status": "ozywrhiasll", - "error": "stcjbhxgmqvjqwzmbvaa", - "result": [ - { - "evaluator": "upnt", - "evaluatorId": "upnt", - "score": 13, - "status": "eoxw", - "reason": "kfrjetziuketgioobeydlugonzfxo", - "version": "lfqwtzuktnbmcxxwkghmlpdtuu", - "threadId": "sldwqklyuxeayfmxpbl", - "runId": "lauehlf", - "error": "lrutmshl", - "additionalDetails": { - "key3590": "vhhneubavtiklsjcdexij" - } - } - ] - } - } - } -} diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json new file mode 100644 index 000000000000..659d92f73bf9 --- /dev/null +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json @@ -0,0 +1,20 @@ +{ + "title": "Evaluations_CreateAgentEvaluationDeprecated", + "operationId": "Evaluations_CreateAgentEvaluationDeprecated", + "parameters": { + "api-version": "2025-07-31-preview", + "evaluation": { + "runId": "rksfkyiajbqcijlxbuiztjsbf", + "evaluators": {}, + "appInsightsConnectionString": "cuskxsdv" + } + }, + "responses": { + "201": { + "body": { + "id": "giui", + "status": "eqyr" + } + } + } +} \ No newline at end of file diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json index e40f40708d81..2f884540cd68 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json @@ -1,57 +1,161 @@ { - "title": "Evaluations_CreateDeprecated_MaximumSet", + "title": "Evaluations_CreateDeprecated", "operationId": "Evaluations_CreateDeprecated", "parameters": { "api-version": "2025-07-31-preview", "evaluation": { - "data": { - "type": "InputData" + "displayName": "Customer Satisfaction Evaluation", + "dataSource": { + "type": "inlineData" }, - "displayName": "ymjxkodggqrhjnmusta", - "description": "cuviaxytqckkybxwc", + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": {}, + "dataMapping": {}, + "name": "tuqexrguotmvzroscysjlomwanpsd" + } + ], + "resultSettings": { + "additionalDestinations": [ + { + "type": "EvaluationDestinationConfiguration", + "connectionName": "zcplllhwtsnopjvutbawpxksdkd", + "state": "NotStarted" + } + ] + }, + "description": "psdgsqjr", + "state": "Succeeded", "tags": { - "key30": "nqrdpedyhkjzgrbtqhxbmkdmlpdf" + "key2067": "rmligdgmzhycryitns", + "key9064": "gerzgbsstbzpewwayymt" }, "properties": { - "key9192": "e" + "key704": "urj", + "key2302": "hoqrucqsjwjgmzramgvtstyvkfck" }, - "evaluators": { - "key2735": { - "id": "gujwtvhptykq", - "initParams": {}, - "dataMapping": { - "key7400": "ijkjfvoswni" + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } } - } - } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2, + "max": 4, + "average": 3, + "standardDeviation": 1, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": {} + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3, + "type": "continuous" + }, + "additionalDetails": {} + } + ] + }, + "resultDatasetId": "iuklrvrubdrb" } }, "responses": { "201": { "body": { - "id": "aarhggbojnh", - "data": { - "type": "InputData" + "displayName": "Customer Satisfaction Evaluation", + "dataSource": { + "type": "inlineData" + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": {}, + "dataMapping": {}, + "name": "tuqexrguotmvzroscysjlomwanpsd" + } + ], + "id": "customer-satisfaction-eval-001", + "resultSettings": { + "additionalDestinations": [ + { + "type": "EvaluationDestinationConfiguration", + "connectionName": "zcplllhwtsnopjvutbawpxksdkd", + "state": "NotStarted" + } + ] }, - "displayName": "ymjxkodggqrhjnmusta", - "description": "cuviaxytqckkybxwc", - "status": "uhejnohmiiogajkx", + "description": "psdgsqjr", + "state": "Succeeded", "tags": { - "key30": "nqrdpedyhkjzgrbtqhxbmkdmlpdf" + "key2067": "rmligdgmzhycryitns", + "key9064": "gerzgbsstbzpewwayymt" }, "properties": { - "key9192": "e" + "key704": "urj", + "key2302": "hoqrucqsjwjgmzramgvtstyvkfck" }, - "evaluators": { - "key2735": { - "id": "gujwtvhptykq", - "initParams": {}, - "dataMapping": { - "key7400": "ijkjfvoswni" + "summary": { + "evaluatorStatus": [ + { + "name": "relevance", + "id": "azureai://built-in/evaluators/relevance", + "state": "Succeeded", + "usage": { + "inputTokens": 102, + "outputTokens": 50 + } } - } + ], + "metrics": [ + { + "name": "Relevance", + "evaluatorName": "relevance", + "statistics": { + "sampleCount": 2, + "passRate": 0.5, + "min": 2, + "max": 4, + "average": 3, + "standardDeviation": 1, + "confidenceInterval95th": { + "lowerBound": 2.5, + "upperBound": 3.5 + }, + "labelFrequency": {} + }, + "metadata": { + "evaluatorId": "azureai://built-in/evaluators/relevance", + "desiredDirection": "increase", + "threshold": 3, + "type": "continuous" + }, + "additionalDetails": {} + } + ] + }, + "resultDatasetId": "iuklrvrubdrb", + "systemData": { + "key8219": "rwxceksngfwvmsbpkcdap" } } } } -} +} \ No newline at end of file From 38c438c53810fbb892129b28c3656e1c7fb9cc0c Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 12:21:17 -0700 Subject: [PATCH 35/50] Removed deprecated routes --- .../Azure.AI.Projects/evaluations/routes.tsp | 31 ---- ...ntEvaluationDeprecated_MinimumSet_Gen.json | 20 --- ...tions_CreateDeprecated_MaximumSet_Gen.json | 161 ------------------ ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 2 +- .../2025-05-15-preview/azure-ai-projects.json | 1 + .../2025-07-31-preview/azure-ai-projects.json | 92 ---------- ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 2 +- 7 files changed, 3 insertions(+), 306 deletions(-) delete mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json delete mode 100644 specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 9e69cd40347a..873982f5e6bb 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -63,21 +63,6 @@ interface Evaluations { >; #deprecated "Use createBatchEvaluation or createOneEvaluation" - #suppress "@azure-tools/typespec-azure-core/use-standard-operations" - @doc("Creates an evaluation run.") - @route("runs:run") - @added(Versions.v2025_07_31_preview) - @post - @sharedRoute - createDeprecated is Azure.Core.Foundations.Operation< - { - @doc("Evaluation to be run") - @body - evaluation: Evaluation; - }, - ResourceCreatedResponse - >; - #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates an agent evaluation run.") @route("runs:runAgent") @@ -93,22 +78,6 @@ interface Evaluations { ResourceCreatedResponse >; - #deprecated "Use createBatchEvaluation or createOneEvaluation" - #suppress "@azure-tools/typespec-azure-core/use-standard-operations" - @doc("Creates an agent evaluation run.") - @route("runs:runAgent") - @added(Versions.v2025_07_31_preview) // Use createOrReplace instead - @post - @sharedRoute - createAgentEvaluationDeprecated is Azure.Core.Foundations.Operation< - { - @doc("Agent evaluation to be run") - @body - evaluation: AgentEvaluationRequest; - }, - ResourceCreatedResponse - >; - #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates a new evaluation with the specified configuration.") @added(Versions.v2025_07_31_preview) diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json deleted file mode 100644 index 659d92f73bf9..000000000000 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "title": "Evaluations_CreateAgentEvaluationDeprecated", - "operationId": "Evaluations_CreateAgentEvaluationDeprecated", - "parameters": { - "api-version": "2025-07-31-preview", - "evaluation": { - "runId": "rksfkyiajbqcijlxbuiztjsbf", - "evaluators": {}, - "appInsightsConnectionString": "cuskxsdv" - } - }, - "responses": { - "201": { - "body": { - "id": "giui", - "status": "eqyr" - } - } - } -} \ No newline at end of file diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json deleted file mode 100644 index 2f884540cd68..000000000000 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateDeprecated_MaximumSet_Gen.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "title": "Evaluations_CreateDeprecated", - "operationId": "Evaluations_CreateDeprecated", - "parameters": { - "api-version": "2025-07-31-preview", - "evaluation": { - "displayName": "Customer Satisfaction Evaluation", - "dataSource": { - "type": "inlineData" - }, - "evaluators": [ - { - "id": "azureai://built-in/evaluators/relevance", - "initializationParameters": {}, - "dataMapping": {}, - "name": "tuqexrguotmvzroscysjlomwanpsd" - } - ], - "resultSettings": { - "additionalDestinations": [ - { - "type": "EvaluationDestinationConfiguration", - "connectionName": "zcplllhwtsnopjvutbawpxksdkd", - "state": "NotStarted" - } - ] - }, - "description": "psdgsqjr", - "state": "Succeeded", - "tags": { - "key2067": "rmligdgmzhycryitns", - "key9064": "gerzgbsstbzpewwayymt" - }, - "properties": { - "key704": "urj", - "key2302": "hoqrucqsjwjgmzramgvtstyvkfck" - }, - "summary": { - "evaluatorStatus": [ - { - "name": "relevance", - "id": "azureai://built-in/evaluators/relevance", - "state": "Succeeded", - "usage": { - "inputTokens": 102, - "outputTokens": 50 - } - } - ], - "metrics": [ - { - "name": "Relevance", - "evaluatorName": "relevance", - "statistics": { - "sampleCount": 2, - "passRate": 0.5, - "min": 2, - "max": 4, - "average": 3, - "standardDeviation": 1, - "confidenceInterval95th": { - "lowerBound": 2.5, - "upperBound": 3.5 - }, - "labelFrequency": {} - }, - "metadata": { - "evaluatorId": "azureai://built-in/evaluators/relevance", - "desiredDirection": "increase", - "threshold": 3, - "type": "continuous" - }, - "additionalDetails": {} - } - ] - }, - "resultDatasetId": "iuklrvrubdrb" - } - }, - "responses": { - "201": { - "body": { - "displayName": "Customer Satisfaction Evaluation", - "dataSource": { - "type": "inlineData" - }, - "evaluators": [ - { - "id": "azureai://built-in/evaluators/relevance", - "initializationParameters": {}, - "dataMapping": {}, - "name": "tuqexrguotmvzroscysjlomwanpsd" - } - ], - "id": "customer-satisfaction-eval-001", - "resultSettings": { - "additionalDestinations": [ - { - "type": "EvaluationDestinationConfiguration", - "connectionName": "zcplllhwtsnopjvutbawpxksdkd", - "state": "NotStarted" - } - ] - }, - "description": "psdgsqjr", - "state": "Succeeded", - "tags": { - "key2067": "rmligdgmzhycryitns", - "key9064": "gerzgbsstbzpewwayymt" - }, - "properties": { - "key704": "urj", - "key2302": "hoqrucqsjwjgmzramgvtstyvkfck" - }, - "summary": { - "evaluatorStatus": [ - { - "name": "relevance", - "id": "azureai://built-in/evaluators/relevance", - "state": "Succeeded", - "usage": { - "inputTokens": 102, - "outputTokens": 50 - } - } - ], - "metrics": [ - { - "name": "Relevance", - "evaluatorName": "relevance", - "statistics": { - "sampleCount": 2, - "passRate": 0.5, - "min": 2, - "max": 4, - "average": 3, - "standardDeviation": 1, - "confidenceInterval95th": { - "lowerBound": 2.5, - "upperBound": 3.5 - }, - "labelFrequency": {} - }, - "metadata": { - "evaluatorId": "azureai://built-in/evaluators/relevance", - "desiredDirection": "increase", - "threshold": 3, - "type": "continuous" - }, - "additionalDetails": {} - } - ] - }, - "resultDatasetId": "iuklrvrubdrb", - "systemData": { - "key8219": "rwxceksngfwvmsbpkcdap" - } - } - } - } -} \ No newline at end of file diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 6efe81a4b7b0..00f226de0e78 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -37,7 +37,7 @@ } }, "responses": { - "201": { + "200": { "body": { "id": "eval_12345", "state": "Succeeded", diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index cb475d40c91b..ab30764fa440 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1082,6 +1082,7 @@ } } }, + "deprecated": true, "x-ms-examples": { "Evaluations_CreateAgentEvaluation_MaximumSet": { "$ref": "./examples/Evaluations_CreateAgentEvaluation_MaximumSet_Gen.json" diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index f638797a3a89..5afe24391e4d 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1129,98 +1129,6 @@ } } }, - "/evaluations/runs:run": { - "post": { - "operationId": "Evaluations_CreateDeprecated", - "description": "Creates an evaluation run.", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "evaluation", - "in": "body", - "description": "Evaluation to be run", - "required": true, - "schema": { - "$ref": "#/definitions/Evaluation" - } - } - ], - "responses": { - "201": { - "description": "The request has succeeded and a new resource has been created as a result.", - "schema": { - "$ref": "#/definitions/Evaluation" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - }, - "deprecated": true, - "x-ms-examples": { - "Evaluations_CreateDeprecated": { - "$ref": "./examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json" - } - } - } - }, - "/evaluations/runs:runAgent": { - "post": { - "operationId": "Evaluations_CreateAgentEvaluationDeprecated", - "description": "Creates an agent evaluation run.", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "evaluation", - "in": "body", - "description": "Agent evaluation to be run", - "required": true, - "schema": { - "$ref": "#/definitions/AgentEvaluationRequest" - } - } - ], - "responses": { - "201": { - "description": "The request has succeeded and a new resource has been created as a result.", - "schema": { - "$ref": "#/definitions/AgentEvaluation" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - }, - "deprecated": true, - "x-ms-examples": { - "Evaluations_CreateAgentEvaluationDeprecated": { - "$ref": "./examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json" - } - } - } - }, "/evaluations/runs:runBatch": { "post": { "operationId": "Evaluations_CreateBatchEvaluation", diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 6efe81a4b7b0..00f226de0e78 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -37,7 +37,7 @@ } }, "responses": { - "201": { + "200": { "body": { "id": "eval_12345", "state": "Succeeded", From 68ec9d185e800b7d5309a83de48b4b8666bf0b66 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 12:22:12 -0700 Subject: [PATCH 36/50] Removed not needed files --- ...ntEvaluationDeprecated_MinimumSet_Gen.json | 20 --- ...tions_CreateDeprecated_MaximumSet_Gen.json | 161 ------------------ 2 files changed, 181 deletions(-) delete mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json delete mode 100644 specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json deleted file mode 100644 index 659d92f73bf9..000000000000 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateAgentEvaluationDeprecated_MinimumSet_Gen.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "title": "Evaluations_CreateAgentEvaluationDeprecated", - "operationId": "Evaluations_CreateAgentEvaluationDeprecated", - "parameters": { - "api-version": "2025-07-31-preview", - "evaluation": { - "runId": "rksfkyiajbqcijlxbuiztjsbf", - "evaluators": {}, - "appInsightsConnectionString": "cuskxsdv" - } - }, - "responses": { - "201": { - "body": { - "id": "giui", - "status": "eqyr" - } - } - } -} \ No newline at end of file diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json deleted file mode 100644 index 2f884540cd68..000000000000 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateDeprecated_MaximumSet_Gen.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "title": "Evaluations_CreateDeprecated", - "operationId": "Evaluations_CreateDeprecated", - "parameters": { - "api-version": "2025-07-31-preview", - "evaluation": { - "displayName": "Customer Satisfaction Evaluation", - "dataSource": { - "type": "inlineData" - }, - "evaluators": [ - { - "id": "azureai://built-in/evaluators/relevance", - "initializationParameters": {}, - "dataMapping": {}, - "name": "tuqexrguotmvzroscysjlomwanpsd" - } - ], - "resultSettings": { - "additionalDestinations": [ - { - "type": "EvaluationDestinationConfiguration", - "connectionName": "zcplllhwtsnopjvutbawpxksdkd", - "state": "NotStarted" - } - ] - }, - "description": "psdgsqjr", - "state": "Succeeded", - "tags": { - "key2067": "rmligdgmzhycryitns", - "key9064": "gerzgbsstbzpewwayymt" - }, - "properties": { - "key704": "urj", - "key2302": "hoqrucqsjwjgmzramgvtstyvkfck" - }, - "summary": { - "evaluatorStatus": [ - { - "name": "relevance", - "id": "azureai://built-in/evaluators/relevance", - "state": "Succeeded", - "usage": { - "inputTokens": 102, - "outputTokens": 50 - } - } - ], - "metrics": [ - { - "name": "Relevance", - "evaluatorName": "relevance", - "statistics": { - "sampleCount": 2, - "passRate": 0.5, - "min": 2, - "max": 4, - "average": 3, - "standardDeviation": 1, - "confidenceInterval95th": { - "lowerBound": 2.5, - "upperBound": 3.5 - }, - "labelFrequency": {} - }, - "metadata": { - "evaluatorId": "azureai://built-in/evaluators/relevance", - "desiredDirection": "increase", - "threshold": 3, - "type": "continuous" - }, - "additionalDetails": {} - } - ] - }, - "resultDatasetId": "iuklrvrubdrb" - } - }, - "responses": { - "201": { - "body": { - "displayName": "Customer Satisfaction Evaluation", - "dataSource": { - "type": "inlineData" - }, - "evaluators": [ - { - "id": "azureai://built-in/evaluators/relevance", - "initializationParameters": {}, - "dataMapping": {}, - "name": "tuqexrguotmvzroscysjlomwanpsd" - } - ], - "id": "customer-satisfaction-eval-001", - "resultSettings": { - "additionalDestinations": [ - { - "type": "EvaluationDestinationConfiguration", - "connectionName": "zcplllhwtsnopjvutbawpxksdkd", - "state": "NotStarted" - } - ] - }, - "description": "psdgsqjr", - "state": "Succeeded", - "tags": { - "key2067": "rmligdgmzhycryitns", - "key9064": "gerzgbsstbzpewwayymt" - }, - "properties": { - "key704": "urj", - "key2302": "hoqrucqsjwjgmzramgvtstyvkfck" - }, - "summary": { - "evaluatorStatus": [ - { - "name": "relevance", - "id": "azureai://built-in/evaluators/relevance", - "state": "Succeeded", - "usage": { - "inputTokens": 102, - "outputTokens": 50 - } - } - ], - "metrics": [ - { - "name": "Relevance", - "evaluatorName": "relevance", - "statistics": { - "sampleCount": 2, - "passRate": 0.5, - "min": 2, - "max": 4, - "average": 3, - "standardDeviation": 1, - "confidenceInterval95th": { - "lowerBound": 2.5, - "upperBound": 3.5 - }, - "labelFrequency": {} - }, - "metadata": { - "evaluatorId": "azureai://built-in/evaluators/relevance", - "desiredDirection": "increase", - "threshold": 3, - "type": "continuous" - }, - "additionalDetails": {} - } - ] - }, - "resultDatasetId": "iuklrvrubdrb", - "systemData": { - "key8219": "rwxceksngfwvmsbpkcdap" - } - } - } - } -} \ No newline at end of file From 78b4491128257723267ce040e5845929cccae9e6 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 14:24:28 -0700 Subject: [PATCH 37/50] Correct Api-version in tspconfig.yaml --- specification/ai/Azure.AI.Projects/tspconfig.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/ai/Azure.AI.Projects/tspconfig.yaml b/specification/ai/Azure.AI.Projects/tspconfig.yaml index 61b6b222a996..ef563fd41048 100644 --- a/specification/ai/Azure.AI.Projects/tspconfig.yaml +++ b/specification/ai/Azure.AI.Projects/tspconfig.yaml @@ -14,7 +14,7 @@ options: package-dir: "azure-ai-projects" package-name: "{package-dir}" namespace: "azure.ai.projects" - api-version: "2025-07-31-preview" + api-version: "v1" flavor: azure generate-test: true generate-sample: false From 09eaf0cd723f07923f722af03c3bb2cf2c2c202f Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Mon, 4 Aug 2025 19:21:18 -0700 Subject: [PATCH 38/50] Using OneEvaluation Model as request and response --- .../Azure.AI.Projects/evaluations/models.tsp | 8 +++++ .../Azure.AI.Projects/evaluations/routes.tsp | 2 +- ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 33 +++++++++++++++++-- .../2025-07-31-preview/azure-ai-projects.json | 15 ++++++++- ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 33 +++++++++++++++++-- 5 files changed, 83 insertions(+), 8 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 894800e94da6..5e768c244b13 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -340,6 +340,14 @@ model OneEvaluation { @doc("Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.") properties?: Record; + + @doc("Usage statistics for the evaluation") + @visibility(Lifecycle.Read) + usage?: Usage; + + @doc("Evaluation results from all the configured evaluators for this data row.") + @visibility(Lifecycle.Read) + metrics?: EvaluationMetric[]; } @doc("Aggregated metric that summarizes evaluation results across multiple data rows.") diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 873982f5e6bb..64a8fb6cba58 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -111,7 +111,7 @@ interface Evaluations { @body oneEvaluation: OneEvaluation; }, - OkResponse + OkResponse >; @doc("Updates specific properties of an existing evaluation. Supports modification of metadata fields including description, display name, and tags. Note: Core evaluation configuration such as data sources and evaluators cannot be modified after creation.") diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 00f226de0e78..335b1a21f531 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -39,9 +39,36 @@ "responses": { "200": { "body": { - "id": "eval_12345", - "state": "Succeeded", - "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "messages": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.8, + "model": "gpt-4o" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}", + "context": "${data.context}" + } + } + ], "usage": { "inputTokens": 45, "outputTokens": 12 diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 5afe24391e4d..70a207a2c4bf 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1213,7 +1213,7 @@ "200": { "description": "The request has succeeded.", "schema": { - "$ref": "#/definitions/EvaluationResult" + "$ref": "#/definitions/OneEvaluation" } }, "default": { @@ -4116,6 +4116,19 @@ "additionalProperties": { "type": "string" } + }, + "usage": { + "$ref": "#/definitions/Usage", + "description": "Usage statistics for the evaluation", + "readOnly": true + }, + "metrics": { + "type": "array", + "description": "Evaluation results from all the configured evaluators for this data row.", + "items": { + "$ref": "#/definitions/EvaluationMetric" + }, + "readOnly": true } }, "required": [ diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 00f226de0e78..335b1a21f531 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -39,9 +39,36 @@ "responses": { "200": { "body": { - "id": "eval_12345", - "state": "Succeeded", - "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", + "dataSource": { + "type": "inlineData", + "data": { + "dataFormat": "queryResponseMessageFormat", + "messages": [ + { + "query": "Hi", + "response": "hi" + }, + { + "query": "tell me a joke", + "response": "no" + } + ] + } + }, + "evaluators": [ + { + "id": "azureai://built-in/evaluators/relevance", + "initializationParameters": { + "threshold": 0.8, + "model": "gpt-4o" + }, + "dataMapping": { + "query": "${data.query}", + "response": "${data.response}", + "context": "${data.context}" + } + } + ], "usage": { "inputTokens": 45, "outputTokens": 12 From 8da49054770d4b11f8439f497f2bdd2be18ae664 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Tue, 5 Aug 2025 16:53:54 -0700 Subject: [PATCH 39/50] Remove state from individual result & put usage within metrics --- .../Azure.AI.Projects/evaluations/models.tsp | 13 +++--------- ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 8 ++++---- ...s_GetEvaluationResults_MaximumSet_Gen.json | 8 ++++---- .../2025-07-31-preview/azure-ai-projects.json | 20 +++++-------------- ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 8 ++++---- ...s_GetEvaluationResults_MaximumSet_Gen.json | 8 ++++---- 6 files changed, 24 insertions(+), 41 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 5e768c244b13..3a7142d781dc 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -341,10 +341,6 @@ model OneEvaluation { @doc("Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.") properties?: Record; - @doc("Usage statistics for the evaluation") - @visibility(Lifecycle.Read) - usage?: Usage; - @doc("Evaluation results from all the configured evaluators for this data row.") @visibility(Lifecycle.Read) metrics?: EvaluationMetric[]; @@ -687,6 +683,9 @@ model EvaluationMetric { @doc("Metadata about the evaluation result") metadata: EvaluationResultMetadata; + @doc("Token consumption for this metric evaluation") + usage?: Usage; + @doc("Additional metadata about the evaluation result, such as processing details or custom fields") additionalDetails?: Record; } @@ -700,9 +699,6 @@ model EvaluationResult { @visibility(Lifecycle.Read) id: string; - @doc("Current processing state of this data row. Tracks progress from initial processing through completion or failure.") - state: OperationState; - @doc("Error message describing why the evaluation failed for this data row, if applicable") error?: string; @@ -710,9 +706,6 @@ model EvaluationResult { @added(Versions.v2025_07_31_preview) inputDataJson?: string; - @doc("Usage statistics for the evaluation") - usage?: Usage; - @doc("Evaluation results from all the configured evaluators for this data row.") metrics?: EvaluationMetric[]; } diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 335b1a21f531..095a370bd3ba 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -69,10 +69,6 @@ } } ], - "usage": { - "inputTokens": 45, - "outputTokens": 12 - }, "metrics": [ { "name": "Relevance", @@ -88,6 +84,10 @@ "desiredDirection": "increase", "threshold": 3.0, "type": "continuous" + }, + "usage": { + "inputTokens": 45, + "outputTokens": 12 } } ] diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json index c9da3e883b34..b96a91869aa1 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -16,10 +16,6 @@ "id": "result_001", "state": "Succeeded", "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", - "usage": { - "inputTokens": 45, - "outputTokens": 12 - }, "metrics": [ { "name": "Relevance", @@ -35,6 +31,10 @@ "desiredDirection": "increase", "threshold": 3.0, "type": "continuous" + }, + "usage": { + "inputTokens": 45, + "outputTokens": 12 } } ] diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 70a207a2c4bf..5bec8282a948 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3002,6 +3002,10 @@ "$ref": "#/definitions/EvaluationResultMetadata", "description": "Metadata about the evaluation result" }, + "usage": { + "$ref": "#/definitions/Usage", + "description": "Token consumption for this metric evaluation" + }, "additionalDetails": { "type": "object", "description": "Additional metadata about the evaluation result, such as processing details or custom fields", @@ -3045,10 +3049,6 @@ "description": "Unique identifier for this evaluation result row", "readOnly": true }, - "state": { - "$ref": "#/definitions/Azure.Core.Foundations.OperationState", - "description": "Current processing state of this data row. Tracks progress from initial processing through completion or failure." - }, "error": { "type": "string", "description": "Error message describing why the evaluation failed for this data row, if applicable" @@ -3057,10 +3057,6 @@ "type": "string", "description": "Original input data for this row in JSON string format. This preserves the exact data that was evaluated for reference and debugging." }, - "usage": { - "$ref": "#/definitions/Usage", - "description": "Usage statistics for the evaluation" - }, "metrics": { "type": "array", "description": "Evaluation results from all the configured evaluators for this data row.", @@ -3070,8 +3066,7 @@ } }, "required": [ - "id", - "state" + "id" ] }, "EvaluationResultMetadata": { @@ -4117,11 +4112,6 @@ "type": "string" } }, - "usage": { - "$ref": "#/definitions/Usage", - "description": "Usage statistics for the evaluation", - "readOnly": true - }, "metrics": { "type": "array", "description": "Evaluation results from all the configured evaluators for this data row.", diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 335b1a21f531..095a370bd3ba 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -69,10 +69,6 @@ } } ], - "usage": { - "inputTokens": 45, - "outputTokens": 12 - }, "metrics": [ { "name": "Relevance", @@ -88,6 +84,10 @@ "desiredDirection": "increase", "threshold": 3.0, "type": "continuous" + }, + "usage": { + "inputTokens": 45, + "outputTokens": 12 } } ] diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json index c9da3e883b34..b96a91869aa1 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -16,10 +16,6 @@ "id": "result_001", "state": "Succeeded", "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", - "usage": { - "inputTokens": 45, - "outputTokens": 12 - }, "metrics": [ { "name": "Relevance", @@ -35,6 +31,10 @@ "desiredDirection": "increase", "threshold": 3.0, "type": "continuous" + }, + "usage": { + "inputTokens": 45, + "outputTokens": 12 } } ] From 24ef6a72c80f4a3fd117988a2af171b222c34e98 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Tue, 5 Aug 2025 17:10:06 -0700 Subject: [PATCH 40/50] Update image url message type --- .../evaluations/chat_messages.tsp | 13 ++++++++++++- .../2025-07-31-preview/azure-ai-projects.json | 19 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp index 255006b6dfcc..3205703a986e 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp @@ -26,14 +26,25 @@ model TextContent extends AIContent { text: string; } +@doc("Image source") +@added(Versions.v2025_07_31_preview) +model ImageSource { + @doc("A publicly accessible image URL.") + url?: string; + + @doc("Base64-encoded image data.") + b64_json?: string; +} + @doc("Content for image URL messages in AI conversations.") @added(Versions.v2025_07_31_preview) model ImageUrlContent extends AIContent { @doc("The content of the image URL message.") type: "image_url"; + #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." @doc("The URL of the image.") - imageUrl: string; + image_url: string | ImageSource; } @doc("Content for text messages in AI conversations.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 5bec8282a948..6956f70f1897 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3680,17 +3680,30 @@ ], "x-ms-discriminator-value": "FoundryModelInline" }, + "ImageSource": { + "type": "object", + "description": "Image source", + "properties": { + "url": { + "type": "string", + "description": "A publicly accessible image URL." + }, + "b64_json": { + "type": "string", + "description": "Base64-encoded image data." + } + } + }, "ImageUrlContent": { "type": "object", "description": "Content for image URL messages in AI conversations.", "properties": { - "imageUrl": { - "type": "string", + "image_url": { "description": "The URL of the image." } }, "required": [ - "imageUrl" + "image_url" ], "allOf": [ { From a2f8ff6ab3d99b2ab7b2c5557eca2ffcb315c074 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 6 Aug 2025 09:50:15 -0700 Subject: [PATCH 41/50] Removing request from response payload for OneEvaluation --- .../Azure.AI.Projects/evaluations/models.tsp | 7 +++-- .../Azure.AI.Projects/evaluations/routes.tsp | 2 +- ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 30 ------------------- .../2025-07-31-preview/azure-ai-projects.json | 23 ++++++++------ ...ns_CreateOneEvaluation_MaximumSet_Gen.json | 30 ------------------- 5 files changed, 20 insertions(+), 72 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 3a7142d781dc..9e7e121282e4 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -340,9 +340,12 @@ model OneEvaluation { @doc("Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.") properties?: Record; +} - @doc("Evaluation results from all the configured evaluators for this data row.") - @visibility(Lifecycle.Read) +@doc("One Evaluation Result") +@added(Versions.v2025_07_31_preview) +model OneEvaluationResult { + @doc("Evaluation results from all the configured evaluators.") metrics?: EvaluationMetric[]; } diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 64a8fb6cba58..4227cb0517f4 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -111,7 +111,7 @@ interface Evaluations { @body oneEvaluation: OneEvaluation; }, - OkResponse + OkResponse >; @doc("Updates specific properties of an existing evaluation. Supports modification of metadata fields including description, display name, and tags. Note: Core evaluation configuration such as data sources and evaluators cannot be modified after creation.") diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 095a370bd3ba..9a453d091a67 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -39,36 +39,6 @@ "responses": { "200": { "body": { - "dataSource": { - "type": "inlineData", - "data": { - "dataFormat": "queryResponseMessageFormat", - "messages": [ - { - "query": "Hi", - "response": "hi" - }, - { - "query": "tell me a joke", - "response": "no" - } - ] - } - }, - "evaluators": [ - { - "id": "azureai://built-in/evaluators/relevance", - "initializationParameters": { - "threshold": 0.8, - "model": "gpt-4o" - }, - "dataMapping": { - "query": "${data.query}", - "response": "${data.response}", - "context": "${data.context}" - } - } - ], "metrics": [ { "name": "Relevance", diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 6956f70f1897..225ea9340602 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1213,7 +1213,7 @@ "200": { "description": "The request has succeeded.", "schema": { - "$ref": "#/definitions/OneEvaluation" + "$ref": "#/definitions/OneEvaluationResult" } }, "default": { @@ -4124,14 +4124,6 @@ "additionalProperties": { "type": "string" } - }, - "metrics": { - "type": "array", - "description": "Evaluation results from all the configured evaluators for this data row.", - "items": { - "$ref": "#/definitions/EvaluationMetric" - }, - "readOnly": true } }, "required": [ @@ -4139,6 +4131,19 @@ "evaluators" ] }, + "OneEvaluationResult": { + "type": "object", + "description": "One Evaluation Result", + "properties": { + "metrics": { + "type": "array", + "description": "Evaluation results from all the configured evaluators.", + "items": { + "$ref": "#/definitions/EvaluationMetric" + } + } + } + }, "PagedConnection": { "type": "object", "description": "Paged collection of Connection items", diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json index 095a370bd3ba..9a453d091a67 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateOneEvaluation_MaximumSet_Gen.json @@ -39,36 +39,6 @@ "responses": { "200": { "body": { - "dataSource": { - "type": "inlineData", - "data": { - "dataFormat": "queryResponseMessageFormat", - "messages": [ - { - "query": "Hi", - "response": "hi" - }, - { - "query": "tell me a joke", - "response": "no" - } - ] - } - }, - "evaluators": [ - { - "id": "azureai://built-in/evaluators/relevance", - "initializationParameters": { - "threshold": 0.8, - "model": "gpt-4o" - }, - "dataMapping": { - "query": "${data.query}", - "response": "${data.response}", - "context": "${data.context}" - } - } - ], "metrics": [ { "name": "Relevance", From f9438995ae9aee989be6dccd9c22736256992f16 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 6 Aug 2025 10:41:12 -0700 Subject: [PATCH 42/50] Fix example --- .../Evaluations_GetEvaluationResults_MaximumSet_Gen.json | 1 - .../Evaluations_GetEvaluationResults_MaximumSet_Gen.json | 1 - 2 files changed, 2 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json index b96a91869aa1..cb3609f78fab 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -14,7 +14,6 @@ "value": [ { "id": "result_001", - "state": "Succeeded", "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", "metrics": [ { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json index b96a91869aa1..cb3609f78fab 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json @@ -14,7 +14,6 @@ "value": [ { "id": "result_001", - "state": "Succeeded", "inputDataJson": "{\"query\":\"What is the capital of France?\",\"response\":\"The capital of France is Paris.\",\"context\":\"France is a country in Western Europe with several major cities including Paris, Lyon, and Marseille.\"}", "metrics": [ { From 5cb2bf70479d4f2e7cce7a237deba84dd09fe28d Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 6 Aug 2025 14:04:09 -0700 Subject: [PATCH 43/50] Mark API is deprecated --- specification/ai/Azure.AI.Projects/evaluations/routes.tsp | 4 ++-- .../preview/2025-05-15-preview/azure-ai-projects.json | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp index 4227cb0517f4..5451fc69d5af 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/routes.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/routes.tsp @@ -46,7 +46,7 @@ interface Evaluations { >; // create is deprecated from version v2025_07_31_preview onwards. - // so it has 2 entries here (create and createDeprecated) + #deprecated "Use /evaluations/runs:runBatch or /evaluations/runs:runSingle instead." #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates an evaluation run.") @route("runs:run") @@ -62,7 +62,7 @@ interface Evaluations { ResourceCreatedResponse >; - #deprecated "Use createBatchEvaluation or createOneEvaluation" + #deprecated "Use /evaluations/runs:runBatch or /evaluations/runs:runSingle instead." #suppress "@azure-tools/typespec-azure-core/use-standard-operations" @doc("Creates an agent evaluation run.") @route("runs:runAgent") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json index ab30764fa440..59cf306779bd 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects.json @@ -1034,6 +1034,7 @@ } } }, + "deprecated": true, "x-ms-examples": { "Evaluations_Create_MaximumSet": { "$ref": "./examples/Evaluations_Create_MaximumSet_Gen.json" From 39d606f724283293f53d021be6ee5fdfbbdbdc8a Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 6 Aug 2025 14:25:04 -0700 Subject: [PATCH 44/50] Remove b64_json --- .../ai/Azure.AI.Projects/evaluations/chat_messages.tsp | 3 --- .../preview/2025-07-31-preview/azure-ai-projects.json | 4 ---- 2 files changed, 7 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp index 3205703a986e..e087259b91d1 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp @@ -31,9 +31,6 @@ model TextContent extends AIContent { model ImageSource { @doc("A publicly accessible image URL.") url?: string; - - @doc("Base64-encoded image data.") - b64_json?: string; } @doc("Content for image URL messages in AI conversations.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 225ea9340602..5ce1190e78ec 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3687,10 +3687,6 @@ "url": { "type": "string", "description": "A publicly accessible image URL." - }, - "b64_json": { - "type": "string", - "description": "Base64-encoded image data." } } }, From 1e639af6b67f3f7fdc495d846aec6775866852af Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Wed, 6 Aug 2025 14:35:41 -0700 Subject: [PATCH 45/50] Fix image_url --- .../ai/Azure.AI.Projects/evaluations/chat_messages.tsp | 3 +-- .../preview/2025-07-31-preview/azure-ai-projects.json | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp index e087259b91d1..655288fbaae5 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/chat_messages.tsp @@ -39,9 +39,8 @@ model ImageUrlContent extends AIContent { @doc("The content of the image URL message.") type: "image_url"; - #suppress "@azure-tools/typespec-autorest/union-unsupported" "External API shape is defined in OpenAPI 3.0 as oneOf." @doc("The URL of the image.") - image_url: string | ImageSource; + image_url: ImageSource; } @doc("Content for text messages in AI conversations.") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 5ce1190e78ec..62fb68320798 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3695,6 +3695,7 @@ "description": "Content for image URL messages in AI conversations.", "properties": { "image_url": { + "$ref": "#/definitions/ImageSource", "description": "The URL of the image." } }, From 84a4673894d03ad484f6cc0f345426feafaa7937 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 8 Aug 2025 10:28:03 -0700 Subject: [PATCH 46/50] Adding error field as well in evaluator status --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 3 +++ .../preview/2025-07-31-preview/azure-ai-projects.json | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 9e7e121282e4..990de4faf113 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -271,6 +271,9 @@ model EvaluatorStatusResult { @doc("State of the evaluation for this evaluator.") state: OperationState; + @doc("Error message if the evaluator failed to process the data rows.") + error?: string; + @doc("Usage statistics for the evaluation") usage: Usage; } diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 62fb68320798..97955d1e92ec 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3420,6 +3420,10 @@ "$ref": "#/definitions/Azure.Core.Foundations.OperationState", "description": "State of the evaluation for this evaluator." }, + "error": { + "type": "string", + "description": "Error message if the evaluator failed to process the data rows." + }, "usage": { "$ref": "#/definitions/Usage", "description": "Usage statistics for the evaluation" From fe1366bcd7a40496df462970d4d9c29ef94ad2b9 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 8 Aug 2025 10:58:17 -0700 Subject: [PATCH 47/50] Fix error message --- specification/ai/Azure.AI.Projects/evaluations/models.tsp | 2 +- .../preview/2025-07-31-preview/azure-ai-projects.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 990de4faf113..9a111f090d3e 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -271,7 +271,7 @@ model EvaluatorStatusResult { @doc("State of the evaluation for this evaluator.") state: OperationState; - @doc("Error message if the evaluator failed to process the data rows.") + @doc("Error message if the evaluator failed to process.") error?: string; @doc("Usage statistics for the evaluation") diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 97955d1e92ec..c47d17cfa84c 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3422,7 +3422,7 @@ }, "error": { "type": "string", - "description": "Error message if the evaluator failed to process the data rows." + "description": "Error message if the evaluator failed to process." }, "usage": { "$ref": "#/definitions/Usage", From 12098d261cf91fbe57d56d1d21bbab3dbdc27034 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 8 Aug 2025 11:37:20 -0700 Subject: [PATCH 48/50] merge latest + fix tsp validation --- .../preview/2025-07-31-preview/azure-ai-projects.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index c47d17cfa84c..53627c4b8459 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -1123,9 +1123,6 @@ "Evaluations_GetEvaluationResults": { "$ref": "./examples/Evaluations_GetEvaluationResults_MaximumSet_Gen.json" } - }, - "x-ms-pageable": { - "nextLinkName": "nextLink" } } }, From c915452a138becc62ae7131ff5e5d246c6b52c08 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 8 Aug 2025 15:39:16 -0700 Subject: [PATCH 49/50] Fixed inline source to not return user data --- .../Azure.AI.Projects/evaluations/models.tsp | 5 + ..._CreateBatchEvaluation_MaximumSet_Gen.json | 14 +-- .../Evaluations_List_MaximumSet_Gen.json | 14 +-- .../Evaluations_Update_MaximumSet_Gen.json | 14 +-- .../2025-07-31-preview/azure-ai-projects.json | 99 ++----------------- ..._CreateBatchEvaluation_MaximumSet_Gen.json | 14 +-- .../Evaluations_List_MaximumSet_Gen.json | 14 +-- .../Evaluations_Update_MaximumSet_Gen.json | 14 +-- 8 files changed, 20 insertions(+), 168 deletions(-) diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 9a111f090d3e..425bb14ef624 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -171,7 +171,12 @@ model InlineDataSource extends EvaluationDataSource { id?: string; @doc("Inline data structured according to the specified format") + @visibility(Lifecycle.Create) data: InlineData; + + @doc("Dataset id for the uploaded inline data.") + @visibility(Lifecycle.Read) + inlineDatasetId?: DatasetId; } @doc("Data source that uses a dataset registered and stored in AI Foundry workspace. This is the recommended approach for large datasets or reusable evaluation data.") diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json index 794dc22153bd..58b808ca5003 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json @@ -47,19 +47,7 @@ "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", "dataSource": { "type": "inlineData", - "data": { - "dataFormat": "queryResponseMessageFormat", - "messages": [ - { - "query": "Hi", - "response": "hi" - }, - { - "query": "tell me a joke", - "response": "no" - } - ] - } + "inlineDatasetId": "dataset://some-inline-dataset-id" }, "evaluators": [ { diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json index 62503866f288..79012b89754f 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_List_MaximumSet_Gen.json @@ -17,19 +17,7 @@ "state": "Succeeded", "dataSource": { "type": "inlineData", - "data": { - "dataFormat": "queryResponseMessageFormat", - "messages": [ - { - "query": "Hi", - "response": "hi" - }, - { - "query": "tell me a joke", - "response": "no" - } - ] - } + "inlineDatasetId": "dataset://some-dataset-id" }, "evaluators": [ { diff --git a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json index 0542a9d2f8d7..8ff3275960a0 100644 --- a/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json +++ b/specification/ai/Azure.AI.Projects/examples/2025-07-31-preview/Evaluations_Update_MaximumSet_Gen.json @@ -28,19 +28,7 @@ "state": "Succeeded", "dataSource": { "type": "inlineData", - "data": { - "dataFormat": "queryResponseMessageFormat", - "messages": [ - { - "query": "Hi", - "response": "hi" - }, - { - "query": "tell me a joke", - "response": "no" - } - ] - } + "inlineDatasetId": "dataset://some-data-set-id" }, "evaluators": [ { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json index 53627c4b8459..3f7d7fc382a0 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/azure-ai-projects.json @@ -3368,39 +3368,6 @@ ], "x-ms-discriminator-value": "evaluationMessagesFormat" }, - "EvaluatorMessagesUpdate": { - "type": "object", - "description": "Multi-turn conversations for evaluating dialogue systems and context awareness.", - "properties": { - "query": { - "type": "array", - "description": "Array of messages representing representing queries", - "items": { - "$ref": "#/definitions/Message" - } - }, - "response": { - "type": "array", - "description": "Array of messages representing responses", - "items": { - "$ref": "#/definitions/Message" - } - }, - "toolDefinitions": { - "type": "array", - "description": "Array of tool definitions that are used in the conversation", - "items": { - "$ref": "#/definitions/AgentToolDefinition" - } - } - }, - "allOf": [ - { - "$ref": "#/definitions/InlineDataUpdate" - } - ], - "x-ms-discriminator-value": "evaluationMessagesFormat" - }, "EvaluatorStatusResult": { "type": "object", "description": "Status of evaluator across all data rows.", @@ -3843,7 +3810,15 @@ }, "data": { "$ref": "#/definitions/InlineData", - "description": "Inline data structured according to the specified format" + "description": "Inline data structured according to the specified format", + "x-ms-mutability": [ + "create" + ] + }, + "inlineDatasetId": { + "$ref": "#/definitions/DatasetId", + "description": "Dataset id for the uploaded inline data.", + "readOnly": true } }, "required": [ @@ -3863,10 +3838,6 @@ "id": { "type": "string", "description": "Optional unique identifier for the inline data source. This can be an agent id or a custom identifier to distinguish between different inline data sources." - }, - "data": { - "$ref": "#/definitions/InlineDataUpdate", - "description": "Inline data structured according to the specified format" } }, "allOf": [ @@ -3876,20 +3847,6 @@ ], "x-ms-discriminator-value": "inlineData" }, - "InlineDataUpdate": { - "type": "object", - "description": "Base class for inline evaluation data with format discrimination.", - "properties": { - "dataFormat": { - "$ref": "#/definitions/InlineDataFormat", - "description": "Format of the inline data structure" - } - }, - "discriminator": "dataFormat", - "required": [ - "dataFormat" - ] - }, "InlineJson": { "type": "object", "description": "Custom JSON format for complex evaluation scenarios requiring flexible data structures.", @@ -3912,25 +3869,6 @@ ], "x-ms-discriminator-value": "inlineJsonFormat" }, - "InlineJsonUpdate": { - "type": "object", - "description": "Custom JSON format for complex evaluation scenarios requiring flexible data structures.", - "properties": { - "messages": { - "type": "array", - "description": "Array of JSON strings with custom fields and metadata", - "items": { - "type": "string" - } - } - }, - "allOf": [ - { - "$ref": "#/definitions/InlineDataUpdate" - } - ], - "x-ms-discriminator-value": "inlineJsonFormat" - }, "InputData": { "type": "object", "description": "Abstract data class.", @@ -4413,25 +4351,6 @@ ], "x-ms-discriminator-value": "queryResponseMessageFormat" }, - "QueryResponseInlineMessagesUpdate": { - "type": "object", - "description": "Query-response pairs for evaluating Q&A systems and response accuracy.", - "properties": { - "messages": { - "type": "array", - "description": "Array of query-response pairs with optional context and ground truth", - "items": { - "$ref": "#/definitions/QueryResponseMessage" - } - } - }, - "allOf": [ - { - "$ref": "#/definitions/InlineDataUpdate" - } - ], - "x-ms-discriminator-value": "queryResponseMessageFormat" - }, "QueryResponseMessage": { "type": "object", "description": "Represents a message that contains a query and its corresponding response.", diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json index 794dc22153bd..58b808ca5003 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_CreateBatchEvaluation_MaximumSet_Gen.json @@ -47,19 +47,7 @@ "description": "Comprehensive evaluation of customer satisfaction using multiple evaluators on agent conversation data", "dataSource": { "type": "inlineData", - "data": { - "dataFormat": "queryResponseMessageFormat", - "messages": [ - { - "query": "Hi", - "response": "hi" - }, - { - "query": "tell me a joke", - "response": "no" - } - ] - } + "inlineDatasetId": "dataset://some-inline-dataset-id" }, "evaluators": [ { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json index 62503866f288..79012b89754f 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_List_MaximumSet_Gen.json @@ -17,19 +17,7 @@ "state": "Succeeded", "dataSource": { "type": "inlineData", - "data": { - "dataFormat": "queryResponseMessageFormat", - "messages": [ - { - "query": "Hi", - "response": "hi" - }, - { - "query": "tell me a joke", - "response": "no" - } - ] - } + "inlineDatasetId": "dataset://some-dataset-id" }, "evaluators": [ { diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json index 0542a9d2f8d7..8ff3275960a0 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-07-31-preview/examples/Evaluations_Update_MaximumSet_Gen.json @@ -28,19 +28,7 @@ "state": "Succeeded", "dataSource": { "type": "inlineData", - "data": { - "dataFormat": "queryResponseMessageFormat", - "messages": [ - { - "query": "Hi", - "response": "hi" - }, - { - "query": "tell me a joke", - "response": "no" - } - ] - } + "inlineDatasetId": "dataset://some-data-set-id" }, "evaluators": [ { From f4eb822bed9814b081d662885cf394fab7a0eaf8 Mon Sep 17 00:00:00 2001 From: Ritesh Kumar Sinha Date: Fri, 8 Aug 2025 15:49:06 -0700 Subject: [PATCH 50/50] Using latest version --- specification/ai/Azure.AI.Projects/tspconfig.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/ai/Azure.AI.Projects/tspconfig.yaml b/specification/ai/Azure.AI.Projects/tspconfig.yaml index 7c54fffa3670..61b6b222a996 100644 --- a/specification/ai/Azure.AI.Projects/tspconfig.yaml +++ b/specification/ai/Azure.AI.Projects/tspconfig.yaml @@ -14,7 +14,7 @@ options: package-dir: "azure-ai-projects" package-name: "{package-dir}" namespace: "azure.ai.projects" - api-version: "2025-05-15-preview" + api-version: "2025-07-31-preview" flavor: azure generate-test: true generate-sample: false