9.3 release of Microsoft.Extensions.AI.Evaluation libraries (dotnet#6100)

Author: peterwald

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/RelevanceTruthAndCompletenessEvaluator.Prompts.cs

@@ -7,93 +7,7 @@ public partial class RelevanceTruthAndCompletenessEvaluator
 {
     private static class Prompts
     {
-        internal static string BuildEvaluationPrompt(string userQuery, string modelResponse, string history)
-        {
-#pragma warning disable S103 // Lines should not be too long
-            return
-                $$"""
-                Read the History, User Query, and Model Response below and produce your response as a single JSON object.
-                Do not include any other text in your response besides the JSON object.
-
-                The JSON object should have the following format. However, do not include any markdown tags in your
-                response. Your response should start with an open curly brace and end with a closing curly brace for the
-                JSON.
-                ```
-                {
-                    "relevance": 1,
-                    "truth": 1,
-                    "completeness": 1
-                }
-                ```
-
-                -----
-
-                History: {{history}}
-
-                -----
-
-                User Query: {{userQuery}}
-
-                -----
-
-                Model Response: {{modelResponse}}
-
-                -----
-
-                That's the History, User Query, and Model Response you will rate. Now, in 3 Steps, you will evaluate the Model Response on 3 criteria.
-
-                -----
-
-                Step 1: Rate the relevance of the response.
-
-                Regardless of truth of statements, how much of the response is directly related to the request?
-
-                1 = None of the response is at all related
-                2 = A little of the response is directly related, or the response is a little bit related
-                3 = Some of the response is related, or the response is somewhat related
-                4 = Most of the response is related, or the response is mostly related
-                5 = 100% of the response is 100% related to the request
-
-                Record your response as the value of the "relevance" property in the JSON output.
-
-                -----
-
-                Step 2: Rate the truth of the response.
-
-                Read the History, Query, and Model Response again.
-
-                Regardless of relevance, how true are the verifiable statements in the response?
-
-                1 = The entire response is totally false
-                2 = A little of the response is true, or the response is a little bit true
-                3 = Some of the response is true, or the response is somewhat true
-                4 = Most of the response is true, or the response is mostly true
-                5 = 100% of the response is 100% true
-
-                Record your response as the value of the "truth" property in the JSON output.
-
-                -----
-
-                Step 3: Rate the completeness of the response.
-
-                Read the History, Query, and Model Response again.
-
-                Regardless of whether the statements made in the response are true, how many of the points necessary to address the request, does the response contain?
-
-                1 = The response omits all points that are necessary to address the request.
-                2 = The response includes a little of the points that are necessary to address the request.
-                3 = The response includes some of the points that are necessary to address the request.
-                4 = The response includes most of the points that are necessary to address the request.
-                5 = The response includes all points that are necessary to address the request. For explain tasks, nothing is left unexplained. For improve tasks, I looked for all potential improvements, and none were left out. For fix tasks, the response purports to get the user all the way to a fixed state (regardless of whether it actually works). For "do task" responses, it does everything requested.
-
-                Record your response as the value of the "completeness" property in the JSON output.
-
-                -----
-                """;
-#pragma warning restore S103
-        }
-
-        internal static string BuildEvaluationPromptWithReasoning(
+        internal static string BuildEvaluationPrompt(
             string userQuery,
             string modelResponse,
             string history)

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/RelevanceTruthAndCompletenessEvaluator.cs

@@ -23,11 +23,10 @@ namespace Microsoft.Extensions.AI.Evaluation.Quality;
 /// <remarks>
 /// <see cref="RelevanceTruthAndCompletenessEvaluator"/> returns three <see cref="NumericMetric"/>s that contain scores
 /// for 'Relevance', 'Truth' and 'Completeness' respectively. Each score is a number between 1 and 5, with 1 indicating
-/// a poor score, and 5 indicating an excellent score.
+/// a poor score, and 5 indicating an excellent score. Each returned score is also accompanied by a
+/// <see cref="EvaluationMetric.Reason"/> that provides an explanation for the score.
 /// </remarks>
-/// <param name="options">Options for <see cref="RelevanceTruthAndCompletenessEvaluator"/>.</param>
-public sealed partial class RelevanceTruthAndCompletenessEvaluator(
-    RelevanceTruthAndCompletenessEvaluatorOptions? options = null) : ChatConversationEvaluator
+public sealed partial class RelevanceTruthAndCompletenessEvaluator : ChatConversationEvaluator
 {
     /// <summary>
     /// Gets the <see cref="EvaluationMetric.Name"/> of the <see cref="NumericMetric"/> returned by
@@ -61,9 +60,6 @@ public sealed partial class RelevanceTruthAndCompletenessEvaluator(
             ResponseFormat = ChatResponseFormat.Json
         };
 
-    private readonly RelevanceTruthAndCompletenessEvaluatorOptions _options =
-        options ?? RelevanceTruthAndCompletenessEvaluatorOptions.Default;
-
     /// <inheritdoc/>
     protected override EvaluationResult InitializeResult()
     {
@@ -101,17 +97,7 @@ userRequest is not null
 
         string renderedHistory = builder.ToString();
 
-        string prompt =
-            _options.IncludeReasoning
-                ? Prompts.BuildEvaluationPromptWithReasoning(
-                    renderedUserRequest,
-                    renderedModelResponse,
-                    renderedHistory)
-                : Prompts.BuildEvaluationPrompt(
-                    renderedUserRequest,
-                    renderedModelResponse,
-                    renderedHistory);
-
+        string prompt = Prompts.BuildEvaluationPrompt(renderedUserRequest, renderedModelResponse, renderedHistory);
         return prompt;
     }
 
@@ -192,23 +178,23 @@ void UpdateResult(Rating rating)
             relevance.Interpretation = relevance.InterpretScore();
             if (!string.IsNullOrWhiteSpace(rating.RelevanceReasoning))
             {
-                relevance.AddDiagnostic(EvaluationDiagnostic.Informational(rating.RelevanceReasoning!));
+                relevance.Reason = rating.RelevanceReasoning!;
             }
 
             NumericMetric truth = result.Get<NumericMetric>(TruthMetricName);
             truth.Value = rating.Truth;
             truth.Interpretation = truth.InterpretScore();
             if (!string.IsNullOrWhiteSpace(rating.TruthReasoning))
             {
-                truth.AddDiagnostic(EvaluationDiagnostic.Informational(rating.TruthReasoning!));
+                truth.Reason = rating.TruthReasoning!;
             }
 
             NumericMetric completeness = result.Get<NumericMetric>(CompletenessMetricName);
             completeness.Value = rating.Completeness;
             completeness.Interpretation = completeness.InterpretScore();
             if (!string.IsNullOrWhiteSpace(rating.CompletenessReasoning))
             {
-                completeness.AddDiagnostic(EvaluationDiagnostic.Informational(rating.CompletenessReasoning!));
+                completeness.Reason = rating.CompletenessReasoning!;
             }
 
             if (!string.IsNullOrWhiteSpace(rating.Error))

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/RelevanceTruthAndCompletenessEvaluatorOptions.cs

@@ -0,0 +1,58 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Text.Json.Serialization.Metadata;
+using static Microsoft.Extensions.AI.Evaluation.Reporting.Storage.AzureStorageResponseCache;
+
+namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization;
+
+internal static partial class AzureStorageJsonUtilities
+{
+    [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")]
+    internal static class Default
+    {
+        private static JsonSerializerOptions? _options;
+        internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: true);
+        internal static JsonTypeInfo<CacheEntry> CacheEntryTypeInfo => Options.GetTypeInfo<CacheEntry>();
+        internal static JsonTypeInfo<ScenarioRunResult> ScenarioRunResultTypeInfo => Options.GetTypeInfo<ScenarioRunResult>();
+    }
+
+    internal static class Compact
+    {
+        private static JsonSerializerOptions? _options;
+        internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: false);
+        internal static JsonTypeInfo<ScenarioRunResult> ScenarioRunResultTypeInfo => Options.GetTypeInfo<ScenarioRunResult>();
+    }
+
+    private static JsonTypeInfo<T> GetTypeInfo<T>(this JsonSerializerOptions options) => (JsonTypeInfo<T>)options.GetTypeInfo(typeof(T));
+
+    private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeIndented)
+    {
+        var options = new JsonSerializerOptions(JsonContext.Default.Options)
+        {
+            WriteIndented = writeIndented,
+            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        };
+        options.TypeInfoResolverChain.Add(AIJsonUtilities.DefaultOptions.TypeInfoResolver!);
+        options.MakeReadOnly();
+        return options;
+    }
+
+    [JsonSerializable(typeof(ScenarioRunResult))]
+    [JsonSerializable(typeof(CacheEntry))]
+    [JsonSourceGenerationOptions(
+        Converters = [
+            typeof(AzureStorageCamelCaseEnumConverter<EvaluationDiagnosticSeverity>),
+            typeof(AzureStorageCamelCaseEnumConverter<EvaluationRating>),
+            typeof(AzureStorageTimeSpanConverter)
+        ],
+        WriteIndented = true,
+        IgnoreReadOnlyProperties = false,
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+        PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)]
+    private sealed partial class JsonContext : JsonSerializerContext;
+
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs

@@ -45,7 +45,7 @@ public static CacheEntry Read(
             CacheEntry cacheEntry =
                 JsonSerializer.Deserialize(
                     content.Value.Content.ToMemory().Span,
-                    AzureStorageSerializerContext.Default.CacheEntry)
+                    AzureStorageJsonUtilities.Default.CacheEntryTypeInfo)
                 ?? throw new JsonException(
                     string.Format(CultureInfo.CurrentCulture, DeserializationFailedMessage, fileClient.Name));
 
@@ -62,7 +62,7 @@ public static async Task<CacheEntry> ReadAsync(
             CacheEntry cacheEntry =
                 await JsonSerializer.DeserializeAsync(
                     content.Value.Content.ToStream(),
-                    AzureStorageSerializerContext.Default.CacheEntry,
+                    AzureStorageJsonUtilities.Default.CacheEntryTypeInfo,
                     cancellationToken).ConfigureAwait(false)
                 ?? throw new JsonException(
                     string.Format(CultureInfo.CurrentCulture, DeserializationFailedMessage, fileClient.Name));
@@ -76,7 +76,7 @@ public void Write(
         {
             MemoryStream stream = new();
 
-            JsonSerializer.Serialize(stream, this, AzureStorageSerializerContext.Default.CacheEntry);
+            JsonSerializer.Serialize(stream, this, AzureStorageJsonUtilities.Default.CacheEntryTypeInfo);
 
             _ = stream.Seek(0, SeekOrigin.Begin);
             _ = fileClient.Upload(stream, overwrite: true, cancellationToken);
@@ -91,7 +91,7 @@ public async Task WriteAsync(
             await JsonSerializer.SerializeAsync(
                 stream,
                 this,
-                AzureStorageSerializerContext.Default.CacheEntry,
+                AzureStorageJsonUtilities.Default.CacheEntryTypeInfo,
                 cancellationToken).ConfigureAwait(false);
 
             _ = stream.Seek(0, SeekOrigin.Begin);

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageSerializerContext.cs

@@ -119,7 +119,7 @@ public async IAsyncEnumerable<ScenarioRunResult> ReadResultsAsync(
 
             ScenarioRunResult? result = await JsonSerializer.DeserializeAsync(
                 content.Value.Content.ToStream(),
-                AzureStorageSerializerContext.Default.ScenarioRunResult,
+                AzureStorageJsonUtilities.Default.ScenarioRunResultTypeInfo,
                 cancellationToken).ConfigureAwait(false)
                     ?? throw new JsonException(
                         string.Format(CultureInfo.CurrentCulture, DeserializationFailedMessage, fileClient.Name));
@@ -171,7 +171,7 @@ public async ValueTask WriteResultsAsync(
             await JsonSerializer.SerializeAsync(
                 stream,
                 result,
-                AzureStorageSerializerContext.Default.ScenarioRunResult,
+                AzureStorageJsonUtilities.Default.ScenarioRunResultTypeInfo,
                 cancellationToken).ConfigureAwait(false);
 
             _ = stream.Seek(0, SeekOrigin.Begin);

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.CacheEntry.cs

@@ -56,7 +56,7 @@ public async ValueTask WriteReportAsync(
         await JsonSerializer.SerializeAsync(
             stream,
             dataset,
-            SerializerContext.Compact.Dataset,
+            JsonUtilities.Compact.DatasetTypeInfo,
             cancellationToken).ConfigureAwait(false);
 
 #if NET

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs

@@ -45,7 +45,7 @@ public async ValueTask WriteReportAsync(
         await JsonSerializer.SerializeAsync(
             stream,
             dataset,
-            SerializerContext.Default.Dataset,
+            JsonUtilities.Default.DatasetTypeInfo,
             cancellationToken).ConfigureAwait(false);
     }
 }