Add Azure DevOps flaky-history annotations and quarantine awareness

Directory.Packages.props

@@ -59,6 +59,7 @@
     <PackageVersion Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageVersion Include="Microsoft.WindowsAppSDK" Version="1.8.251003001" />
     <PackageVersion Include="OpenTelemetry" Version="1.15.3" />
+    <PackageVersion Include="System.Text.Json" Version="8.0.5" />
     <PackageVersion Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsVersion)" />
   </ItemGroup>
   <ItemGroup Label="Test dependencies">

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs

@@ -6,5 +6,8 @@ namespace Microsoft.Testing.Extensions.Reporting;
 internal static class AzureDevOpsCommandLineOptions
 {
     public const string AzureDevOpsOptionName = "report-azdo";
+    public const string AzureDevOpsDemoteKnownFlaky = "report-azdo-demote-known-flaky";
+    public const string AzureDevOpsFlakyHistory = "report-azdo-flaky-history";
+    public const string AzureDevOpsQuarantineFile = "report-azdo-quarantine-file";
     public const string AzureDevOpsReportSeverity = "report-azdo-severity";
 }

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs

@@ -26,31 +26,61 @@ public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions()
         =>
         [
             new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName, AzureDevOpsResources.OptionDescription, ArgumentArity.Zero, false),
+            new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky, AzureDevOpsResources.DemoteKnownFlakyOptionDescription, ArgumentArity.Zero, false),
+            new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, AzureDevOpsResources.FlakyHistoryOptionDescription, ArgumentArity.ExactlyOne, false),
+            new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile, AzureDevOpsResources.QuarantineFileOptionDescription, ArgumentArity.ExactlyOne, false),
             new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false),
         ];
 
     public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
+        => commandOption.Name switch
+        {
+            AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory => ValidateFlakyHistoryArgumentsAsync(arguments),
+            AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity => ValidateSeverityArgumentsAsync(arguments),
+            _ => ValidationResult.ValidTask,
+        };
+
+    public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
     {
-        if (commandOption.Name == AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity)
+        string? errorMessage = null;
+        if (!commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName))
         {
-            if (!SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase))
+            if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky))
+            {
+                errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresAzureDevOps;
+            }
+            else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory))
+            {
+                errorMessage = AzureDevOpsResources.AzureDevOpsFlakyHistoryRequiresAzureDevOps;
+            }
+            else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile))
+            {
+                errorMessage = AzureDevOpsResources.AzureDevOpsQuarantineFileRequiresAzureDevOps;
+            }
+            else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity))
             {
-                return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0]));
+                errorMessage = AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps;
             }
         }
-
-        return ValidationResult.ValidTask;
-    }
-
-    public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
-    {
-        if (!commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) &&
-            commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity))
+        else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky)
+            && !commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory))
         {
-            // If report-azdo is not set, but report-azdo-severity is set, it's invalid.
-            return ValidationResult.InvalidTask(AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps);
+            errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresFlakyHistory;
         }
 
-        return ValidationResult.ValidTask;
+        return errorMessage is null
+            ? ValidationResult.ValidTask
+            : ValidationResult.InvalidTask(errorMessage);
     }
+
+    private static Task<ValidationResult> ValidateFlakyHistoryArgumentsAsync(string[] arguments)
+        => int.TryParse(arguments[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int days)
+            && days is >= 1 and <= 90
+                ? ValidationResult.ValidTask
+                : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidFlakyHistoryDays, arguments[0]));
+
+    private static Task<ValidationResult> ValidateSeverityArgumentsAsync(string[] arguments)
+        => SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase)
+            ? ValidationResult.ValidTask
+            : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0]));
 }

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs

@@ -4,7 +4,6 @@
 using Microsoft.Testing.Extensions.AzureDevOpsReport;
 using Microsoft.Testing.Extensions.Reporting;
 using Microsoft.Testing.Platform.Builder;
-using Microsoft.Testing.Platform.Extensions;
 using Microsoft.Testing.Platform.Services;
 
 namespace Microsoft.Testing.Extensions;
@@ -20,17 +19,33 @@ public static class AzureDevOpsExtensions
     /// <param name="builder">The test application builder.</param>
     public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
     {
-        var compositeTestSessionAzDoService =
-           new CompositeExtensionFactory<AzureDevOpsReporter>(serviceProvider =>
-               new AzureDevOpsReporter(
-                   serviceProvider.GetCommandLineOptions(),
-                   serviceProvider.GetEnvironment(),
-                   serviceProvider.GetFileSystem(),
-                   serviceProvider.GetOutputDevice(),
-                   serviceProvider.GetLoggerFactory()));
+        builder.TestHost.AddDataConsumer(serviceProvider =>
+            new AzureDevOpsReporter(
+                serviceProvider.GetCommandLineOptions(),
+                serviceProvider.GetEnvironment(),
+                serviceProvider.GetFileSystem(),
+                serviceProvider.GetOutputDevice(),
+                serviceProvider.GetLoggerFactory(),
+                GetOrCreateHistoryService(serviceProvider)));
+        builder.TestHost.AddTestSessionLifetimeHandler(serviceProvider => (AzureDevOpsHistoryService)GetOrCreateHistoryService(serviceProvider));
+        builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider());
+    }
 
-        builder.TestHost.AddDataConsumer(compositeTestSessionAzDoService);
+    private static IAzureDevOpsHistoryService GetOrCreateHistoryService(IServiceProvider serviceProvider)
+    {
+        if (serviceProvider.GetService<IAzureDevOpsHistoryService>() is IAzureDevOpsHistoryService historyService)
+        {
+            return historyService;
+        }
 
-        builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider());
+        historyService = new AzureDevOpsHistoryService(
+            serviceProvider.GetCommandLineOptions(),
+            serviceProvider.GetEnvironment(),
+            serviceProvider.GetClock(),
+            new AzureDevOpsHistoryClient(serviceProvider.GetTask(), serviceProvider.GetClock()),
+            serviceProvider.GetTask(),
+            serviceProvider.GetLoggerFactory());
+        ((ServiceProvider)serviceProvider).AddService(historyService, throwIfSameInstanceExit: false);
+        return historyService;
     }
 }