Enable running example tests massively in parallel via test runner UI and dotnet test

Author: jnm2

tools/ExampleExtractor/ExampleMetadata.cs

@@ -84,4 +84,6 @@ public class ExampleMetadata
 
     [JsonIgnore]
     public string Source => $"{MarkdownFile}:{StartLine}-{EndLine}";
+
+    public override string ToString() => Name;
 }

tools/ExampleTester/ExampleTester.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
@@ -11,6 +11,9 @@
     <PackageReference Include="Microsoft.Build.Locator" Version="1.9.1" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.14.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" Version="4.14.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
+    <PackageReference Include="NUnit" Version="4.3.2" />
+    <PackageReference Include="NUnit3TestAdapter" Version="5.0.0" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>
 

tools/ExampleTester/ExampleTests.cs

@@ -0,0 +1,35 @@
+using NUnit.Framework;
+using Utilities;
+
+[assembly: Parallelizable(ParallelScope.Children)]
+
+namespace ExampleTester;
+
+public static class ExampleTests
+{
+    private static TesterConfiguration TesterConfiguration { get; } = new(FindTmpDirectory());
+    private static StatusCheckLogger StatusCheckLogger { get; } = new("..", "Example tester");
+
+    public static IEnumerable<object[]> LoadExamples() =>
+        from example in GeneratedExample.LoadAllExamples(TesterConfiguration.ExtractedOutputDirectory)
+        select new object[] { example };
+
+    [TestCaseSource(nameof(LoadExamples))]
+    public static async Task ExamplePasses(GeneratedExample example)
+    {
+        if (!await example.Test(TesterConfiguration, StatusCheckLogger))
+            Assert.Fail("There were one or more failures. See the logged output for details.");
+    }
+
+    private static string FindTmpDirectory()
+    {
+        for (string? current = AppDomain.CurrentDomain.BaseDirectory; current != null; current = Path.GetDirectoryName(current))
+        {
+            string testPath = Path.Join(current, "tmp");
+            if (Directory.Exists(testPath))
+                return testPath;
+        }
+
+        throw new InvalidOperationException($"Can't find 'tmp' directory in {AppDomain.CurrentDomain.BaseDirectory} or any parent directories.");
+    }
+}

tools/ExampleTester/GeneratedExample.cs

@@ -26,6 +26,8 @@ private GeneratedExample(string directory)
         Metadata = JsonConvert.DeserializeObject<ExampleMetadata>(metadataJson) ?? throw new ArgumentException($"Invalid (null) metadata in {directory}");
     }
 
+    public override string? ToString() => Metadata.ToString();
+
     internal static List<GeneratedExample> LoadAllExamples(string parentDirectory) =>
         Directory.GetDirectories(parentDirectory).Select(Load).ToList();
 

tools/ExampleTester/TesterConfiguration.cs

@@ -5,12 +5,9 @@ namespace ExampleTester;
 
 public record TesterConfiguration(
     string ExtractedOutputDirectory,
-    bool Quiet,
-    string? SourceFile,
-    string? ExampleName)
-{
-    
-}
+    bool Quiet = false,
+    string? SourceFile = null,
+    string? ExampleName = null);
 
 public class TesterConfigurationBinder : BinderBase<TesterConfiguration>
 {