Tweak the testing to support matrix tests

Author: APErebus

build/Build.CalamariTesting.cs

@@ -4,19 +4,56 @@ namespace Calamari.Build;
 
 partial class Build
 {
-    [Parameter(Name = "CalamariFlavour")] readonly string? CalamariFlavourToTest;
-    [Parameter(Name = "VSTest_TestCaseFilter")] readonly string? CalamariFlavourTestCaseFilter;
+    [Parameter(Name = "ProjectToTest")] readonly string? CalamariProjectToTest;
+    
+    [PublicAPI]
+    Target LinuxHostTests =>
+        target => target
+            .Executes(async () =>
+                      {
+                          var dotnetPath = await LocateOrInstallDotNetSdk();
+
+                          var project = CalamariProjectToTest ?? "Calamari";
 
+                          var dll = $"TestBinaries/{project}.Tests.dll";
+
+                          CreateTestRun(dll)
+                              .WithDotNetPath(dotnetPath)
+                              .WithFilter("TestCategory != Windows") // Linux is all tests that are non-Windows specific
+                              .Execute();
+                      });
+    
     [PublicAPI]
-    Target TestCalamariFlavourProject =>
+    Target WindowsHostTests =>
         target => target
             .Executes(async () =>
                       {
                           var dotnetPath = await LocateOrInstallDotNetSdk();
 
-                          CreateTestRun($"CalamariTests/Calamari.{CalamariFlavourToTest}.Tests.dll")
+                          var project = CalamariProjectToTest ?? "Calamari";
+
+                          var dll = $"TestBinaries/{project}.Tests.dll";
+
+                          CreateTestRun(dll)
+                              .WithDotNetPath(dotnetPath)
+                              .WithFilter("TestCategory != Linux && TestCategory != MacOs") // Windows is all tests that are non-Linux or MacOS specific
+                              .Execute();
+                      });
+    
+    [PublicAPI]
+    Target Windows2026OrLaterHostTests =>
+        target => target
+            .Executes(async () =>
+                      {
+                          var dotnetPath = await LocateOrInstallDotNetSdk();
+
+                          var project = CalamariProjectToTest ?? "Calamari";
+
+                          var dll = $"TestBinaries/{project}.Tests.dll";
+
+                          CreateTestRun(dll)
                               .WithDotNetPath(dotnetPath)
-                              .WithFilter(CalamariFlavourTestCaseFilter)
+                              .WithFilter("TestCategory == Windows2016OrLater")  // Windows 2016 or later specific tests
                               .Execute();
                       });
 }

build/Build.TestCalamariFlavourProject.cs

@@ -18,6 +18,8 @@
 namespace Calamari.AzureResourceGroup.Tests
 {
     [TestFixture]
+    [Category(TestCategories.Windows2016OrLater)]
+    [RequiresWindowsServer2016OrAbove("This test requires the az cli, which relies on python 3.10, which doesn't run on windows 2012/2012R2")]
     [TestPlatforms(TestPlatforms.Windows)] // NOTE: We should look at having the Azure CLI installed on Linux boxes so that these steps can be tested there, particularly if we're moving cloud to a Ubuntu Default Worker.
     class DeployAzureBicepTemplateCommandFixture
     {
@@ -93,7 +95,6 @@ await armClient.GetResourceGroupResource(ResourceGroupResource.CreateResourceIde
         }
 
         [Test]
-        [RequiresWindowsServer2016OrAbove("This test requires the az cli, which relies on python 3.10, which doesn't run on windows 2012/2012R2")]
         public async Task DeployAzureBicepTemplate_PackageSource()
         {
             await CommandTestBuilder.CreateAsync<DeployAzureBicepTemplateCommand, Program>()
@@ -108,7 +109,6 @@ await CommandTestBuilder.CreateAsync<DeployAzureBicepTemplateCommand, Program>()
         }
 
         [Test]
-        [RequiresWindowsServer2016OrAbove("This test requires the az cli, which relies on python 3.10, which doesn't run on windows 2012/2012R2")]
         public async Task DeployAzureBicepTemplate_GitSource()
         {
             // For the purposes of Bicep templates in Calamari, a template in a Git Repository
@@ -127,7 +127,6 @@ await CommandTestBuilder.CreateAsync<DeployAzureBicepTemplateCommand, Program>()
         }
 
         [Test]
-        [RequiresWindowsServer2016OrAbove("This test requires the az cli, which relies on python 3.10, which doesn't run on windows 2012/2012R2")]
         public async Task DeployAzureBicepTemplate_InlineSource()
         {
             var templateFileContent = File.ReadAllText(Path.Combine(packagePath, "azure_website_template.bicep"));

source/Calamari.AzureResourceGroup.Tests/DeployAzureBicepTemplateCommandFixture.cs

@@ -4,7 +4,7 @@
 
 namespace Calamari.Testing.Requirements;
 
-public class RequiresWindowsServer2016OrAboveAttribute(string reason) : TestAttribute, ITestAction
+public class RequiresWindowsServer2016OrAboveAttribute(string reason) : NUnitAttribute, ITestAction
 {
     public void BeforeTest(ITest testDetails)
     {

source/Calamari.Testing/Requirements/RequiresWindowsServer2016OrAboveAttribute.cs

@@ -0,0 +1,6 @@
+namespace Calamari.Testing;
+
+public static class TestCategories
+{
+    public const string Windows2016OrLater = "Windows2016OrLater";
+}