Improved testing of package test text output

Author: CharliePoole

recipe/build-settings.cake

@@ -230,7 +230,7 @@ public static class BuildSettings
 				FileName = "build-results/NUnitConsole.binlog",
 				Imports = MSBuildBinaryLoggerImports.Embed
 			}
-		}.WithProperty("Version", BuildSettings.PackageVersion)
+		}.WithProperty("Version", PackageVersion)
 	};
 
 	// File Header Checks

recipe/output-checks.cake

@@ -1,5 +1,5 @@
 //////////////////////////////////////////////////////////////////////
-// SYNTAX FOR EXPRESSING A STRING CONSTRAINT
+// STATIC SYNTAX FOR EXPRESSING OUTPUT CHECKS
 //////////////////////////////////////////////////////////////////////
 
 //public static class StringConstraints
@@ -22,6 +22,11 @@
 //	}
 //}
 
+public static OutputContainsCheck Contains(string expectedText, int atleast = 1, int exactly = -1)
+    => new OutputContainsCheck(expectedText, atleast, exactly);
+
+public static OutputDoesNotContain DoesNotContain(string text) => new OutputDoesNotContain(text);
+
 // OutputCheck is used to check content of redirected package test output
 public abstract class OutputCheck
 {
@@ -43,9 +48,9 @@ public abstract class OutputCheck
 	public string Message { get; protected set; }
 }
 
-public class OutputContains : OutputCheck
+public class OutputContainsCheck : OutputCheck
 {
-    public OutputContains(string expectedText, int atleast = 1, int exactly = -1) : base(expectedText, atleast, exactly) { }
+    public OutputContainsCheck(string expectedText, int atleast = 1, int exactly = -1) : base(expectedText, atleast, exactly) { }
 
     public override bool Matches(string output)
     {

recipe/package-checks.cake

@@ -1,23 +1,7 @@
 //////////////////////////////////////////////////////////////////////
-// SYNTAX FOR EXPRESSING CHECKS
+// STATIC SYNTAX FOR EXPRESSING PACKAGE CHECKS
 //////////////////////////////////////////////////////////////////////
 
-public static class Check
-{
-	public static void That(DirectoryPath testDirPath, IList<PackageCheck> checks)
-	{
-		if (checks == null)
-			throw new ArgumentNullException(nameof(checks));
-
-		bool allOK = true;
-
-		foreach (var check in checks)
-			allOK &= check.ApplyTo(testDirPath);
-
-        if (!allOK) throw new Exception("Verification failed!");
-    }
-}
-
 public static FileCheck HasFile(FilePath file) => HasFiles(new[] { file });
 public static FileCheck HasFiles(params FilePath[] files) => new FileCheck(files);
 

recipe/package-definition.cake

@@ -242,7 +242,7 @@ public abstract class PackageDefinition
                 string arguments = $"{packageTest.Arguments} {ExtraTestArguments} --work={testResultDir}";
                 if (CommandLineOptions.TraceLevel.Value != "Off")
                     arguments += $" --trace:{CommandLineOptions.TraceLevel.Value}";
-                bool redirectOutput = packageTest.OutputCheck != null;
+                bool redirectOutput = packageTest.ExpectedOutput != null;
 
                 int rc = runner.RunPackageTest(arguments, redirectOutput);
 

recipe/package-test.cake

@@ -14,7 +14,7 @@ public class PackageTest
     public string Arguments { get; set; }
     public int ExpectedReturnCode { get; set; } = 0;
     public ExpectedResult ExpectedResult { get; set; }
-    public OutputCheck OutputCheck { get; set; }
+    public OutputCheck[] ExpectedOutput { get; set; }
     public ExtensionSpecifier[] ExtensionsNeeded { get; set; } = new ExtensionSpecifier[0];
     public IPackageTestRunner[] TestRunners { get; set; } = new IPackageTestRunner[0];
 

recipe/test-reports.cake

@@ -13,7 +13,7 @@ public class PackageTestReport
 		Runner = runner;
 
 		var expectedResult = test.ExpectedResult;
-		var expectedOutput = test.OutputCheck;
+		var expectedOutput = test.ExpectedOutput;
 
 		if (expectedResult != null)
 		{
@@ -57,10 +57,17 @@ public class PackageTestReport
 		}
 
 		if (expectedOutput != null)
-        {
-            if (!expectedOutput.Matches(runner.Output))
-                Errors.Add(expectedOutput.Message);
-        }
+		{
+			var output = runner.Output;
+			if (output is not null)
+				foreach (var outputCheck in expectedOutput)
+				{
+					if (!outputCheck.Matches(output))
+						Errors.Add(outputCheck.Message);
+				}
+			else
+				Errors.Add("No output was produced");
+		}
     }
 
 	public PackageTestReport(PackageTest test, int rc, IPackageTestRunner runner = null)
@@ -71,11 +78,12 @@ public class PackageTestReport
 
 		if (rc != test.ExpectedReturnCode)
 			Errors.Add($"   Expected: rc = {test.ExpectedReturnCode} But was: {rc}");
-		else if (test.OutputCheck != null)
-		{
-			if (!test.OutputCheck.Matches(runner.Output))
-				Errors.Add(test.OutputCheck.Message);
-		}
+		else if (test.ExpectedOutput != null)
+            foreach (var outputCheck in test.ExpectedOutput)
+            {
+                if (!outputCheck.Matches(runner.Output))
+				Errors.Add(outputCheck.Message);
+			}
 	}
 
 	public PackageTestReport(PackageTest test, Exception ex, IPackageTestRunner runner = null)

recipe/test-runners.cake

@@ -21,7 +21,7 @@ public interface IPackageTestRunner
     string PackageId { get; }
     string Version { get; }
 
-    IEnumerable<string> Output { get; }
+    string Output { get; }
 
     int RunPackageTest(string arguments, bool redirectOutput = false);
 }
@@ -44,7 +44,7 @@ public abstract class TestRunner
 	public string PackageId { get; protected set; }
 	public string Version { get; protected set; }
 
-	public IEnumerable<string> Output { get; protected set; }
+	public string Output { get; private set; }
 
     protected int RunPackageTest(FilePath executablePath, string arguments = null, bool redirectOutput = false)
     {
@@ -90,14 +90,14 @@ public abstract class TestRunner
 		IEnumerable<string> output;
 		// If Redirected Output was not requested, output will be null
 		int rc = Context.StartProcess(executablePath, processSettings, out output);
-		Output = output;
+		Output = output != null ? string.Join("\r\n", output) : null;
 		return rc;
     }
 
 	internal string OutputHandler(string output)
 	{
 		// Ensure that package test output displays and is also re-directed.
-		// If the derive class doesn't need the output, it doesn't retrieve it.
+		// If the derived class doesn't need the output, it doesn't retrieve it.
 		Console.WriteLine(output);
 		return output;
 	}