[tests] Build, run, and clean up missing test suites (#3859)

Context: https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=3196159&view=artifacts

I recently noticed the win-build-test-results artifact that we were
uploading was enormous, even when all tests had passed. This was due to
the fact that we weren't cleaning up after successful tests from the
CodeBehind, CodeGen-MkBundle, and EmbeddedDSOs suites. I've also
updated our result-packaging globbing to exclude NuGet packages from
our test result zips to further reduce unnecessary uploading.

This also identified an issue in our test execution. The three suites
mentioned above are not running in any test stages against an installer.
To address this our build pipeline has been updated to ensure these
assemblies are built and uploaded during the Mac Build job, and then
downloadedd and ran in the Test MSBuild - macOS job.

Author: pjcollins

build-tools/Xamarin.Android.Tools.BootstrapTasks/result-packaging.targets

@@ -26,7 +26,7 @@
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\compatibility\*" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\logcat*" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\*log" />
-    <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\temp\**\*">
+    <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\temp\**\*" Exclude="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\temp\packages\**">
       <SubDirectory>temp\</SubDirectory>
     </_TestResultFiles>
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\TestOutput-*.txt" />

build-tools/automation/azure-pipelines.yaml

@@ -68,6 +68,7 @@ stages:
       inputs:
         version: $(DotNetCoreVersion)
 
+    # Prepare and build everything
     - script: make prepare-update-mono V=1 CONFIGURATION=$(XA.Build.Configuration) PREPARE_CI=1 PREPARE_AUTOPROVISION=1
       displayName: make prepare-update-mono
 
@@ -80,6 +81,25 @@ stages:
     - script: make jenkins V=1 CONFIGURATION=$(XA.Build.Configuration) PREPARE_CI=1 PREPARE_AUTOPROVISION=1
       displayName: make jenkins
 
+    # Build and package test assemblies
+    - task: MSBuild@1
+      displayName: msbuild tests/CodeBehind/UnitTests/CodeBehindUnitTests.csproj
+      inputs:
+        solution: $(System.DefaultWorkingDirectory)/tests/CodeBehind/UnitTests/CodeBehindUnitTests.csproj
+        configuration: $(XA.Build.Configuration)
+
+    - task: MSBuild@1
+      displayName: msbuild tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-UnitTests/Xamarin.Android.MakeBundle-UnitTests.csproj
+      inputs:
+        solution: $(System.DefaultWorkingDirectory)/tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-UnitTests/Xamarin.Android.MakeBundle-UnitTests.csproj
+        configuration: $(XA.Build.Configuration)
+
+    - task: MSBuild@1
+      displayName: msbuild tests/EmbeddedDSOs/EmbeddedDSO-UnitTests/EmbeddedDSO-UnitTests.csproj
+      inputs:
+        solution: $(System.DefaultWorkingDirectory)/tests/EmbeddedDSOs/EmbeddedDSO-UnitTests/EmbeddedDSO-UnitTests.csproj
+        configuration: $(XA.Build.Configuration)
+
     - script: |
         cp -r bin/$(XA.Build.Configuration)/bcl-tests bin/Test$(XA.Build.Configuration)/bcl-tests
         cp bin/Build$(XA.Build.Configuration)/ProfileAssemblies.projitems bin/Test$(XA.Build.Configuration)/bcl-tests/
@@ -91,6 +111,7 @@ stages:
         artifactName: $(TestAssembliesArtifactName)
         targetPath: bin/Test$(XA.Build.Configuration)
 
+    # Create installers
     - template: install-certificates.yml@yaml
       parameters:
         DeveloperIdApplication: $(developer-id-application)
@@ -549,6 +570,24 @@ stages:
         artifactName: $(TestAssembliesArtifactName)
         downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
 
+    - template: yaml-templates/run-nunit-tests.yaml
+      parameters:
+        testRunTitle: CodeBehindUnitTests - macOS
+        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/CodeBehind/CodeBehindUnitTests.dll
+        testResultsFile: TestResult-CodeBehindUnitTests-macOS-$(XA.Build.Configuration).xml
+
+    - template: yaml-templates/run-nunit-tests.yaml
+      parameters:
+        testRunTitle: Xamarin.Android.MakeBundle-UnitTests - macOS
+        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/Xamarin.Android.MakeBundle-UnitTests.dll
+        testResultsFile: TestResult-MakeBundleUnitTests-macOS-$(XA.Build.Configuration).xml
+
+    - template: yaml-templates/run-nunit-tests.yaml
+      parameters:
+        testRunTitle: EmbeddedDSOUnitTests - macOS
+        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/EmbeddedDSOUnitTests.dll
+        testResultsFile: TestResult-EmbeddedDSOUnitTests-macOS-$(XA.Build.Configuration).xml
+
     - template: yaml-templates/run-nunit-tests.yaml
       parameters:
         testRunTitle: Xamarin.Android.Build.Tests - macOS

tests/CodeBehind/UnitTests/BuildTests.cs

@@ -553,8 +553,15 @@ void RunTest (string testName, bool many, bool dtb, Action<TestProjectInfo, bool
 				}
 			} catch {
 				CopyLogs (testInfo, false);
+				foreach (var file in Directory.GetFiles (testInfo.OutputDirectory, "*.log", SearchOption.AllDirectories)) {
+					TestContext.AddTestAttachment (file);
+				}
 				throw;
 			}
+
+			// Clean up successful tests
+			FileSystemUtils.SetDirectoryWriteable (testInfo.OutputDirectory);
+			Directory.Delete (testInfo.OutputDirectory, recursive: true);
 		}
 
 		bool WasParsedInParallel (TestProjectInfo testInfo)

tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-UnitTests/BuildTests.cs

@@ -90,6 +90,15 @@ public void BuildProject ()
 			Assert.That (success, Is.True, "Should have been built");
 		}
 
+		[OneTimeTearDown]
+		public void CleanUp ()
+		{
+			if (TestContext.CurrentContext.Result.FailCount == 0) {
+				FileSystemUtils.SetDirectoryWriteable (TestOutputDir);
+				Directory.Delete (TestOutputDir, recursive: true);
+			}
+		}
+
 		[Test]
 		public void BinariesExist ()
 		{

tests/EmbeddedDSOs/EmbeddedDSO-UnitTests/BuildTests.cs

@@ -83,6 +83,15 @@ public void BuildProject ()
 			androidSdkDir = AndroidSdkResolver.GetAndroidSdkPath ();
 		}
 
+		[OneTimeTearDown]
+		public void CleanUp ()
+		{
+			if (TestContext.CurrentContext.Result.FailCount == 0) {
+				FileSystemUtils.SetDirectoryWriteable (TestOutputDir);
+				Directory.Delete (TestOutputDir, recursive: true);
+			}
+		}
+
 		[Test]
 		public void BinariesExist ()
 		{