Update matrix

eng/Versions.props

@@ -89,30 +89,30 @@
     <XUnitFrameworkVersion>2.4.2</XUnitFrameworkVersion>
     <XUnitAdapterVersion>2.4.5</XUnitAdapterVersion>
     <XUnitConsoleRunnerVersion>2.4.2</XUnitConsoleRunnerVersion>
-    <NUnit3FrameworkVersion>3.14.0</NUnit3FrameworkVersion>
-    <NUnit3AdapterVersion>3.17.0</NUnit3AdapterVersion>
-    <NUnitConsoleRunnerVersion>3.16.3</NUnitConsoleRunnerVersion>
+    <NUnit3FrameworkVersion>4.5.1</NUnit3FrameworkVersion>
+    <NUnit3AdapterVersion>6.1.0</NUnit3AdapterVersion>
+    <NUnitConsoleRunnerVersion>3.22.0</NUnitConsoleRunnerVersion>
     <!--
       Versions that are used when building projects from TestAssets.sln for compatibility tests. See Invoke-TestAssetsBuild in scripts/build.ps1.
       Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo.
       These versions need to be "statically" readable because we read this file as xml in our build and tests.
     -->
     <!-- <MSTestFrameworkLatestVersion></MSTestFrameworkLatestVersion> is not here, because we don't build MSTest locally, so we don't have access to the latest version. -->
-    <MSTestFrameworkLatestPreviewVersion>[3.9.3]</MSTestFrameworkLatestPreviewVersion>
-    <MSTestFrameworkLatestStableVersion>[3.9.3]</MSTestFrameworkLatestStableVersion>
-    <MSTestFrameworkRecentStableVersion>[3.3.1]</MSTestFrameworkRecentStableVersion>
+    <MSTestFrameworkLatestPreviewVersion>[4.1.0]</MSTestFrameworkLatestPreviewVersion>
+    <MSTestFrameworkLatestStableVersion>[4.1.0]</MSTestFrameworkLatestStableVersion>
+    <MSTestFrameworkRecentStableVersion>[4.0.2]</MSTestFrameworkRecentStableVersion>
     <MSTestFrameworkMostDownloadedVersion>[2.2.10]</MSTestFrameworkMostDownloadedVersion>
-    <MSTestFrameworkPreviousStableVersion>[2.2.10]</MSTestFrameworkPreviousStableVersion>
+    <MSTestFrameworkPreviousStableVersion>[3.11.1]</MSTestFrameworkPreviousStableVersion>
     <MSTestFrameworkLegacyStableVersion>[1.4.0]</MSTestFrameworkLegacyStableVersion>
     <!-- Versions that are used to restore previous versions of console, translation layer, and test.sdk for compatibility tests.
     See Invoke-TestAssetsBuild in scripts/build.ps1. Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo.
     These versions need to be "statically" readable because we read this file as xml in our build and tests. -->
     <!-- <VSTestConsoleLatestVersion></VSTestConsoleLatestVersion> is not here, NETTestSdkVersion is used instead, because that is the version of the locally built latest package.  -->
-    <VSTestConsoleLatestPreviewVersion>[17.10.0]</VSTestConsoleLatestPreviewVersion>
-    <VSTestConsoleLatestStableVersion>[17.10.0]</VSTestConsoleLatestStableVersion>
-    <VSTestConsoleRecentStableVersion>[17.9.0]</VSTestConsoleRecentStableVersion>
-    <VSTestConsoleMostDownloadedVersion>[17.6.2]</VSTestConsoleMostDownloadedVersion>
-    <VSTestConsolePreviousStableVersion>[16.11.0]</VSTestConsolePreviousStableVersion>
+    <VSTestConsoleLatestPreviewVersion>[18.3.0]</VSTestConsoleLatestPreviewVersion>
+    <VSTestConsoleLatestStableVersion>[18.3.0]</VSTestConsoleLatestStableVersion>
+    <VSTestConsoleRecentStableVersion>[18.0.1]</VSTestConsoleRecentStableVersion>
+    <VSTestConsoleMostDownloadedVersion>[17.12.0]</VSTestConsoleMostDownloadedVersion>
+    <VSTestConsolePreviousStableVersion>[17.14.1]</VSTestConsolePreviousStableVersion>
     <VSTestConsoleLegacyStableVersion>[15.9.2]</VSTestConsoleLegacyStableVersion>
     <ILAsmPackageVersion>5.0.0</ILAsmPackageVersion>
   </PropertyGroup>

eng/build.ps1

@@ -129,8 +129,10 @@ if ($filters.Count -gt 0 -or $testParameters.Count -gt 0) {
   $filterString = "--filter \`"$($filters -join '&')\`""
   $testParameterString = ($testParameters.GetEnumerator() | ForEach-Object { "--test-parameter $($_.Key)=$($_.Value)" }) -join ' '
 
+  if (-not $PSBoundParameters.ContainsKey('properties')) {
+    $PSBoundParameters['properties'] = @()
+  }
   $PSBoundParameters['properties'] += "/p:TestRunnerExternalArguments=$filterString $testParameterString"
-  Write-Host "Running tests with filter: $filterString and additional parameters: $testParameterString"
 }
 
 # Call the build script provided by Arcade

global.json

@@ -13,21 +13,11 @@
   "tools": {
     "runtimes": {
       "dotnet/x64": [
-        "2.1.30",
-        "3.1.32",
-        "5.0.17",
-        "6.0.32",
-        "7.0.20",
         "8.0.25",
         "9.0.14",
         "10.0.4"
       ],
       "dotnet/x86": [
-        "2.1.30",
-        "3.1.32",
-        "5.0.17",
-        "6.0.32",
-        "7.0.20",
         "8.0.25",
         "9.0.14",
         "10.0.4"

src/vstest.console/TestPlatformHelpers/TestRequestManager.cs

@@ -1592,6 +1592,7 @@ internal static class KnownPlatformSourceFilter
 
         // NUnit
         "NUnit3.TestAdapter.dll",
+        "testcentric.engine.metadata.dll",
 
         // XUnit
         "xunit.runner.visualstudio.testadapter.dll",

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs

@@ -62,8 +62,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(R
 
         InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty);
 
-        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty);
-
         ValidateSummaryStatus(2, 2, 2);
         ExitCodeEquals(1); // failing tests
     }
@@ -81,8 +79,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations3(R
 
         InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty);
 
-        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty);
-
         ValidateSummaryStatus(2, 2, 2);
         ExitCodeEquals(1); // failing tests
     }
@@ -401,8 +397,8 @@ public void ExecuteTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(Runne
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
-    [NetCoreTargetFrameworkDataSource]
+    // This is a built-in assembly filter test. It changes with vstest.version, so testing against 1 version of console is enough.
+    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
     public void RunXunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
     {
         // This is the default filter of AzDo VSTest task:
@@ -426,8 +422,8 @@ public void RunXunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
-    [NetCoreTargetFrameworkDataSource()]
+    // This is a built-in assembly filter test. It changes with vstest.version, so testing against 1 version of console is enough.
+    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
     public void RunMstestTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
     {
         // This is the default filter of AzDo VSTest task:
@@ -452,8 +448,8 @@ public void RunMstestTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
     }
 
     [TestMethod]
-    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
-    [NetCoreTargetFrameworkDataSource()]
+    // This is a built-in assembly filter test. It changes with vstest.version, so testing against 1 version of console is enough.
+    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
     public void RunNunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
     {
         // This is the default filter of AzDo VSTest task:

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/TestCaseFilterTests.cs

@@ -185,7 +185,7 @@ public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo)
     /// </summary>
     [TestMethod]
     [TestCategory("Windows-Review")]
-    // todo: remove we don't support tmi anomore, and remove the test settings
+    // todo: remove we don't support tmi anymore, and remove the test settings
     [Ignore("Temporary ignoring, because of incomplete interop work for legacy TP")]
     [NetFullTargetFrameworkDataSource]
     public void DiscoverTmiTestsWithOnlyPropertyValue(RunnerInfo runnerInfo)

test/Microsoft.TestPlatform.Library.IntegrationTests/AppDomainTests.cs

@@ -21,7 +21,7 @@ public class AppDomainTests : AcceptanceTestBase
 {
     [TestMethod]
     [TestCategory("Windows-Review")]
-    // AppDomains are .NET Framework only, run in .NET Framework runner and
+    // AppDomains are .NET Framework only, run in .NET Framework runner and .NET runner
     [NetFullTargetFrameworkDataSource]
     public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo)
     {

test/Microsoft.TestPlatform.Library.IntegrationTests/TranslationLayerTests/CodeCoverageTests.cs

@@ -436,7 +436,7 @@ public async Task EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runn
         var processManager = (_vstestConsoleWrapper).GetType().GetField("_vstestConsoleProcessManager", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!.GetValue(_vstestConsoleWrapper)!;
         var processId = (int)processManager.GetType().GetProperty("ProcessId", System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance)!.GetValue(processManager)!;
         var consoleProcess = Process.GetProcessById(processId);
-        Assert.IsFalse(consoleProcess.HasExited, $"vstest.console process did not start");
+        Assert.IsFalse(consoleProcess.HasExited, $"vstest.console should be running");
 
         // act
         _vstestConsoleWrapper!.EndSession();

test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBuild.cs

@@ -36,6 +36,7 @@ public static void BuildTestAssetsForIntegrationTests(TestContext context)
         if (skipBuild)
         {
             Debug.WriteLine("SkipIntegrationBuild parameter is true, skipping build.");
+            return;
         }
 
         var buildCompatibility = GetTestParameter(context, "BuildCompatibility");

test/TestAssets/MSTestProject1/UnitTest1.cs

@@ -3,6 +3,9 @@
 
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
+// Class level just because we want the tests to run in predictable order.
+[assembly: Parallelize(Scope = ExecutionScope.ClassLevel, Workers = 0)]
+
 namespace MSTestProject1;
 
 [TestClass]

test/TestAssets/MSTestProject2/UnitTest1.cs

@@ -3,6 +3,9 @@
 
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
+// Class level just because we want the tests to run in predictable order.
+[assembly: Parallelize(Scope = ExecutionScope.ClassLevel, Workers = 0)]
+
 namespace MSTestProject2;
 
 [TestClass]