Build isolated test assets for single TFM instead of 7

eng/build.ps1

@@ -125,15 +125,20 @@ if ($integrationTest -or $performanceTest -or $compatibilityTest -or $smokeTest)
 }
 
 if ($filters.Count -gt 0 -or $testParameters.Count -gt 0) {
-  # We have to double escape, otherwise the filter is passed as string with & in it and interpreted directly as a separate command to run.
-  # Ignoring exit code 8 which means no tests found, otherwise we will fail the build when we run with a filter that doesn't match any test in a project, which is common when we have multiple projects and some of them don't have certain categories of tests. https://github.com/microsoft/testfx/issues/7457
-  $filterString = "--filter \`"$($filters -join '&')\`" -ignore-exit-code 8"
+  if ($filters.Count -gt 0) {
+
+    # We have to double escape by '\"', otherwise the filter is passed as string with & in it and interpreted directly as a separate command to run.
+    # Ignoring exit code 8 which means no tests found, otherwise we will fail the build when we run with a filter that doesn't match any test in a project, which is common when we have multiple projects and some of them don't have certain categories of tests. https://github.com/microsoft/testfx/issues/7457
+    $filterString = "--filter \`"$($filters -join '&')\`""
+    $filterParameters = "$filterString --ignore-exit-code 8"
+  }
+
   $testParameterString = ($testParameters.GetEnumerator() | ForEach-Object { "--test-parameter $($_.Key)=$($_.Value)" }) -join ' '
 
   if (-not $PSBoundParameters.ContainsKey('properties')) {
     $PSBoundParameters['properties'] = @()
   }
-  $PSBoundParameters['properties'] += "/p:TestRunnerExternalArguments=$filterString $testParameterString"
+  $PSBoundParameters['properties'] += "/p:TestRunnerExternalArguments=$filterParameters $testParameterString"
 }
 
 # Call the build script provided by Arcade

src/vstest.console/Internal/ISystemTimersTimer.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Timers;
+
+namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Internal;
+
+internal interface ISystemTimersTimer : IDisposable
+{
+    event ElapsedEventHandler? Elapsed;
+
+    void Start();
+    void Stop();
+}

src/vstest.console/Internal/ProgressIndicator.cs

@@ -6,8 +6,6 @@
 
 using Microsoft.VisualStudio.TestPlatform.Utilities;
 
-using Timer = System.Timers.Timer;
-
 namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Internal;
 
 /// <summary>
@@ -16,10 +14,11 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Internal;
 internal sealed class ProgressIndicator : IProgressIndicator, IDisposable
 {
     private readonly object _syncObject = new();
-    private int _dotCounter;
-    private Timer? _timer;
+    private readonly ISystemTimersTimer _timer;
     private readonly string _testRunProgressString;
 
+    private int _dotCounter;
+
     /// <summary>
     /// Used to output to the console
     /// </summary>
@@ -35,8 +34,9 @@ internal sealed class ProgressIndicator : IProgressIndicator, IDisposable
     /// </summary>
     public bool IsRunning { get; private set; }
 
-    public ProgressIndicator(IOutput output, IConsoleHelper consoleHelper)
+    public ProgressIndicator(IOutput output, IConsoleHelper consoleHelper, ISystemTimersTimer? timer = null)
     {
+        _timer = timer ?? new SystemTimersTimer(1000);
         ConsoleOutput = output;
         ConsoleHelper = consoleHelper;
         _testRunProgressString = string.Format(CultureInfo.CurrentCulture, "{0}...", Resources.Resources.ProgressIndicatorString);
@@ -47,12 +47,8 @@ public void Start()
     {
         lock (_syncObject)
         {
-            if (_timer == null)
-            {
-                _timer = new Timer(1000);
-                _timer.Elapsed += Timer_Elapsed;
-                _timer.Start();
-            }
+            _timer.Elapsed += Timer_Elapsed;
+            _timer.Start();
 
             // Print the string based on the previous state, that is dotCounter
             // This is required for smooth transition

src/vstest.console/Internal/SystemTimersTimer.cs

@@ -0,0 +1,23 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Timer = System.Timers.Timer;
+
+namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Internal;
+
+internal sealed class SystemTimersTimer : ISystemTimersTimer
+{
+    private readonly Timer _timer;
+    public SystemTimersTimer(int interval)
+    {
+        _timer = new Timer(interval);
+    }
+    public event System.Timers.ElapsedEventHandler? Elapsed
+    {
+        add => _timer.Elapsed += value;
+        remove => _timer.Elapsed -= value;
+    }
+    public void Start() => _timer.Start();
+    public void Stop() => _timer.Stop();
+    public void Dispose() => _timer.Dispose();
+}

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs

@@ -23,7 +23,7 @@ public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(Ru
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj");
+        var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj", runnerInfo.TargetFramework);
         // Forcing terminal logger so we can see the output when it is redirected
         InvokeDotnetTest($@"{projectPath} -tl:on -nodereuse:false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath));
 
@@ -53,7 +53,7 @@ public void MSBuildLoggerCanBeDisabledByBuildProperty(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj");
+        var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj", runnerInfo.TargetFramework);
         InvokeDotnetTest($@"{projectPath} -nodereuse:false /p:VsTestUseMSBuildOutput=false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath));
 
         // Check that we see the summary that is printed from the console logger, meaning the new output is disabled.
@@ -73,7 +73,7 @@ public void MSBuildLoggerCanBeDisabledByEnvironmentVariableProperty(RunnerInfo r
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj");
+        var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj", runnerInfo.TargetFramework);
         InvokeDotnetTest($@"{projectPath} -nodereuse:false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", environmentVariables: new Dictionary<string, string?> { ["MSBUILDENSURESTDOUTFORTASKPROCESSES"] = "1" }, workingDirectory: Path.GetDirectoryName(projectPath));
 
         // Check that we see the summary that is printed from the console logger, meaning the new output is disabled.

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs

@@ -26,7 +26,7 @@ public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var projectPath = GetIsolatedTestAsset("SimpleTestProject.csproj");
+        var projectPath = GetIsolatedTestAsset("SimpleTestProject.csproj", runnerInfo.TargetFramework);
         InvokeDotnetTest($@"{projectPath} -tl:off /p:VSTestNoLogo=false --logger:""Console;Verbosity=normal"" /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath));
 
         // ensure our dev version is used
@@ -60,7 +60,7 @@ public void RunDotnetTestWithCsprojPassInlineSettings(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var projectPath = GetIsolatedTestAsset("ParametrizedTestProject.csproj");
+        var projectPath = GetIsolatedTestAsset("ParametrizedTestProject.csproj", runnerInfo.TargetFramework);
         InvokeDotnetTest($@"{projectPath} --logger:""Console;Verbosity=normal"" -tl:off /p:VSTestNoLogo=false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion} -- TestRunParameters.Parameter(name =\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(projectPath));
 
         // ensure our dev version is used

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs

@@ -510,7 +510,7 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo)
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
         var projectName = "ProjectFileRunSettingsTestProject.csproj";
-        var projectPath = GetIsolatedTestAsset(projectName);
+        var projectPath = GetIsolatedTestAsset(projectName, runnerInfo.TargetFramework);
         InvokeDotnetTest($@"{projectPath} /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal"" /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath));
         ValidateSummaryStatus(0, 1, 0);
 

test/Microsoft.TestPlatform.TestUtilities/AcceptanceTestBase.cs

@@ -4,6 +4,7 @@
 using System;
 using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
 
 namespace Microsoft.TestPlatform.TestUtilities;
 
@@ -137,7 +138,7 @@ public static string GetRunSettingsWithTargetFramework(string targetFramework)
         return runSettingsXml;
     }
 
-    protected string GetIsolatedTestAsset(string assetName)
+    protected string GetIsolatedTestAsset(string assetName, string targetFramework)
     {
         var projectPath = GetProjectFullPath(assetName);
 
@@ -147,6 +148,13 @@ protected string GetIsolatedTestAsset(string assetName)
 
             if (file.Extension.Equals(".csproj", StringComparison.OrdinalIgnoreCase))
             {
+                // Build just for the given tfm
+                var projFile = Path.Combine(TempDirectory.Path, Path.GetFileName(file.FullName));
+                var csprojContent = File.ReadAllText(projFile);
+                csprojContent = Regex.Replace(csprojContent, "<TargetFramework>.*?</TargetFramework>", $"<TargetFramework>{targetFramework}</TargetFramework>");
+                csprojContent = Regex.Replace(csprojContent, "<TargetFrameworks>.*?</TargetFrameworks>", $"<TargetFramework>{targetFramework}</TargetFramework>");
+                File.WriteAllText(projFile, csprojContent);
+
                 string root = IntegrationTestEnvironment.RepoRootDirectory;
                 var testAssetsRoot = Path.GetFullPath(Path.Combine(root, "test", "TestAssets"));
 

test/vstest.console.UnitTests/Internal/ProgressIndicatorTests.cs

@@ -12,6 +12,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Internal;
 [TestClass]
 public class ProgressIndicatorTests
 {
+    private readonly SteppableTimer _steppableTimer;
     private readonly ProgressIndicator _indicator;
     private readonly Mock<IOutput> _consoleOutput;
     private readonly Mock<IConsoleHelper> _consoleHelper;
@@ -22,7 +23,8 @@ public ProgressIndicatorTests()
         _consoleHelper = new Mock<IConsoleHelper>();
         _consoleHelper.Setup(c => c.WindowWidth).Returns(100);
         _consoleHelper.Setup(c => c.CursorTop).Returns(20);
-        _indicator = new ProgressIndicator(_consoleOutput.Object, _consoleHelper.Object);
+        _steppableTimer = new SteppableTimer();
+        _indicator = new ProgressIndicator(_consoleOutput.Object, _consoleHelper.Object, _steppableTimer);
     }
 
     [TestCleanup]
@@ -40,16 +42,23 @@ public void StartShouldStartPrintingProgressMessage()
     }
 
     [TestMethod]
-    public void StartShouldShowProgressMessage()
+    public void StartShouldShowProgressMessageAndDotForEveryTimeATimerFires()
     {
-        _indicator.Start();
-
         _consoleHelper.Setup(c => c.CursorLeft).Returns(30);
-        System.Threading.Thread.Sleep(1500);
 
+        _indicator.Start();
         Assert.IsTrue(_indicator.IsRunning);
         _consoleOutput.Verify(m => m.Write("Test run in progress.", OutputLevel.Information), Times.Once);
+
+        _steppableTimer.Step();
+
         _consoleOutput.Verify(m => m.Write(".", OutputLevel.Information), Times.Once);
+
+        _steppableTimer.Step();
+
+        _consoleOutput.Verify(m => m.Write(".", OutputLevel.Information), Times.Exactly(2));
+
+        _indicator.Stop();
     }
 
     [TestMethod]

test/vstest.console.UnitTests/Internal/SteppableTimer.cs

@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Timers;
+
+using Microsoft.VisualStudio.TestPlatform.CommandLine.Internal;
+
+namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Internal;
+
+internal class SteppableTimer : ISystemTimersTimer
+{
+    private bool _isStarted;
+
+    public void Step()
+    {
+        if (_isStarted)
+        {
+            // Some craziness with no constructor here.
+#if NET
+            var elapsed = new System.Timers.ElapsedEventArgs(DateTime.UtcNow);
+#else
+            var elapsed = ElapsedEventArgs.Empty as ElapsedEventArgs;
+#endif
+            Elapsed?.Invoke(this, elapsed);
+        }
+    }
+
+    private event ElapsedEventHandler? Elapsed = delegate { };
+
+    event ElapsedEventHandler? ISystemTimersTimer.Elapsed
+    {
+        add
+        {
+            Elapsed += value;
+        }
+
+        remove
+        {
+            Elapsed -= value;
+        }
+    }
+
+    void IDisposable.Dispose()
+    {
+    }
+
+    void ISystemTimersTimer.Start()
+    {
+        _isStarted = true;
+    }
+
+    void ISystemTimersTimer.Stop()
+    {
+        _isStarted = false;
+    }
+}