Skip to content

Commit bd12521

Browse files
nohwndnohwnd
authored
Allow sharing testhosts for non-paralell runs on a Shared testhost (#3682)
1 parent 3330a06 commit bd12521

File tree

6 files changed

+231
-97
lines changed

6 files changed

+231
-97
lines changed

‎playground/TestPlatform.Playground/Program.cs‎

Lines changed: 88 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -36,59 +36,121 @@ static void Main(string[] args)
3636

3737
var thisAssemblyPath = Assembly.GetEntryAssembly().Location;
3838
var here = Path.GetDirectoryName(thisAssemblyPath);
39-
var playground = Path.GetFullPath(Path.Combine(here, "..", "..", "..", ".."));
39+
//var playground = Path.GetFullPath(Path.Combine(here, "..", "..", "..", ".."));
4040

4141
var console = Path.Combine(here, "vstest.console", "vstest.console.exe");
4242

43+
var discoverySettings = $@"
44+
<RunSettings>
45+
<RunConfiguration>
46+
<InIsolation>true</InIsolation>
47+
<MaxCpuCount>0</MaxCpuCount>
48+
<DisableAppDomain>False</DisableAppDomain>
49+
<BatchSize>10</BatchSize>
50+
</RunConfiguration>
51+
</RunSettings>
52+
";
4353

4454
var sourceSettings = @"
4555
<RunSettings>
4656
<RunConfiguration>
4757
<InIsolation>true</InIsolation>
48-
<MaxCpuCount>4</MaxCpuCount>
58+
<MaxCpuCount>0</MaxCpuCount>
4959
</RunConfiguration>
5060
</RunSettings>
5161
";
5262

5363
var sources = new[] {
54-
Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll"),
55-
Path.Combine(playground, "MSTest1", "bin", "Debug", "net5.0", "MSTest1.dll"),
56-
@"C:\Users\jajares\source\repos\TestProject48\TestProject48\bin\Debug\net48\TestProject48.dll",
57-
@"C:\Users\jajares\source\repos\TestProject48\TestProject1\bin\Debug\net48\win10-x64\TestProject1.dll"
58-
};
5964

60-
// console mode
61-
var settingsFile = Path.GetTempFileName();
62-
try
63-
{
64-
File.WriteAllText(settingsFile, sourceSettings);
65-
var process = Process.Start(console, string.Join(" ", sources) + " --settings:" + settingsFile + " --listtests");
66-
process.WaitForExit();
67-
if (process.ExitCode != 0)
68-
{
69-
throw new Exception($"Process failed with {process.ExitCode}");
70-
}
71-
}
72-
finally
73-
{
74-
try { File.Delete(settingsFile); } catch { }
75-
}
65+
// @"C:\t\TestProject13_\TestProject1\bin\Debug\net48\TestProject1.dll",
66+
67+
// @"C:\t\TestProject13_for_mstest\TestProject1\bin\Debug\net48\TestProject1.dll",
68+
69+
// @"C:\t\TestProject13_for_mstest\TestProject5\bin\Debug\net472\TestProject5.dll",
70+
//@"C:\t\TestProject13_for_mstest\TestProject6\bin\Debug\net472\TestProject6.dll"
71+
72+
73+
//// // net6
74+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test1\bin\Debug\net6.0\Test1.dll",
75+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test7\bin\Debug\net6.0\Test7.dll",
76+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test2\bin\Debug\net6.0\Test2.dll",
77+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test6\bin\Debug\net6.0\Test6.dll",
78+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test8\bin\Debug\net6.0\Test8.dll",
79+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test5\bin\Debug\net6.0\Test5.dll",
80+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test3\bin\Debug\net6.0\Test3.dll",
81+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test4\bin\Debug\net6.0\Test4.dll",
82+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test10\bin\Debug\net6.0\Test10.dll",
83+
@"C:\t\ParallelDiscovery2\ReproNetCore\Test9\bin\Debug\net6.0\Test9.dll",
84+
85+
//// netfx
86+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project4\bin\Debug\net472\Project4.dll",
87+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project1\bin\Debug\net472\Project1.dll",
88+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project2\bin\Debug\net472\Project2.dll",
89+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project3\bin\Debug\net472\Project3.dll",
90+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project9\bin\Debug\net472\Project9.dll",
91+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project10\bin\Debug\net472\Project10.dll",
92+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project5\bin\Debug\net472\Project5.dll",
93+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project8\bin\Debug\net472\Project8.dll",
94+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project7\bin\Debug\net472\Project7.dll",
95+
@"C:\t\ParallelDiscovery2\ReproNetFx\Project6\bin\Debug\net472\Project6.dll",
96+
97+
98+
//// mix
99+
@"C:\t\MultipleTfmAndArch\Tst1\bin\Debug\net472\win7-x86\Tst1.dll",
100+
@"C:\t\MultipleTfmAndArch\Tst1\bin\Debug\net48\win7-x86\Tst1.dll",
101+
@"C:\t\MultipleTfmAndArch\Tst3\bin\Debug\net48\win7-x86\Tst3.dll",
102+
@"C:\t\MultipleTfmAndArch\Tst1\bin\Debug\net5.0\win7-x86\Tst1.dll",
103+
@"C:\t\MultipleTfmAndArch\Tst2\bin\Debug\net472\win7-x64\Tst2.dll",
104+
@"C:\t\MultipleTfmAndArch\Tst3\bin\Debug\net472\win7-x86\Tst3.dll",
105+
@"C:\t\MultipleTfmAndArch\Tst2\bin\Debug\net48\win7-x64\Tst2.dll",
106+
@"C:\t\MultipleTfmAndArch\Tst2\bin\Debug\netcoreapp3.1\win7-x64\Tst2.dll",
107+
@"C:\t\MultipleTfmAndArch\Tst3\bin\Debug\netcoreapp3.1\win7-x86\Tst3.dll",
108+
@"C:\t\MultipleTfmAndArch\Tst2\bin\Debug\net5.0\win7-x64\Tst2.dll",
109+
@"C:\t\MultipleTfmAndArch\Tst3\bin\Debug\net5.0\win7-x86\Tst3.dll",
110+
@"C:\t\MultipleTfmAndArch\Tst1\bin\Debug\netcoreapp3.1\win7-x86\Tst1.dll",
111+
112+
};
113+
//// console mode
114+
//var settingsFile = Path.GetTempFileName();
115+
//try
116+
//{
117+
// File.WriteAllText(settingsFile, sourceSettings);
118+
// var process = Process.Start(console, string.Join(" ", sources) + " --settings:" + settingsFile + " --listtests");
119+
// var cmd = console + "\n\n" + string.Join(" ", sources) + " --settings:" + settingsFile + " --listtests";
120+
// var swc = Stopwatch.StartNew();
121+
// process.WaitForExit();
122+
// if (process.ExitCode != 0)
123+
// {
124+
// throw new Exception($"Process failed with {process.ExitCode}");
125+
// }
126+
// Console.WriteLine($"Done in {swc.ElapsedMilliseconds} ms");
127+
//}
128+
//finally
129+
//{
130+
// try { File.Delete(settingsFile); } catch { }
131+
//}
76132

77133
// design mode
78134
var consoleOptions = new ConsoleParameters
79135
{
80136
LogFilePath = Path.Combine(here, "logs", "log.txt"),
81-
TraceLevel = TraceLevel.Verbose,
137+
TraceLevel = TraceLevel.Off,
82138
};
83139
var options = new TestPlatformOptions();
84140
var r = new VsTestConsoleWrapper(console, consoleOptions);
85141
var sessionHandler = new TestSessionHandler();
86142
#pragma warning disable CS0618 // Type or member is obsolete
87-
r.StartTestSession(sources, sourceSettings, sessionHandler);
143+
// r.StartTestSession(sources, sourceSettings, sessionHandler);
88144
#pragma warning restore CS0618 // Type or member is obsolete
89145
var discoveryHandler = new PlaygroundTestDiscoveryHandler();
90-
r.DiscoverTests(sources, sourceSettings, options, sessionHandler.TestSessionInfo, discoveryHandler);
146+
var sw = Stopwatch.StartNew();
147+
r.DiscoverTests(sources, discoverySettings, options, sessionHandler.TestSessionInfo, discoveryHandler);
148+
var dd = sw.ElapsedMilliseconds;
149+
Console.WriteLine($"Discovery done in {sw.ElapsedMilliseconds} ms");
150+
sw.Restart();
91151
r.RunTestsWithCustomTestHost(discoveryHandler.TestCases, sourceSettings, options, sessionHandler.TestSessionInfo, new TestRunHandler(), new DebuggerTestHostLauncher());
152+
var rd = sw.ElapsedMilliseconds;
153+
Console.WriteLine($"Discovery: {dd} ms, Run: {rd} ms, Total: {dd + rd} ms");
92154
}
93155

94156
public class PlaygroundTestDiscoveryHandler : ITestDiscoveryEventsHandler, ITestDiscoveryEventsHandler2

‎playground/TestPlatform.Playground/TestPlatform.Playground.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
</ItemGroup>
3939

4040
<Target Name="PostBuild" AfterTargets="PostBuildEvent" Condition=" '$(OS)' == 'WINDOWS_NT' ">
41-
<Exec Command="xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\vstest.console\bin\$(Configuration)\net451\ $(TargetDir)\vstest.console\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\testhost\bin\$(Configuration)\net451\win7-x64\ $(TargetDir)\vstest.console\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\Microsoft.TestPlatform.TestHostProvider\bin\$(Configuration)\net451\ $(TargetDir)\vstest.console\Extensions\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\testhost.x86\bin\$(Configuration)\net472\win7-x86 $(TargetDir)\vstest.console\TestHost\" />
41+
<Exec Command="xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\vstest.console\bin\$(Configuration)\net451\ $(TargetDir)\vstest.console\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\testhost\bin\$(Configuration)\net451\win7-x64\ $(TargetDir)\vstest.console\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\Microsoft.TestPlatform.TestHostProvider\bin\$(Configuration)\net451\ $(TargetDir)\vstest.console\Extensions\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\testhost.x86\bin\$(Configuration)\net472\win7-x86 $(TargetDir)\vstest.console\TestHost\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\testhost\bin\$(Configuration)\net472\win7-x64 $(TargetDir)\vstest.console\TestHost\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\testhost.x86\bin\$(Configuration)\net48\win7-x86 $(TargetDir)\vstest.console\TestHost\&#xD;&#xA;xcopy /i /y $(MSBuildProjectDirectory)\..\..\src\testhost\bin\$(Configuration)\net48\win7-x64 $(TargetDir)\vstest.console\TestHost\" />
4242
</Target>
4343
<Import Project="$(TestPlatformRoot)scripts\build\TestPlatform.targets" />
4444
</Project>

‎src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs‎

Lines changed: 39 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,12 @@ internal class ParallelProxyDiscoveryManager : IParallelProxyDiscoveryManager
2424
{
2525
private readonly IDataSerializer _dataSerializer;
2626
private readonly DiscoveryDataAggregator _dataAggregator;
27+
private readonly bool _isParallel;
2728
private readonly ParallelOperationManager<IProxyDiscoveryManager, ITestDiscoveryEventsHandler2, DiscoveryCriteria> _parallelOperationManager;
2829
private readonly Dictionary<string, TestRuntimeProviderInfo> _sourceToTestHostProviderMap;
2930
private int _discoveryCompletedClients;
3031
private int _availableTestSources = -1;
31-
32+
private int _availableWorkloads;
3233
private bool _skipDefaultAdapters;
3334
private readonly IRequestData _requestData;
3435

@@ -60,6 +61,7 @@ internal ParallelProxyDiscoveryManager(
6061
_requestData = requestData;
6162
_dataSerializer = dataSerializer;
6263
_dataAggregator = dataAggregator;
64+
_isParallel = parallelLevel > 1;
6365
_parallelOperationManager = new(actualProxyManagerCreator, parallelLevel);
6466
_sourceToTestHostProviderMap = testHostProviders
6567
.SelectMany(provider => provider.SourceDetails.Select(s => new KeyValuePair<string, TestRuntimeProviderInfo>(s.Source, provider)))
@@ -78,7 +80,8 @@ public void Initialize(bool skipDefaultAdapters)
7880
public void DiscoverTests(DiscoveryCriteria discoveryCriteria!!, ITestDiscoveryEventsHandler2 eventHandler!!)
7981
{
8082
var workloads = SplitToWorkloads(discoveryCriteria, _sourceToTestHostProviderMap);
81-
_availableTestSources = workloads.Count;
83+
_availableTestSources = workloads.SelectMany(w => w.Work.Sources).Count();
84+
_availableWorkloads = workloads.Count();
8285

8386
EqtTrace.Verbose("ParallelProxyDiscoveryManager.DiscoverTests: Start discovery. Total sources: " + _availableTestSources);
8487

@@ -148,7 +151,7 @@ public bool HandlePartialDiscoveryComplete(IProxyDiscoveryManager proxyDiscovery
148151
_discoveryCompletedClients++;
149152

150153
// If there are no more sources/testcases, a parallel executor is truly done with discovery
151-
allDiscoverersCompleted = _discoveryCompletedClients == _availableTestSources;
154+
allDiscoverersCompleted = _discoveryCompletedClients == _availableWorkloads;
152155

153156
EqtTrace.Verbose("ParallelProxyDiscoveryManager.HandlePartialDiscoveryComplete: Total completed clients = {0}, Discovery complete = {1}, Aborted = {2}, Abort requested: {3}.", _discoveryCompletedClients, allDiscoverersCompleted, isAborted, IsAbortRequested);
154157
}
@@ -182,21 +185,47 @@ public bool HandlePartialDiscoveryComplete(IProxyDiscoveryManager proxyDiscovery
182185

183186
private List<ProviderSpecificWorkload<DiscoveryCriteria>> SplitToWorkloads(DiscoveryCriteria discoveryCriteria, Dictionary<string, TestRuntimeProviderInfo> sourceToTestHostProviderMap)
184187
{
188+
var sources = discoveryCriteria.Sources;
189+
// Each source is grouped with its respective provider.
190+
var providerGroups = sources
191+
.Select(source => new ProviderSpecificWorkload<string>(source, sourceToTestHostProviderMap[source]))
192+
.GroupBy(psw => psw.Provider);
193+
185194
List<ProviderSpecificWorkload<DiscoveryCriteria>> workloads = new();
186-
foreach (var source in discoveryCriteria.Sources)
195+
foreach (var group in providerGroups)
187196
{
188-
var testHostProviderInfo = sourceToTestHostProviderMap[source];
189-
var runsettingsXml = testHostProviderInfo.RunSettings;
190-
var updatedDiscoveryCriteria = new ProviderSpecificWorkload<DiscoveryCriteria>(NewDiscoveryCriteriaFromSourceAndSettings(source, discoveryCriteria, runsettingsXml), testHostProviderInfo);
191-
workloads.Add(updatedDiscoveryCriteria);
197+
var testhostProviderInfo = group.Key;
198+
// If the run is not parallel and the host is shared, put all testcases on single testhost.
199+
// For parallel we prefer to run each source on its own host because we don't know how big the source is
200+
// (how many tests there are to discover), and running 10 sources on 10 parallel testhosts is faster
201+
// in almost all cases than running 10 sources on 1 testhost.
202+
List<string[]> sourceBatches;
203+
if (!_isParallel && testhostProviderInfo.Shared)
204+
{
205+
// Create one big source batch that will be single workload for single testhost.
206+
sourceBatches = new List<string[]> { group.Select(w => w.Work).ToArray() };
207+
}
208+
else
209+
{
210+
// Create multiple source batches, each having one source, so each testhost will end up running one source.
211+
sourceBatches = group.Select(w => new[] { w.Work }).ToList();
212+
}
213+
214+
foreach (var sourcesToDiscover in sourceBatches)
215+
{
216+
var runsettings = testhostProviderInfo.RunSettings;
217+
var updatedCriteria = NewDiscoveryCriteriaFromSourceAndSettings(sourcesToDiscover, discoveryCriteria, runsettings);
218+
var workload = new ProviderSpecificWorkload<DiscoveryCriteria>(updatedCriteria, testhostProviderInfo);
219+
workloads.Add(workload);
220+
}
192221
}
193222

194223
return workloads;
195224

196-
static DiscoveryCriteria NewDiscoveryCriteriaFromSourceAndSettings(string source, DiscoveryCriteria discoveryCriteria, string runsettingsXml)
225+
static DiscoveryCriteria NewDiscoveryCriteriaFromSourceAndSettings(IEnumerable<string> sources, DiscoveryCriteria discoveryCriteria, string runsettingsXml)
197226
{
198227
var criteria = new DiscoveryCriteria(
199-
new[] { source },
228+
sources,
200229
discoveryCriteria.FrequencyOfDiscoveredTestsEvent,
201230
discoveryCriteria.DiscoveredTestEventTimeout,
202231
runsettingsXml,

0 commit comments

Comments
 (0)