Skip to content

Commit 8b62787

Browse files
filzrevfilzrev
authored
feat: add vstestadapter filter support (#2788)
1 parent 81a4e55 commit 8b62787

File tree

6 files changed

+267
-2
lines changed

6 files changed

+267
-2
lines changed

‎samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
<OutputType>Exe</OutputType>
88
<TargetFrameworks>net462;net8.0</TargetFrameworks>
99
<GenerateProgramFile>false</GenerateProgramFile>
10+
<!-- Disable parallel tests between TargetFrameworks -->
11+
<TestTfmsInParallel>false</TestTfmsInParallel>
1012
</PropertyGroup>
1113
<ItemGroup>
1214
<Compile Include="Program.fs" />

‎samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
<NoWarn>$(NoWarn);CA1018;CA5351;CA1825</NoWarn>
1414
<!-- Disable entry point generation as this project has it's own entry point -->
1515
<GenerateProgramFile>false</GenerateProgramFile>
16+
<!-- Disable parallel tests between TargetFrameworks -->
17+
<TestTfmsInParallel>false</TestTfmsInParallel>
1618
</PropertyGroup>
1719
<ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
1820
<Reference Include="System.Reflection" />

‎src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
2+
using System.Diagnostics;
3+
using System.IO;
4+
5+
namespace BenchmarkDotNet.TestAdapter;
6+
7+
internal class LoggerHelper
8+
{
9+
public LoggerHelper(IMessageLogger logger, Stopwatch stopwatch)
10+
{
11+
InnerLogger = logger;
12+
Stopwatch = stopwatch;
13+
}
14+
15+
public IMessageLogger InnerLogger { get; private set; }
16+
17+
public Stopwatch Stopwatch { get; private set; }
18+
19+
public void Log(string format, params object[] args)
20+
{
21+
SendMessage(TestMessageLevel.Informational, null, string.Format(format, args));
22+
}
23+
24+
public void LogWithSource(string source, string format, params object[] args)
25+
{
26+
SendMessage(TestMessageLevel.Informational, source, string.Format(format, args));
27+
}
28+
29+
public void LogError(string format, params object[] args)
30+
{
31+
SendMessage(TestMessageLevel.Error, null, string.Format(format, args));
32+
}
33+
34+
public void LogErrorWithSource(string source, string format, params object[] args)
35+
{
36+
SendMessage(TestMessageLevel.Error, source, string.Format(format, args));
37+
}
38+
39+
public void LogWarning(string format, params object[] args)
40+
{
41+
SendMessage(TestMessageLevel.Warning, null, string.Format(format, args));
42+
}
43+
44+
public void LogWarningWithSource(string source, string format, params object[] args)
45+
{
46+
SendMessage(TestMessageLevel.Warning, source, string.Format(format, args));
47+
}
48+
49+
private void SendMessage(TestMessageLevel level, string? assemblyName, string message)
50+
{
51+
var assemblyText = assemblyName == null
52+
? "" :
53+
$"{Path.GetFileNameWithoutExtension(assemblyName)}: ";
54+
55+
InnerLogger.SendMessage(level, $"[BenchmarkDotNet {Stopwatch.Elapsed:hh\\:mm\\:ss\\.ff}] {assemblyText}{message}");
56+
}
57+
}

‎src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
2+
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
3+
using System;
4+
using System.Collections.Generic;
5+
using System.IO;
6+
using System.Linq;
7+
using System.Reflection;
8+
9+
namespace BenchmarkDotNet.TestAdapter;
10+
11+
internal class TestCaseFilter
12+
{
13+
private const string DisplayNameString = "DisplayName";
14+
private const string FullyQualifiedNameString = "FullyQualifiedName";
15+
16+
private readonly HashSet<string> knownTraits;
17+
private List<string> supportedPropertyNames;
18+
private readonly ITestCaseFilterExpression? filterExpression;
19+
private readonly bool successfullyGotFilter;
20+
private readonly bool isDiscovery;
21+
22+
public TestCaseFilter(IDiscoveryContext discoveryContext, LoggerHelper logger)
23+
{
24+
// Traits are not known at discovery time because we load them from benchmarks
25+
isDiscovery = true;
26+
knownTraits = [];
27+
supportedPropertyNames = GetSupportedPropertyNames();
28+
successfullyGotFilter = GetTestCaseFilterExpressionFromDiscoveryContext(discoveryContext, logger, out filterExpression);
29+
}
30+
31+
public TestCaseFilter(IRunContext runContext, LoggerHelper logger, string assemblyFileName, HashSet<string> knownTraits)
32+
{
33+
this.knownTraits = knownTraits;
34+
supportedPropertyNames = GetSupportedPropertyNames();
35+
successfullyGotFilter = GetTestCaseFilterExpression(runContext, logger, assemblyFileName, out filterExpression);
36+
}
37+
38+
public string GetTestCaseFilterValue()
39+
{
40+
return successfullyGotFilter
41+
? filterExpression?.TestCaseFilterValue ?? ""
42+
: "";
43+
}
44+
45+
public bool MatchTestCase(TestCase testCase)
46+
{
47+
if (!successfullyGotFilter)
48+
{
49+
// Had an error while getting filter, match no testcase to ensure discovered test list is empty
50+
return false;
51+
}
52+
else if (filterExpression == null)
53+
{
54+
// No filter specified, keep every testcase
55+
return true;
56+
}
57+
58+
return filterExpression.MatchTestCase(testCase, p => PropertyProvider(testCase, p));
59+
}
60+
61+
public object? PropertyProvider(TestCase testCase, string name)
62+
{
63+
// Traits filtering
64+
if (isDiscovery || knownTraits.Contains(name))
65+
{
66+
var result = new List<string>();
67+
68+
foreach (var trait in GetTraits(testCase))
69+
if (string.Equals(trait.Key, name, StringComparison.OrdinalIgnoreCase))
70+
result.Add(trait.Value);
71+
72+
if (result.Count > 0)
73+
return result.ToArray();
74+
}
75+
76+
// Property filtering
77+
switch (name.ToLowerInvariant())
78+
{
79+
// FullyQualifiedName
80+
case "fullyqualifiedname":
81+
return testCase.FullyQualifiedName;
82+
// DisplayName
83+
case "displayname":
84+
return testCase.DisplayName;
85+
default:
86+
return null;
87+
}
88+
}
89+
90+
private bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression? filter)
91+
{
92+
filter = null;
93+
94+
try
95+
{
96+
filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!);
97+
return true;
98+
}
99+
catch (TestPlatformFormatException e)
100+
{
101+
logger.LogWarning("{0}: Exception filtering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.Message);
102+
return false;
103+
}
104+
}
105+
106+
private bool GetTestCaseFilterExpressionFromDiscoveryContext(IDiscoveryContext discoveryContext, LoggerHelper logger, out ITestCaseFilterExpression? filter)
107+
{
108+
filter = null;
109+
110+
if (discoveryContext is IRunContext runContext)
111+
{
112+
try
113+
{
114+
filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!);
115+
return true;
116+
}
117+
catch (TestPlatformException e)
118+
{
119+
logger.LogWarning("Exception filtering tests: {0}", e.Message);
120+
return false;
121+
}
122+
}
123+
else
124+
{
125+
try
126+
{
127+
// GetTestCaseFilter is present on DiscoveryContext but not in IDiscoveryContext interface
128+
var method = discoveryContext.GetType().GetRuntimeMethod("GetTestCaseFilter", [typeof(IEnumerable<string>), typeof(Func<string, TestProperty>)]);
129+
filter = (ITestCaseFilterExpression)method?.Invoke(discoveryContext, [supportedPropertyNames, null])!;
130+
131+
return true;
132+
}
133+
catch (TargetInvocationException e)
134+
{
135+
if (e?.InnerException is TestPlatformException ex)
136+
{
137+
logger.LogWarning("Exception filtering tests: {0}", ex.InnerException.Message ?? "");
138+
return false;
139+
}
140+
141+
throw e!.InnerException;
142+
}
143+
}
144+
}
145+
146+
private List<string> GetSupportedPropertyNames()
147+
{
148+
// Returns the set of well-known property names usually used with the Test Plugins (Used Test Traits + DisplayName + FullyQualifiedName)
149+
if (supportedPropertyNames == null)
150+
{
151+
supportedPropertyNames = knownTraits.ToList();
152+
supportedPropertyNames.Add(DisplayNameString);
153+
supportedPropertyNames.Add(FullyQualifiedNameString);
154+
}
155+
156+
return supportedPropertyNames;
157+
}
158+
159+
private static IEnumerable<KeyValuePair<string, string>> GetTraits(TestCase testCase)
160+
{
161+
var traitProperty = TestProperty.Find("TestObject.Traits");
162+
return traitProperty != null
163+
? testCase.GetPropertyValue(traitProperty, Array.Empty<KeyValuePair<string, string>>())
164+
: [];
165+
}
166+
}

‎src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
55
using System;
66
using System.Collections.Generic;
7+
using System.Diagnostics;
78
using System.IO;
89
using System.Linq;
910
using System.Reflection;
@@ -42,11 +43,18 @@ public void DiscoverTests(
4243
IMessageLogger logger,
4344
ITestCaseDiscoverySink discoverySink)
4445
{
46+
var stopwatch = Stopwatch.StartNew();
47+
var loggerHelper = new LoggerHelper(logger, stopwatch);
48+
var testCaseFilter = new TestCaseFilter(discoveryContext, loggerHelper);
49+
4550
foreach (var source in sources)
4651
{
4752
ValidateSourceIsAssemblyOrThrow(source);
4853
foreach (var testCase in GetVsTestCasesFromAssembly(source, logger))
4954
{
55+
if (!testCaseFilter.MatchTestCase(testCase))
56+
continue;
57+
5058
discoverySink.SendTestCase(testCase);
5159
}
5260
}
@@ -67,14 +75,19 @@ public void RunTests(IEnumerable<TestCase>? tests, IRunContext? runContext, IFra
6775

6876
cts ??= new CancellationTokenSource();
6977

78+
var stopwatch = Stopwatch.StartNew();
79+
var logger = new LoggerHelper(frameworkHandle, stopwatch);
80+
7081
foreach (var testsPerAssembly in tests.GroupBy(t => t.Source))
82+
{
7183
RunBenchmarks(testsPerAssembly.Key, frameworkHandle, testsPerAssembly);
84+
}
7285

7386
cts = null;
7487
}
7588

7689
/// <summary>
77-
/// Runs all benchmarks in the given set of sources (assemblies).
90+
/// Runs all/filtered benchmarks in the given set of sources (assemblies).
7891
/// </summary>
7992
/// <param name="sources">The assemblies to run.</param>
8093
/// <param name="runContext">A context that the run is performed in.</param>
@@ -88,8 +101,31 @@ public void RunTests(IEnumerable<string>? sources, IRunContext? runContext, IFra
88101

89102
cts ??= new CancellationTokenSource();
90103

104+
var stopwatch = Stopwatch.StartNew();
105+
var logger = new LoggerHelper(frameworkHandle, stopwatch);
106+
91107
foreach (var source in sources)
92-
RunBenchmarks(source, frameworkHandle);
108+
{
109+
var filter = new TestCaseFilter(runContext!, logger, source, ["Category"]);
110+
if (filter.GetTestCaseFilterValue() != "")
111+
{
112+
var discoveredBenchmarks = GetVsTestCasesFromAssembly(source, frameworkHandle);
113+
var filteredTestCases = discoveredBenchmarks.Where(x => filter.MatchTestCase(x))
114+
.ToArray();
115+
116+
if (filteredTestCases.Length == 0)
117+
continue;
118+
119+
// Run filtered tests.
120+
RunBenchmarks(source, frameworkHandle, filteredTestCases);
121+
}
122+
else
123+
{
124+
// Run all benchmarks
125+
RunBenchmarks(source, frameworkHandle);
126+
}
127+
}
128+
93129

94130
cts = null;
95131
}

‎src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
<Project InitialTargets="GenerateBDNEntryPoint" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
22
<PropertyGroup>
33
<EntryPointSourceDirectory>$(MSBuildThisFileDirectory)..\entrypoints\</EntryPointSourceDirectory>
4+
<!-- Disable parallel tests between TargetFrameworks, if it's not explicitly specified. -->
5+
<TestTfmsInParallel Condition="'$(TestTfmsInParallel)' = ''">false</TestTfmsInParallel>
46
</PropertyGroup>
57
<!--
68
Microsoft.NET.Test.Sdk uses a property called "GenerateProgramFile" to determine whether it generates an entry

0 commit comments

Comments
 (0)