Add a VSTest Adapter (#2438)

Author: caaavik-msft

BenchmarkDotNet.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.27130.2027
+# Visual Studio Version 17
+VisualStudioVersion = 17.8.34004.107
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D6597E3A-6892-4A68-8E14-042FC941FDA2}"
 EndProject
@@ -51,6 +51,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.Diagnostics
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks", "tests\BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks\BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj", "{AACA2C63-A85B-47AB-99FC-72C3FF408B14}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.TestAdapter", "src\BenchmarkDotNet.TestAdapter\BenchmarkDotNet.TestAdapter.csproj", "{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -137,6 +139,10 @@ Global
 		{AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -162,6 +168,7 @@ Global
 		{B620D10A-CD8E-4A34-8B27-FD6257E63AD0} = {63B94FD6-3F3D-4E04-9727-48E86AC4384C}
 		{C5BDA61F-3A56-4B59-901D-0A17E78F4076} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 		{AACA2C63-A85B-47AB-99FC-72C3FF408B14} = {14195214-591A-45B7-851A-19D3BA2413F9}
+		{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC} = {D6597E3A-6892-4A68-8E14-042FC941FDA2}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F}

docs/articles/features/toc.yml

@@ -11,4 +11,6 @@
 - name: EtwProfiler
   href: etwprofiler.md
 - name: EventPipeProfiler
-  href: event-pipe-profiler.md
+  href: event-pipe-profiler.md
+- name: VSTest
+  href: vstest.md

docs/articles/features/vstest.md

@@ -0,0 +1,74 @@
+---
+uid: docs.baselines
+name: Running with VSTest
+---
+
+# Running with VSTest
+BenchmarkDotNet has support for discovering and executing benchmarks through VSTest. This provides an alternative user experience to running benchmarks with the CLI and may be preferable for those who like their IDE's VSTest integrations that they may have used when running unit tests.
+
+Below is an example of running some benchmarks from the BenchmarkDotNet samples project in Visual Studio's Test Explorer.
+
+![](../../images/vs-testexplorer-demo.png)
+
+## About VSTest
+
+VSTest is one of the most popular test platforms in use in the .NET ecosystem, with test frameworks such as MSTest, xUnit, and NUnit providing support for it. Many IDEs, including Visual Studio and Rider, provide UIs for running tests through VSTest which some users may find more accessible than running them through the command line. 
+
+It may seem counterintuitive to run performance tests on a platform that is designed for unit tests that expect a boolean outcome of "Passed" or "Failed", however VSTest provides good value as a protocol for discovering and executing tests. In addition, we can still make use of this boolean output to indicate if the benchmark had validation errors that caused them to fail to run.
+
+## Caveats and things to know
+- The VSTest adapter will not call your application's entry point.
+  - If you use the entry point to customize how your benchmarks are run, you will need to do this through other means such as an assembly-level `IConfigSource`.
+  - For more about this, please read: [Setting a default configuration](#setting-a-default-configuration).
+- The benchmark measurements may be affected by the VSTest host and your IDE 
+  - If you want to have more accurate performance results, it is recommended to run benchmarks through the CLI instead without other processes on the machine impacting performance.
+  - This does not mean that the measurements are useless though, it will still be able to provide useful measurements during development when comparing different approaches.
+- The test adapter will not display or execute benchmarks if optimizations are disabled.
+  - Please ensure you are compiling in Release mode or with `Optimize` set to true.
+  - Using an `InProcess` toolchain will let you run your benchmarks with optimizations disabled and will let you attach the debugger as well.
+- The test adapter will generate an entry point for you automatically
+  - The generated entry point will pass the command line arguments and the current assembly into `BenchmarkSwitcher`, so you can still use it in your CLI as well as in VSTest.
+  - This means you can delete your entry point and only need to define your benchmarks.
+  - If you want to use a custom entry point, you can still do so by setting `GenerateProgramFile` to `false` in your project file.
+
+## How to use it
+
+You need to install two packages into your benchmark project:
+
+- `BenchmarkDotNet.TestAdapter`: Implements the VSTest protocol for BenchmarkDotNet
+- `Microsoft.NET.Test.Sdk`: Includes all the pieces needed for the VSTest host to run and load the VSTest adapter.
+
+As mentioned in the caveats section, `BenchmarkDotNet.TestAdapter` will generate an entry point for you automatically, so if you have an entry point already you will either need to delete it or set `GenerateProgramFile` to `false` in your project file to continue using your existing one.
+
+After doing this, you can set your build configuration to `Release`, run a build, and you should be able to see the benchmarks in your IDE's VSTest integration.
+
+## Setting a default configuration
+
+Previously, it was common for the default configuration to be defined inside the entry point. Since the entry point is not used when running benchmarks through VSTest, the default configuration must be specified using a `Config` attribute instead that is set on the assembly. 
+
+First, create a class that extends `ManualConfig` or `IConfig` which sets the default configuration you want:
+
+```csharp
+class MyDefaultConfig : ManualConfig
+{
+    public MyDefaultConfig()
+    {
+        AddJob(Job.Dry);
+        AddLogger(Loggers.ConsoleLogger.Default); 
+        AddValidator(JitOptimizationsValidator.DontFailOnError); 
+    }
+}
+```
+
+Then, set an assembly attribute with the following.
+
+```csharp
+[assembly: Config(typeof(MyDefaultConfig))]
+```
+
+By convention, assembly attributes are usually defined inside `AssemblyInfo.cs` in a directory called `Properties`.
+
+## Viewing the results
+The full output from BenchmarkDotNet that you would have been used to seeing in the past will be sent to the "Tests" output of your IDE. Use this view if you want to see the tabular view that compares multiple benchmarks with each other, or if you want to see the results for each individual iteration.
+
+One more place where you can view the results is in each individual test's output messages. In Visual Studio this can be viewed by clicking on the test in the Test Explorer after running it, and looking at the Test Detail Summary. Since this only displays statistics for a single benchmark case, it does not show the tabulated view that compares multiple benchmark cases, but instead displays a histogram and various other useful statistics. Not all IDEs support displaying these output messages, so you may only be able to view the results using the "Tests" output.

docs/images/vs-testexplorer-demo.png

@@ -6,12 +6,14 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net462;net8.0</TargetFrameworks>
+    <GenerateProgramFile>false</GenerateProgramFile>
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="Program.fs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
+    <ProjectReference Include="..\..\src\BenchmarkDotNet.TestAdapter\BenchmarkDotNet.TestAdapter.csproj"/>
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
@@ -21,5 +23,6 @@
   <ItemGroup>
     <PackageReference Update="FSharp.Core" Version="4.6.0" />
     <PackageReference Update="System.ValueTuple" Version="4.5.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
   </ItemGroup>
 </Project>

samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj

@@ -11,6 +11,8 @@
     <PlatformTarget>AnyCPU</PlatformTarget>
     <DebugSymbols>true</DebugSymbols>
     <NoWarn>$(NoWarn);CA1018;CA5351;CA1825</NoWarn>
+    <!-- Disable entry point generation as this project has it's own entry point -->
+    <GenerateProgramFile>false</GenerateProgramFile>
   </PropertyGroup>
   <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
     <Reference Include="System.Reflection" />
@@ -19,10 +21,13 @@
   <ItemGroup>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="System.Drawing.Common" Version="4.7.2" />
+    <!-- The Test SDK is required only for the VSTest Adapter to work -->
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\BenchmarkDotNet.Diagnostics.dotTrace\BenchmarkDotNet.Diagnostics.dotTrace.csproj" />
     <ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
     <ProjectReference Include="..\..\src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" />
+    <ProjectReference Include="..\..\src\BenchmarkDotNet.TestAdapter\BenchmarkDotNet.TestAdapter.csproj" />
   </ItemGroup>
 </Project>

samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj

@@ -0,0 +1,94 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Running;
+using Microsoft.TestPlatform.AdapterUtilities;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using System;
+
+namespace BenchmarkDotNet.TestAdapter
+{
+    /// <summary>
+    /// A set of extensions for BenchmarkCase to support converting to VSTest TestCase objects.
+    /// </summary>
+    internal static class BenchmarkCaseExtensions
+    {
+        /// <summary>
+        /// Converts a BDN BenchmarkCase to a VSTest TestCase.
+        /// </summary>
+        /// <param name="benchmarkCase">The BenchmarkCase to convert.</param>
+        /// <param name="assemblyPath">The dll or exe of the benchmark project.</param>
+        /// <param name="includeJobInName">Whether or not the display name should include the job name.</param>
+        /// <returns>The VSTest TestCase.</returns>
+        internal static TestCase ToVsTestCase(this BenchmarkCase benchmarkCase, string assemblyPath, bool includeJobInName = false)
+        {
+            var benchmarkMethod = benchmarkCase.Descriptor.WorkloadMethod;
+            var fullClassName = benchmarkCase.Descriptor.Type.GetCorrectCSharpTypeName();
+            var benchmarkMethodName = benchmarkCase.Descriptor.WorkloadMethod.Name;
+            var benchmarkFullMethodName = $"{fullClassName}.{benchmarkMethodName}";
+
+            // Display name has arguments as well.
+            var displayMethodName = FullNameProvider.GetMethodName(benchmarkCase);
+            if (includeJobInName)
+                displayMethodName += $" [{benchmarkCase.GetUnrandomizedJobDisplayInfo()}]";
+
+            var displayName = $"{fullClassName}.{displayMethodName}";
+
+            var vsTestCase = new TestCase(benchmarkFullMethodName, VsTestAdapter.ExecutorUri, assemblyPath)
+            {
+                DisplayName = displayName,
+                Id = GetTestCaseId(benchmarkCase)
+            };
+
+            var benchmarkAttribute = benchmarkMethod.ResolveAttribute<BenchmarkAttribute>();
+            if (benchmarkAttribute != null)
+            {
+                vsTestCase.CodeFilePath = benchmarkAttribute.SourceCodeFile;
+                vsTestCase.LineNumber = benchmarkAttribute.SourceCodeLineNumber;
+            }
+
+            var categories = DefaultCategoryDiscoverer.Instance.GetCategories(benchmarkMethod);
+            foreach (var category in categories)
+                vsTestCase.Traits.Add("Category", category);
+
+            vsTestCase.Traits.Add("", "BenchmarkDotNet");
+
+            return vsTestCase;
+        }
+
+        /// <summary>
+        /// If an ID is not provided, a random string is used for the ID. This method will identify if randomness was
+        /// used for the ID and return the Job's DisplayInfo with that randomness removed so that the same benchmark
+        /// can be referenced across multiple processes.
+        /// </summary>
+        /// <param name="benchmarkCase">The benchmark case.</param>
+        /// <returns>The benchmark case' job's DisplayInfo without randomness.</returns>
+        internal static string GetUnrandomizedJobDisplayInfo(this BenchmarkCase benchmarkCase)
+        {
+            var jobDisplayInfo = benchmarkCase.Job.DisplayInfo;
+            if (!benchmarkCase.Job.HasValue(CharacteristicObject.IdCharacteristic) && benchmarkCase.Job.ResolvedId.StartsWith("Job-", StringComparison.OrdinalIgnoreCase))
+            {
+                // Replace Job-ABCDEF with Job
+                jobDisplayInfo = "Job" + jobDisplayInfo.Substring(benchmarkCase.Job.ResolvedId.Length);
+            }
+
+            return jobDisplayInfo;
+        }
+
+        /// <summary>
+        /// Gets an ID for a given BenchmarkCase that is uniquely identifiable from discovery to execution phase.
+        /// </summary>
+        /// <param name="benchmarkCase">The benchmark case.</param>
+        /// <returns>The test case ID.</returns>
+        internal static Guid GetTestCaseId(this BenchmarkCase benchmarkCase)
+        {
+            var testIdProvider = new TestIdProvider();
+            testIdProvider.AppendString(VsTestAdapter.ExecutorUriString);
+            testIdProvider.AppendString(benchmarkCase.Descriptor.DisplayInfo);
+            testIdProvider.AppendString(benchmarkCase.GetUnrandomizedJobDisplayInfo());
+            testIdProvider.AppendString(benchmarkCase.Parameters.DisplayInfo);
+            return testIdProvider.GetId();
+        }
+    }
+}

src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\build\common.props" />
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0;net462</TargetFrameworks>
+    <AssemblyTitle>BenchmarkDotNet.TestAdapter</AssemblyTitle>
+    <AssemblyName>BenchmarkDotNet.TestAdapter</AssemblyName>
+    <PackageId>BenchmarkDotNet.TestAdapter</PackageId>
+    <ProduceReferenceAssembly>True</ProduceReferenceAssembly>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.TestPlatform.AdapterUtilities" Version="17.7.2" />
+    <PackageReference Include="Microsoft.TestPlatform.ObjectModel" Version="17.7.2" />
+    <PackageReference Include="Microsoft.TestPlatform.TranslationLayer" Version="17.7.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\BenchmarkDotNet\BenchmarkDotNet.csproj" />
+  </ItemGroup>
+
+  <!-- Include files in nuget package for generating entry point -->
+  <ItemGroup>
+    <Compile Remove="Package\EntryPoint.*" />
+    <None Include="Package\EntryPoint.*" Pack="true" PackagePath="entrypoints\" />
+    <None Include="Package\BenchmarkDotNet.TestAdapter.props" Pack="true" PackagePath="build\" />
+  </ItemGroup>
+</Project>

src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj

@@ -0,0 +1,48 @@
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Helpers;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+
+namespace BenchmarkDotNet.TestAdapter
+{
+    /// <summary>
+    /// A class used for enumerating all the benchmarks in an assembly.
+    /// </summary>
+    internal static class BenchmarkEnumerator
+    {
+        /// <summary>
+        /// Returns all the BenchmarkRunInfo objects from a given assembly.
+        /// </summary>
+        /// <param name="assemblyPath">The dll or exe of the benchmark project.</param>
+        /// <returns>The benchmarks inside the assembly.</returns>
+        public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPath)
+        {
+            var assembly = Assembly.LoadFrom(assemblyPath);
+
+            var isDebugAssembly = assembly.IsJitOptimizationDisabled() ?? false;
+
+            return GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks())
+                .Select(type =>
+                {
+                    var benchmarkRunInfo = BenchmarkConverter.TypeToBenchmarks(type);
+                    if (isDebugAssembly)
+                    {
+                        // If the assembly is a debug assembly, then only display them if they will run in-process
+                        // This will allow people to debug their benchmarks using VSTest if they wish.
+                        benchmarkRunInfo = new BenchmarkRunInfo(
+                            benchmarkRunInfo.BenchmarksCases.Where(c => c.GetToolchain().IsInProcess).ToArray(),
+                            benchmarkRunInfo.Type,
+                            benchmarkRunInfo.Config);
+                    }
+
+                    return benchmarkRunInfo;
+                })
+                .Where(runInfo => runInfo.BenchmarksCases.Length > 0)
+                .ToArray();
+        }
+    }
+}