Update bdn 0.16 binarydata and async

docs/benchmarkdotnet.md

@@ -27,7 +27,7 @@
     - [Running In Process](#running-in-process)
     - [CoreRun](#corerun)
     - [dotnet cli](#dotnet-cli)
     - [Private CLR Build](#private-clr-build)
     - [Private CoreRT Build](#private-corert-build)
 
 ## Main Concepts
@@ -289,7 +289,7 @@
 
 The `--runtimes` or just `-r` allows you to run the benchmarks for **multiple Runtimes**.
 
-Available options are: Mono, wasmnet70, CoreRT, net462, net47, net471, net472, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0.
+Available options are: Mono, wasmnet70, CoreRT, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0.
 
 Example: run the benchmarks for .NET 7.0 and 8.0:
 
@@ -361,18 +361,6 @@
 
 This is very useful when you want to compare different builds of .NET.
 
-### Private CLR Build
-
-It's possible to benchmark a private build of .NET Runtime. You just need to pass the value of `COMPLUS_Version` to BenchmarkDotNet. You can do that by either using `--clrVersion $theVersion` as an argument or `Job.ShortRun.With(new ClrRuntime(version: "$theVersion"))` in the code.
-
-So if you made a change in CLR and want to measure the difference, you can run the benchmarks with:
-
-```cmd
-dotnet run -c Release -f net48 -- --clrVersion $theVersion
-```
-
-More info can be found in [BenchmarkDotNet issue #706](https://github.com/dotnet/BenchmarkDotNet/issues/706).
-
 ### Private CoreRT Build
 
 To run benchmarks with private CoreRT build you need to provide the `IlcPath`. Example:

eng/Versions.props

@@ -11,7 +11,7 @@
     <MicrosoftNETILLinkPackageVersion>11.0.0-preview.4.26210.111</MicrosoftNETILLinkPackageVersion>
     <SystemThreadingChannelsPackageVersion>11.0.0-preview.4.26210.111</SystemThreadingChannelsPackageVersion>
     <MicrosoftExtensionsLoggingPackageVersion>11.0.0-preview.4.26210.111</MicrosoftExtensionsLoggingPackageVersion>
-    <BenchmarkDotNetVersion>0.16.0-nightly.20260320.467</BenchmarkDotNetVersion>
+    <BenchmarkDotNetVersion>0.16.0-nightly.20260504.1025</BenchmarkDotNetVersion>
     <MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>11.0.0-preview.4.26210.111</MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>
     <MicrosoftDotNetXHarnessCLIVersion>11.0.0-prerelease.26204.1</MicrosoftDotNetXHarnessCLIVersion>
   </PropertyGroup>

scripts/ci_setup.py

@@ -424,6 +424,11 @@ def main(args: CiSetupArgs):
     if args.experiment_name == "jitoptrepeat":
         experiment_config = variable_format % ('DOTNET_JitOptRepeat', '*')
 
+    if args.experiment_name == "runtimeasync":
+        # Surfaced to MSBuild as the $(EnableRuntimeAsync) property; gates the
+        # runtime-async Features flag in src/Directory.Build.targets.
+        experiment_config = variable_format % ('EnableRuntimeAsync', 'true')
+
     output = ''
 
     with push_dir(get_repo_root_path()):

src/Directory.Build.props

@@ -5,27 +5,27 @@
     <!-- Avoid spawning any-long living compiler processes to avoid BenchmarkDotNet issues with "file in use",
          and automation failing to clean up the folders once the runs are over -->
     <UseSharedCompilation>false</UseSharedCompilation>
-    
+
     <!-- Use the latest C# compiler features -->
     <LangVersion>latest</LangVersion>
-    
+
     <!-- This repo does not produce any libraries, therefore generating docs is disabled -->
     <GenerateDocumentationFile>False</GenerateDocumentationFile>
-    
+
     <!-- every warning is important, we want to enforce best practices here to keep high quality of the code and avoid common mistakes -->
     <NoWarn>$(NoWarn);NU1507</NoWarn> <!-- Darc does not seem to support auto updating of packageSourceMapping causing tool publishes to fail without manual updating of the NuGet.config with package mappings, disable this check so we don't have to manually update the source mappings. -->
     <NoWarn>$(NoWarn);NETSDK1138</NoWarn> <!-- Disable warning about EOL target frameworks as we target them to test them -->
     <NoWarn>$(NoWarn);CS9057</NoWarn> <!-- Suppress analyzer version mismatch when BDN analyzers target a newer Roslyn than the SDK provides -->
     <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
     <WarningLevel>4</WarningLevel>
-    
+
     <!-- we are testing latest bits and we must use preview SDK version -->
     <SuppressNETCoreSdkPreviewMessage>True</SuppressNETCoreSdkPreviewMessage>
 
     <!-- Disable SourceLink -->
     <EnableSourceLink>false</EnableSourceLink>
     <EnableSourceControlManagerQueries>false</EnableSourceControlManagerQueries>
-    
+
     <!-- Explicit disable signing the built assemblies here to stop Arcade attempting to sign them -->
     <SignAssembly>false</SignAssembly>
   </PropertyGroup>

src/Directory.Build.targets

@@ -0,0 +1,11 @@
+<Project>
+    <Import Project="../Directory.Build.targets" />
+
+    <PropertyGroup>
+      <!-- Runtime-async is gated behind the 'runtimeasync' experiment so it only runs
+           in the dedicated experiment lane. ci_setup.py sets EnableRuntimeAsync=true as
+           an env var (which MSBuild surfaces as a property) when the experiment name
+           is 'runtimeasync'. -->
+      <Features Condition="'$(EnableRuntimeAsync)' == 'true' and $([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net11.0'))">$(Features);runtime-async=on</Features>
+    </PropertyGroup>
+</Project>

src/benchmarks/micro/MicroBenchmarks.csproj

@@ -122,14 +122,6 @@
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(ExtensionsVersion)" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="$(ExtensionsVersion)" />
   </ItemGroup>
-  <!-- These package versions are only for the opt-in net472 path (for example, PERFLAB_TARGET_FRAMEWORKS=net472). -->
-  <ItemGroup Condition="'$(TargetFramework)' != '' and '$(TargetFrameworkIdentifier)' == '.NETFramework'">
-    <PackageReference Include="System.Memory" Version="4.6.3" />
-    <PackageReference Include="System.Numerics.Vectors" Version="4.6.1" />
-    <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.6.3" />
-    <PackageReference Include="System.IO.Pipelines" Version="10.0.3" />
-    <PackageReference Include="System.Threading.Channels" Version="$(SystemVersion)" />
-  </ItemGroup>
   <ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0')) and !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net10.0'))">
     <!-- Newer System.Numerics.Tensors packages require net10+, so keep net8.0/net9.0 on the last compatible package line. -->
     <PackageReference Include="System.Numerics.Tensors" Version="10.0.*-*" />

src/benchmarks/micro/Program.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Threading.Tasks;
 using BenchmarkDotNet.Running;
 using System.IO;
 using BenchmarkDotNet.Extensions;
@@ -14,7 +15,7 @@ namespace MicroBenchmarks
 {
     class Program
     {
-        static int Main(string[] args)
+        static async Task<int> Main(string[] args)
         {
             var argsList = new List<string>(args);
             int? partitionCount;
@@ -40,9 +41,14 @@ static int Main(string[] args)
                 return 1;
             }
 
-            return BenchmarkSwitcher
+            // Use RunAsync (not Run) so BDN does not install its single-threaded
+            // BenchmarkDotNetSynchronizationContext on the entrypoint thread. The sync
+            // entrypoint installs that context before benchmark discovery, which
+            // deadlocks any sync-over-async work performed by [ParamsSource]/[ArgumentsSource]
+            // callbacks (e.g. SslStreamTests.GetTls13Support).
+            var summaries = await BenchmarkSwitcher
                 .FromAssembly(typeof(Program).Assembly)
-                .Run(argsList.ToArray(), 
+                .RunAsync(argsList.ToArray(),
                     RecommendedConfig.Create(
                         artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")), 
                         mandatoryCategories: ImmutableHashSet.Create([Categories.Libraries, Categories.Runtime, Categories.ThirdParty, Categories.Sve]),
@@ -52,7 +58,9 @@ static int Main(string[] args)
                         categoryExclusionFilterValue: categoryExclusionFilterValue,
                         getDiffableDisasm: getDiffableDisasm)
                     .AddValidator(new NoWasmValidator(Categories.NoWASM)))
-                .ToExitCode();
+                .ConfigureAwait(false);
+
+            return summaries.ToExitCode();
         }
     }
 }

src/benchmarks/micro/README.md

@@ -12,7 +12,7 @@ To learn more about designing benchmarks, please read [Microbenchmark Design Gui
 
 ## Quick Start
 
-The first thing that you need to choose is the Target Framework. Available options are: `netcoreapp3.1|net6.0|net7.0|net8.0|net9.0|net10.0|net11.0|net472`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework.
+The first thing that you need to choose is the Target Framework. Available options are: `net8.0|net9.0|net10.0|net11.0`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework.
 
 The following commands are run from the `src/benchmarks/micro` directory.
 

src/benchmarks/micro/Serializers/DataGenerator.cs

@@ -32,7 +32,7 @@ internal static T Generate<T>()
                 return (T)(object)CreateIndexViewModel();
             if (typeof(T) == typeof(MyEventsListerViewModel))
                 return (T)(object)CreateMyEventsListerViewModel();
-            if (typeof(T) == typeof(BinaryData))
+            if (typeof(T) == typeof(BinaryDataPayload))
                 return (T)(object)CreateBinaryData(1024);
             if (typeof(T) == typeof(CollectionsOfPrimitives))
                 return (T)(object)CreateCollectionsOfPrimitives(1024); // 1024 values was copied from CoreFX benchmarks
@@ -163,8 +163,8 @@ private static MyEventsListerItem CreateMyEventsListerItem()
                     }, 4).ToList()
             };
 
-        private static BinaryData CreateBinaryData(int size)
-            => new BinaryData
+        private static BinaryDataPayload CreateBinaryData(int size)
+            => new BinaryDataPayload
             {
                 ByteArray = CreateByteArray(size)
             };
@@ -383,7 +383,13 @@ public string FormattedDate
     [Serializable]
     [ProtoContract]
     [MessagePackObject]
-    public class BinaryData
+    // Renamed from BinaryData to avoid name collision with System.BinaryData
+    // (introduced in net6+ via System.Memory.Data, transitive via Azure.Core).
+    // The collision caused [GenericTypeArguments(typeof(BinaryData))] to resolve
+    // to System.BinaryData in some compile contexts (e.g. when the harness targets
+    // net8.0 instead of netstandard2.0), bypassing the data generator and producing
+    // NotImplementedException at runtime.
+    public class BinaryDataPayload
     {
         [ProtoMember(1)] [Key(0)] public byte[] ByteArray { get; set; }
     }
@@ -496,7 +502,7 @@ public enum SystemTextJsonSerializationMode
     [JsonSerializable(typeof(Location))]
     [JsonSerializable(typeof(IndexViewModel))]
     [JsonSerializable(typeof(MyEventsListerViewModel))]
-    [JsonSerializable(typeof(BinaryData))]
+    [JsonSerializable(typeof(BinaryDataPayload))]
     [JsonSerializable(typeof(CollectionsOfPrimitives))]
     [JsonSerializable(typeof(XmlElement))]
     [JsonSerializable(typeof(SimpleStructWithProperties))]

src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs

@@ -18,7 +18,7 @@ namespace System.Text.Json.Serialization.Tests
     [GenericTypeArguments(typeof(Location))]
     [GenericTypeArguments(typeof(IndexViewModel))]
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
-    [GenericTypeArguments(typeof(BinaryData))]
+    [GenericTypeArguments(typeof(BinaryDataPayload))]
     [GenericTypeArguments(typeof(Dictionary<string, string>))]
     [GenericTypeArguments(typeof(ImmutableDictionary<string, string>))]
     [GenericTypeArguments(typeof(ImmutableSortedDictionary<string, string>))]

src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs

@@ -20,7 +20,7 @@ namespace System.Text.Json.Serialization.Tests
     [GenericTypeArguments(typeof(Location))]
     [GenericTypeArguments(typeof(IndexViewModel))]
     [GenericTypeArguments(typeof(MyEventsListerViewModel))]
-    [GenericTypeArguments(typeof(BinaryData))]
+    [GenericTypeArguments(typeof(BinaryDataPayload))]
     [GenericTypeArguments(typeof(Dictionary<string, string>))]
     [GenericTypeArguments(typeof(ImmutableDictionary<string, string>))]
     [GenericTypeArguments(typeof(ImmutableSortedDictionary<string, string>))]

src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj

@@ -1,7 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Library</OutputType>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <!-- net8.0 is the lowest TFM exercised by the perf pipelines (see scripts/channel_map.py).
+         net472 (and other netfx) consumers are no longer supported; the harness uses BDN APIs
+         (e.g. IValidator.ValidateAsync returning IAsyncEnumerable<T>) that would otherwise require
+         polyfill packages on netstandard2.0. -->
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ProduceReferenceAssembly>true</ProduceReferenceAssembly>
   </PropertyGroup>
@@ -13,7 +17,7 @@
     <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="$(BenchmarkDotNetVersion)" />
   </ItemGroup>
   <ItemGroup Condition="'$(BenchmarkDotNetSources)' != ''">
-    <ProjectReference Include="$(_BenchmarkDotNetSourcesN)src\BenchmarkDotNet\BenchmarkDotNet.csproj" SetTargetFramework="TargetFramework=netstandard2.0" />
+    <ProjectReference Include="$(_BenchmarkDotNetSourcesN)src\BenchmarkDotNet\BenchmarkDotNet.csproj" SetTargetFramework="TargetFramework=net8.0" />
     <ProjectReference Include="$(_BenchmarkDotNetSourcesN)src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" SetTargetFramework="TargetFramework=netstandard2.0" />
   </ItemGroup>
   <ItemGroup>

src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs

@@ -61,26 +61,26 @@ internal static string BuildDisassemblyString(DisassemblyResult disassemblyResul
 
         private static Func<object, T> GetElementGetter<T>(string name)
         {
-            var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier");
+            var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!;
 
-            type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic);
+            type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic)!;
 
-            var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic);
+            var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic)!;
 
-            var method = property.GetGetMethod(nonPublic: true);
+            var method = property.GetGetMethod(nonPublic: true)!;
 
             var generic = typeof(Func<,>).MakeGenericType(type, typeof(T));
 
             var @delegate = method.CreateDelegate(generic);
 
-            return (obj) => (T)@delegate.DynamicInvoke(obj); // cast to (Func<object, T>) throws
+            return (obj) => (T)@delegate.DynamicInvoke(obj)!; // cast to (Func<object, T>) throws
         }
 
         private static Func<DisassembledMethod, DisassemblyResult, DisassemblyDiagnoserConfig, string, IReadOnlyList<object>> GetPrettifyMethod()
         {
-            var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier");
+            var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!;
 
-            var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic);
+            var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic)!;
 
             var @delegate = method.CreateDelegate(typeof(Func<DisassembledMethod, DisassemblyResult, DisassemblyDiagnoserConfig, string, IReadOnlyList<object>>));
 

src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs

@@ -21,7 +21,7 @@ public class MandatoryCategoryValidator : IValidator
 
         public MandatoryCategoryValidator(ImmutableHashSet<string> categories) => _mandatoryCategories = categories;
 
-        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
+        public IAsyncEnumerable<ValidationError> ValidateAsync(ValidationParameters validationParameters)
             => validationParameters.Benchmarks
                 .Where(benchmark => !benchmark.Descriptor.Categories.Any(category => _mandatoryCategories.Contains(category)))
                 .Select(benchmark => benchmark.Descriptor.GetFilterName())
@@ -30,6 +30,7 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
                     new ValidationError(
                         isCritical: TreatsWarningsAsErrors,
                         $"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]")
-                );
+                )
+                .ToAsyncEnumerable();
     }
 }

src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs

@@ -23,7 +23,7 @@ public class NoWasmValidator : IValidator
 
         public NoWasmValidator(string noWasmCategory) => _noWasmCategory = noWasmCategory;
 
-        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
+        public IAsyncEnumerable<ValidationError> ValidateAsync(ValidationParameters validationParameters)
             => validationParameters.Benchmarks
                 .Where(benchmark => IsAsyncMethod(benchmark.Descriptor.WorkloadMethod) && !benchmark.Descriptor.Categories.Any(category => category.Equals(_noWasmCategory, StringComparison.Ordinal)))
                 .Select(benchmark => benchmark.Descriptor.GetFilterName())
@@ -32,7 +32,8 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
                     new ValidationError(
                         isCritical: TreatsWarningsAsErrors,
                         $"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]")
-                );
+                )
+                .ToAsyncEnumerable();
 
         private bool IsAsyncMethod(MethodInfo workloadMethod)
         {