Skip to content

Commit d94b0a5

Browse files
neueccneuecc
authored
Merge pull request #168 from filzrev/chore-update-nuget-versions-benchmarks
chore: Update NuGet versions benchmark
2 parents c5650c5 + b6afc38 commit d94b0a5

15 files changed

+125
-103
lines changed

‎sandbox/Benchmark/Benchmark.csproj‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818

1919
<!-- Add .NET 10 support when running build inside Visual Studio Preview-->
2020
<PropertyGroup Condition="'$(BuildingInsideVisualStudio)' != '' AND $(VisualStudioDir.Contains('Preview'))">
21-
<!-- TODO: Temporary comment out, because BenchmarkDotNet 0.14.0 don't support .NET 10 -->
22-
<!--<TargetFrameworks>net10.0;$(TargetFrameworks)</TargetFrameworks>-->
21+
<TargetFrameworks>net10.0;$(TargetFrameworks)</TargetFrameworks>
2322
</PropertyGroup>
2423

2524
<ItemGroup>
@@ -32,8 +31,8 @@
3231

3332
<ItemGroup>
3433
<PackageReference Include="BenchmarkDotNet" Version="0.15.0" />
35-
<PackageReference Include="Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers" Version="17.14.36021.1" />
36-
<PackageReference Include="Microsoft.VisualStudio.DiagnosticsHub.UserMarks" Version="17.14.36021.1" />
34+
<PackageReference Include="Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers" Version="18.0.36127.1" />
35+
<PackageReference Include="Microsoft.VisualStudio.DiagnosticsHub.UserMarks" Version="18.0.36127.1" />
3736
<PackageReference Include="Kokuban" Version="0.2.0" />
3837
</ItemGroup>
3938

‎sandbox/Benchmark/BenchmarkDotNet/BenchmarkConfigs/BaseBenchmarkConfig.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,8 @@ protected virtual void AddFilters()
8585
protected virtual void AddDiagnosers()
8686
{
8787
AddDiagnoser(MemoryDiagnoser.Default);
88-
89-
// TODO: Enable following diagnoser (Temporary disabled because extra columns are shown it has no data)
90-
//AddDiagnoser(ExceptionDiagnoser.Default);
91-
//AddDiagnoser(ThreadingDiagnoser.Default);
88+
AddDiagnoser(new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false)));
89+
AddDiagnoser(new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false)));
9290
}
9391

9492
protected virtual void AddValidators()

‎sandbox/Benchmark/BenchmarkDotNet/BenchmarkConfigs/NuGetVersionsBenchmarkConfig.cs‎

Lines changed: 37 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,8 @@ public NuGetVersionsBenchmarkConfig() : base()
1616
.WithToolchain(Constants.DefaultToolchain)
1717
.Freeze();
1818

19-
string[] targetNuGetVersions = [GetCurrentZLinqVersion()];
20-
21-
// Note: Enable following code when comparing multiple ZLinq versions.
22-
// targetNuGetVersions = GetTargetZlinqVersions();
19+
// Compare LocalBuild version with latest NuGet package version.
20+
var targetNuGetVersions = GetTargetZlinqVersions();
2321

2422
// 1. Add jobs that use ZLinq NuGet package with specified versions
2523
foreach (var targetVersion in targetNuGetVersions)
@@ -29,9 +27,7 @@ public NuGetVersionsBenchmarkConfig() : base()
2927
.WithArguments(
3028
[
3129
new MsBuildArgument($"/p:{Constants.MSBuildProperties.TargetZLinqVersion}={targetVersion}"),
32-
new MsBuildArgument($"/p:{Constants.MSBuildProperties.ZLinqDefineConstants}={GetDefineConstantsValue(targetVersion)}"),
33-
// TODO: Enable following code and remove settings from csproj after .NET SDK issue is resolved. See:https://github.com/dotnet/sdk/issues/45638
34-
// new MsBuildArgument($"/p:DefineConstants={GetDefineConstantsValue(targetVersion)}"),
30+
new MsBuildArgument($"/p:{Constants.MSBuildProperties.ZLinqDefineConstants}=\\\"{GetZLinqDefineConstants(targetVersion)}\\\""), // Surround value with `\"` to use list separator char without escaping.
3531
])
3632
.WithId($"v{targetVersion}");
3733

@@ -42,8 +38,8 @@ public NuGetVersionsBenchmarkConfig() : base()
4238
AddJob(job);
4339
}
4440

45-
// 2. Add LocalBuild job.
46-
if (targetNuGetVersions.Length == 1)
41+
// 2. Add LocalBuild job
42+
if (targetNuGetVersions.Count() == 1)
4743
AddJob(baseJobConfig.WithId("vLocalBuild")); // Add `v` prefix to change display order.
4844

4945
// Configure additional settings
@@ -57,18 +53,33 @@ protected override void AddColumnHidingRules()
5753
HideColumns(Column.BuildConfiguration);
5854
}
5955

56+
protected override void AddFilters()
57+
{
58+
base.AddFilters();
59+
AddFilter(ZLinqNuGetVersionFilter.Instance);
60+
}
61+
62+
protected override void AddLogicalGroupRules()
63+
{
64+
AddLogicalGroupRules(
65+
[
66+
BenchmarkLogicalGroupRule.ByCategory,
67+
BenchmarkLogicalGroupRule.ByMethod,
68+
]);
69+
}
70+
6071
/// <summary>
6172
/// Gets DefineConstants settings for using target ZLinq version.
6273
/// </summary>
63-
private static string GetDefineConstantsValue(string versionText)
74+
private static string GetZLinqDefineConstants(string versionText)
6475
{
65-
// Currently it need to escape MSBuild special characters (https://learn.microsoft.com/en-us/visualstudio/msbuild/msbuild-special-characters)
66-
// Because MSBuildArgument is passed as commandline parameter.
67-
// See: https://github.com/dotnet/BenchmarkDotNet/issues/2719
68-
const string ListSeparator = "%3B"; // Escapec semicolon char.
76+
const string ListSeparator = ";";
6977

7078
StringBuilder sb = new();
7179

80+
// Add custom symbol to distinguish NuGet package build.
81+
sb.Append(Constants.DefineConstants.USE_ZLINQ_NUGET_PACKAGE);
82+
7283
// Add target package version symbol
7384
sb.Append($"{ListSeparator}ZLINQ_{versionText.Replace('.', '_')}");
7485

@@ -90,48 +101,22 @@ private static string GetDefineConstantsValue(string versionText)
90101
return sb.ToString();
91102
}
92103

93-
private static string GetCurrentZLinqVersion()
104+
private static IEnumerable<string> GetTargetZlinqVersions()
94105
{
95-
DirectoryInfo? dirInfo = new DirectoryInfo(AppContext.BaseDirectory);
96-
while (true)
97-
{
98-
if (dirInfo.GetFiles().Any(x => x.Name == "ZLinq.slnx"))
99-
break;
100-
101-
dirInfo = dirInfo.Parent;
102-
if (dirInfo == null)
103-
throw new FileNotFoundException("ZLinq.slnx is not found.");
104-
}
106+
// Download availabe ZLinq package versions.
107+
const string url = "https://api.nuget.org/v3-flatcontainer/zlinq/index.json"; // Must be lower-case.
108+
var json = new HttpClient().GetStringAsync(url).GetAwaiter().GetResult();
105109

106-
var solutionDir = dirInfo.FullName;
107-
var resolvedPath = Path.Combine(solutionDir, "src/ZLinq.Unity/Assets/ZLinq.Unity/package.json");
108-
if (!File.Exists(resolvedPath) && !Directory.Exists(resolvedPath))
109-
throw new FileNotFoundException($"File is not found. path: {resolvedPath}");
110+
// Note: Uncomment following line when comparing all NuGet package versions performance.
111+
// return versions;
110112

111-
var json = File.ReadAllText(resolvedPath);
112-
var latestVersion = JsonNode.Parse(json)!["version"]!.GetValue<string>();
113-
return latestVersion;
114-
}
113+
var node = JsonNode.Parse(json)!;
114+
var versions = node["versions"]!.AsArray().GetValues<string>().ToArray();
115115

116-
private static string[] GetTargetZlinqVersions()
117-
{
118-
// Currently multi NuGet versions benchmark is not supported.
119-
// It require following BenchmarkDotNet feature.
120-
// https://github.com/dotnet/BenchmarkDotNet/pull/2676
121-
throw new NotSupportedException("Currently multi NuGet versions benchmark is not supported.");
122-
123-
// Available package versions: https://api.nuget.org/v3-flatcontainer/ZLinq/index.json
124-
////return
125-
////[
126-
//// "1.0.0",
127-
//// "1.1.0",
128-
//// "1.2.0",
129-
//// "1.3.0",
130-
//// "1.3.1",
131-
//// "1.4.0",
132-
//// "1.4.1",
133-
//// "1.4.2",
134-
////];
116+
bool isRunningOnGitHubActions = Environment.GetEnvironmentVariable("GITHUB_ACTIONS") == "true";
117+
return isRunningOnGitHubActions
118+
? versions.TakeLast(2) // Compare performance between latest 2 versions.
119+
: versions.TakeLast(1); // Compare performance between latest/LocalBuild versions.
135120
}
136121

137122
private static string GetCustomBuildConfigurationName(string targetVersion)

‎sandbox/Benchmark/BenchmarkDotNet/BenchmarkConfigs/SystemLinqBenchmarkConfig.cs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,6 @@ public SystemLinqBenchmarkConfig() : base()
2020
// Add job for System.Linq benchmarks.
2121
AddJob(baseJobConfig.WithToolchain(Constants.DefaultToolchain)
2222
.WithCustomBuildConfiguration(Constants.CustomBuildConfigurations.SystemLinq)
23-
// TODO: Enable following code and remove settings from csproj after .NET SDK issue is resolved. See:https://github.com/dotnet/sdk/issues/45638
24-
//.WithArguments(
25-
//[
26-
// new MsBuildArgument("/p:DefineConstants=USE_SYSTEM_LINQ")
27-
//])
2823
.WithId(SystemLinqJobId)
2924
.AsBaseline());
3025

‎sandbox/Benchmark/BenchmarkDotNet/EventProcessors/BenchmarkEventProcessor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public override void OnBuildComplete(BuildPartition partition, BuildResult build
6868

6969
if (buildResult.GenerateException != null)
7070
Logger.WriteLineError($"{buildResult.GenerateException.ToString()}");
71-
71+
7272
return;
7373
}
7474

‎sandbox/Benchmark/BenchmarkDotNet/ExtensionMethods/ConsumerExtensions.UnaryOperations.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,14 @@ public static void Consume<T>(this ValueEnumerable<WhereArray<T>, T> source, Con
333333
}
334334
#endif
335335

336+
// Where FromList
337+
public static void Consume<T>(this ValueEnumerable<Where<FromList<T>, T>, T> source, Consumer consumer)
338+
{
339+
using var e = source.Enumerator;
340+
while (e.TryGetNext(out var item))
341+
consumer.Consume(in item);
342+
}
343+
336344
// Where FromEnumerable
337345
public static void Consume<T>(this ValueEnumerable<Where<FromEnumerable<T>, T>, T> source, Consumer consumer)
338346
{

‎sandbox/Benchmark/BenchmarkDotNet/Filters/ZLinqNuGetVersionFilter.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
using BenchmarkDotNet.Filters;
2+
using BenchmarkDotNet.Running;
3+
4+
namespace Benchmark;
5+
6+
/// <summary>
7+
/// Custom BenchmarkDotNet filter to exclude benchmarks based on ZLinq version category.
8+
/// </summary>
9+
public partial class ZLinqNuGetVersionFilter : IFilter
10+
{
11+
public static readonly ZLinqNuGetVersionFilter Instance = new();
12+
13+
public virtual bool Predicate(BenchmarkCase benchmarkCase)
14+
{
15+
var jobId = benchmarkCase.Job.Id;
16+
if (jobId == "vLocalBuild")
17+
return true;
18+
19+
var version = Version.Parse(jobId.Substring(1)); // Gets version from JobId (Remove `v` prefix)
20+
var categories = benchmarkCase.Descriptor.Categories;
21+
foreach (var category in categories.OrderDescending())
22+
{
23+
switch (category)
24+
{
25+
case Categories.Filters.ZLINQ_1_4_0_OR_GREATER:
26+
return version >= Version.Parse("1.4.0");
27+
case Categories.Filters.ZLINQ_1_3_1_OR_GREATER:
28+
return version >= Version.Parse("1.3.1");
29+
case Categories.Filters.ZLINQ_1_2_0_OR_GREATER:
30+
return version >= Version.Parse("1.2.0");
31+
default:
32+
continue;
33+
}
34+
}
35+
36+
return true;
37+
}
38+
}

‎sandbox/Benchmark/Benchmarks/ZLinq/MicroBenchmarks/Sinks/ToImmutableArrayBenchmark.cs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,22 @@
1-
using System.Buffers;
2-
using ZLinq;
1+
using ZLinq;
32

43
namespace Benchmark.ZLinq;
54

6-
#if !USE_ZLINQ_NUGET_PACKAGE || ZLINQ_1_2_0_OR_GREATER
7-
#if NET8_0_OR_GREATER
85
[BenchmarkCategory(Categories.Methods.ToImmutableArray)]
6+
[BenchmarkCategory(Categories.Filters.NET8_0_OR_GREATER)]
7+
[BenchmarkCategory(Categories.Filters.ZLINQ_1_2_0_OR_GREATER)]
98
public partial class ToImmutableArrayBenchmark<T> : EnumerableBenchmarkBase_WithBasicTypes<T>
109
{
10+
#if NET8_0_OR_GREATER
11+
#if !USE_ZLINQ_NUGET_PACKAGE || ZLINQ_1_2_0_OR_GREATER
1112
[Benchmark]
1213
[BenchmarkCategory(Categories.From.Default)]
13-
[BenchmarkCategory(Categories.Filters.NET8_0_OR_GREATER)]
14-
[BenchmarkCategory(Categories.Filters.ZLINQ_1_2_0_OR_GREATER)]
1514
public void ToImmutableArray()
1615
{
1716
_ = source.Default
1817
.AsValueEnumerable()
1918
.ToImmutableArray();
2019
}
21-
}
2220
#endif
2321
#endif
22+
}

‎sandbox/Benchmark/Benchmarks/ZLinq/MicroBenchmarks/Sinks/ToImmutableDictionaryBenchmark.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,23 @@
22

33
namespace Benchmark.ZLinq;
44

5-
#if !USE_ZLINQ_NUGET_PACKAGE || ZLINQ_1_2_0_OR_GREATER
6-
#if NET8_0_OR_GREATER
75
[BenchmarkCategory(Categories.Methods.ToImmutableDictionary)]
6+
[BenchmarkCategory(Categories.Filters.NET8_0_OR_GREATER)]
7+
[BenchmarkCategory(Categories.Filters.ZLINQ_1_2_0_OR_GREATER)]
88
public partial class ToImmutableDictionaryBenchmark<T> : EnumerableBenchmarkBase_WithBasicTypes<T>
99
where T : notnull
1010
{
11+
#if NET8_0_OR_GREATER
12+
#if !USE_ZLINQ_NUGET_PACKAGE || ZLINQ_1_2_0_OR_GREATER
1113
[Benchmark]
12-
[BenchmarkCategory(Categories.From.Default)]
13-
[BenchmarkCategory(Categories.Filters.NET8_0_OR_GREATER)]
14-
[BenchmarkCategory(Categories.Filters.ZLINQ_1_2_0_OR_GREATER)]
14+
[BenchmarkCategory(Categories.From.Default)]
1515
public void ToImmutableDictionary()
1616
{
1717
_ = source.Default
1818
.AsValueEnumerable()
1919
.Distinct()
2020
.ToImmutableDictionary(x => x, x => x);
2121
}
22-
}
2322
#endif
2423
#endif
24+
}

‎sandbox/Benchmark/Benchmarks/ZLinq/MicroBenchmarks/Sinks/ToImmutableHashSetBenchmark.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,21 @@
22

33
namespace Benchmark.ZLinq;
44

5-
#if !USE_ZLINQ_NUGET_PACKAGE || ZLINQ_1_2_0_OR_GREATER
6-
#if NET8_0_OR_GREATER
75
[BenchmarkCategory(Categories.Methods.ToImmutableHashSet)]
6+
[BenchmarkCategory(Categories.Filters.NET8_0_OR_GREATER)]
7+
[BenchmarkCategory(Categories.Filters.ZLINQ_1_2_0_OR_GREATER)]
88
public partial class ToImmutableHashSetBenchmark<T> : EnumerableBenchmarkBase_WithBasicTypes<T>
99
{
10+
#if NET8_0_OR_GREATER
11+
#if !USE_ZLINQ_NUGET_PACKAGE || ZLINQ_1_2_0_OR_GREATER
1012
[Benchmark]
1113
[BenchmarkCategory(Categories.From.Default)]
12-
[BenchmarkCategory(Categories.Filters.NET8_0_OR_GREATER)]
13-
[BenchmarkCategory(Categories.Filters.ZLINQ_1_2_0_OR_GREATER)]
1414
public void ToImmutableHashSet()
1515
{
1616
_ = source.Default
1717
.AsValueEnumerable()
1818
.ToImmutableHashSet();
1919
}
20-
}
2120
#endif
2221
#endif
22+
}

0 commit comments

Comments
 (0)