Skip to content

Commit 3062103

Browse files
AndreyAkinshinAndreyAkinshin
committed
Bump Perfolizer: 0.2.1->0.3.16
1 parent 3305b34 commit 3062103

File tree

84 files changed

+1086
-1068
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

84 files changed

+1086
-1068
lines changed

‎BenchmarkDotNet.sln.DotSettings

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OSX/@EntryIndexedValue">OSX</s:String>
5353
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RT/@EntryIndexedValue">RT</s:String>
5454
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateInstanceFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;</s:String>
55+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/UserRules/=4a98fdf6_002D7d98_002D4f5a_002Dafeb_002Dea44ad98c70c/@EntryIndexedValue">&lt;Policy&gt;&lt;Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"&gt;&lt;ElementKinds&gt;&lt;Kind Name="FIELD" /&gt;&lt;Kind Name="READONLY_FIELD" /&gt;&lt;/ElementKinds&gt;&lt;/Descriptor&gt;&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;&lt;/Policy&gt;</s:String>
56+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/UserRules/=f9fce829_002De6f4_002D4cb2_002D80f1_002D5497c44f51df/@EntryIndexedValue">&lt;Policy&gt;&lt;Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"&gt;&lt;ElementKinds&gt;&lt;Kind Name="FIELD" /&gt;&lt;/ElementKinds&gt;&lt;/Descriptor&gt;&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;&lt;/Policy&gt;</s:String>
5557
<s:Boolean x:Key="/Default/Environment/Filtering/ExcludeCoverageFilters/=_002A_003B_002A_003BJetBrains_002EAnnotations_002E_002A_003B_002A/@EntryIndexedValue">True</s:Boolean>
5658
<s:Boolean x:Key="/Default/Environment/Filtering/ExcludeCoverageFilters/=_002A_003B_002A_003BSimpleJson_002E_002A_003B_002A/@EntryIndexedValue">True</s:Boolean>
5759
<s:Int64 x:Key="/Default/Environment/Hierarchy/Build/BuildTool/MsbuildVersion/@EntryValue">983040</s:Int64>
@@ -68,6 +70,7 @@
6870
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateThisQualifierSettings/@EntryIndexedValue">True</s:Boolean>
6971
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateInstanceFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;</s:String>
7072
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateStaticFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;</s:String>
73+
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EPredefinedNamingRulesToUserRulesUpgrade/@EntryIndexedValue">True</s:Boolean>
7174
<s:String x:Key="/Default/FilterSettingsManager/CoverageFilterXml/@EntryValue">&lt;data&gt;&lt;IncludeFilters /&gt;&lt;ExcludeFilters&gt;&lt;Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="JetBrains.Annotations.*" FunctionMask="*" IsEnabled="True" /&gt;&lt;Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="SimpleJson.*" FunctionMask="*" IsEnabled="True" /&gt;&lt;/ExcludeFilters&gt;&lt;/data&gt;</s:String>
7275
<s:Boolean x:Key="/Default/UserDictionary/Words/=Acpi/@EntryIndexedValue">True</s:Boolean>
7376
<s:Boolean x:Key="/Default/UserDictionary/Words/=Affinitize/@EntryIndexedValue">True</s:Boolean>
@@ -162,6 +165,7 @@
162165
<s:Boolean x:Key="/Default/UserDictionary/Words/=parameterless/@EntryIndexedValue">True</s:Boolean>
163166
<s:Boolean x:Key="/Default/UserDictionary/Words/=Partitioner/@EntryIndexedValue">True</s:Boolean>
164167
<s:Boolean x:Key="/Default/UserDictionary/Words/=pdbonly/@EntryIndexedValue">True</s:Boolean>
168+
<s:Boolean x:Key="/Default/UserDictionary/Words/=Perfolizer/@EntryIndexedValue">True</s:Boolean>
165169
<s:Boolean x:Key="/Default/UserDictionary/Words/=poco/@EntryIndexedValue">True</s:Boolean>
166170
<s:Boolean x:Key="/Default/UserDictionary/Words/=Prerelease/@EntryIndexedValue">True</s:Boolean>
167171
<s:Boolean x:Key="/Default/UserDictionary/Words/=Prettifier/@EntryIndexedValue">True</s:Boolean>

‎samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,10 @@
11
using System.Threading;
22
using BenchmarkDotNet.Attributes;
3-
using Perfolizer.Mathematics.SignificanceTesting;
4-
using Perfolizer.Mathematics.Thresholds;
53

64
namespace BenchmarkDotNet.Samples
75
{
8-
[StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Microseconds, 1, true)]
9-
[StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Microseconds, 1, true)]
10-
[StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.03, true)]
11-
[StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.03, true)]
6+
[StatisticalTestColumn("1us")]
7+
[StatisticalTestColumn("3%")]
128
[SimpleJob(warmupCount: 0, iterationCount: 5)]
139
public class IntroStatisticalTesting
1410
{

‎src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using BenchmarkDotNet.Running;
77
using Microsoft.Diagnostics.Tracing.Parsers;
88
using Microsoft.Diagnostics.Tracing.Session;
9+
using Perfolizer.Metrology;
910

1011
namespace BenchmarkDotNet.Diagnostics.Windows
1112
{
@@ -97,7 +98,7 @@ private sealed class JitAllocatedMemoryDescriptor : IMetricDescriptor
9798
public bool TheGreaterTheBetter => false;
9899
public string NumberFormat => "N0";
99100
public UnitType UnitType => UnitType.Size;
100-
public string Unit => SizeUnit.B.Name;
101+
public string Unit => SizeUnit.B.Abbreviation;
101102
public int PriorityInCategory => 0;
102103
public bool GetIsAvailable(Metric metric) => true;
103104
}

‎src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
using System.Collections.Generic;
33
using System.Diagnostics.CodeAnalysis;
44
using System.Linq;
5-
using BenchmarkDotNet.Columns;
65
using BenchmarkDotNet.Diagnosers;
76
using BenchmarkDotNet.Engines;
87
using BenchmarkDotNet.Extensions;
@@ -12,6 +11,7 @@
1211
using Microsoft.Diagnostics.Tracing.Etlx;
1312
using Microsoft.Diagnostics.Tracing.Parsers.Kernel;
1413
using Microsoft.Diagnostics.Tracing.Stacks;
14+
using Perfolizer.Metrology;
1515
using Address = System.UInt64;
1616

1717
namespace BenchmarkDotNet.Diagnostics.Windows.Tracing
@@ -246,12 +246,14 @@ bool IsCallStackIn(StackSourceCallStackIndex index)
246246
var memoryAllocatedPerOperation = totalAllocation / totalOperation;
247247
var memoryLeakPerOperation = nativeLeakSize / totalOperation;
248248

249-
logger.WriteLine($"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}");
249+
logger.WriteLine(
250+
$"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, null, benchmarkCase.Config.CultureInfo)}");
250251
logger.WriteLine($"Count of allocated object: {countOfAllocatedObject / totalOperation}");
251252

252253
if (nativeLeakSize != 0)
253254
{
254-
logger.WriteLine($"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}");
255+
logger.WriteLine(
256+
$"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, null, benchmarkCase.Config.CultureInfo)}");
255257
}
256258

257259
var heapInfoList = heaps.Select(h => new { Address = h.Key, h.Value.Count, types = h.Value.Values });
@@ -267,7 +269,8 @@ bool IsCallStackIn(StackSourceCallStackIndex index)
267269
};
268270
}
269271

270-
private static Dictionary<Address, long> CreateHeapCache(Address heapHandle, Dictionary<Address, Dictionary<Address, long>> heaps, ref Dictionary<Address, long> lastHeapAllocs, ref Address lastHeapHandle)
272+
private static Dictionary<Address, long> CreateHeapCache(Address heapHandle, Dictionary<Address, Dictionary<Address, long>> heaps,
273+
ref Dictionary<Address, long> lastHeapAllocs, ref Address lastHeapHandle)
271274
{
272275
Dictionary<Address, long> ret;
273276

@@ -282,4 +285,4 @@ private static Dictionary<Address, long> CreateHeapCache(Address heapHandle, Dic
282285
return ret;
283286
}
284287
}
285-
}
288+
}

‎src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkRep
131131
var formatter = statistics.CreateNanosecondFormatter(cultureInfo);
132132

133133
var builder = new StringBuilder();
134-
var histogram = HistogramBuilder.Adaptive.Build(statistics.OriginalValues);
134+
var histogram = HistogramBuilder.Adaptive.Build(statistics.Sample.Values);
135135
builder.AppendLine("-------------------- Histogram --------------------");
136136
builder.AppendLine(histogram.ToString(formatter));
137137
builder.AppendLine("---------------------------------------------------");

‎src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ protected override IEnumerable<Conclusion> AnalyseReport(BenchmarkReport report,
2020
if (statistics == null || statistics.N < EngineResolver.DefaultMinWorkloadIterationCount)
2121
yield break;
2222

23-
double mValue = MValueCalculator.Calculate(statistics.OriginalValues);
23+
double mValue = MValueCalculator.Calculate(statistics.Sample.Values);
2424
if (mValue > 4.2)
2525
yield return Create("is multimodal", mValue, report, summary.GetCultureInfo());
2626
else if (mValue > 3.2)

‎src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Linq;
44
using BenchmarkDotNet.Engines;
55
using BenchmarkDotNet.Extensions;
6+
using BenchmarkDotNet.Helpers;
67
using BenchmarkDotNet.Jobs;
78
using BenchmarkDotNet.Reports;
89
using JetBrains.Annotations;
@@ -53,7 +54,7 @@ string Format(int n, string verb)
5354
return $"{n} {words} {verb}";
5455
}
5556

56-
var rangeMessages = new List<string?> { GetRangeMessage(lowerOutliers, cultureInfo), GetRangeMessage(upperOutliers, cultureInfo) };
57+
var rangeMessages = new List<string?> { GetRangeMessage(lowerOutliers), GetRangeMessage(upperOutliers) };
5758
rangeMessages.RemoveAll(string.IsNullOrEmpty);
5859
string rangeMessage = rangeMessages.Any()
5960
? " (" + string.Join(", ", rangeMessages) + ")"
@@ -66,20 +67,17 @@ string Format(int n, string verb)
6667
return Format(actualOutliers.Length, "removed") + ", " + Format(allOutliers.Length, "detected") + rangeMessage;
6768
}
6869

69-
private static string? GetRangeMessage(double[] values, CultureInfo cultureInfo)
70+
private static string? GetRangeMessage(double[] values)
7071
{
71-
string Format(double value) => TimeInterval.FromNanoseconds(value).ToString(cultureInfo, "N2");
72+
string Format(double value) => TimeInterval.FromNanoseconds(value).ToDefaultString("N2");
7273

73-
switch (values.Length) {
74-
case 0:
75-
return null;
76-
case 1:
77-
return Format(values.First());
78-
case 2:
79-
return Format(values.Min()) + ", " + Format(values.Max());
80-
default:
81-
return Format(values.Min()) + ".." + Format(values.Max());
82-
}
74+
return values.Length switch
75+
{
76+
0 => null,
77+
1 => Format(values.First()),
78+
2 => Format(values.Min()) + ", " + Format(values.Max()),
79+
_ => Format(values.Min()) + ".." + Format(values.Max())
80+
};
8381
}
8482
}
8583
}

‎src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,17 @@ protected override IEnumerable<Conclusion> AnalyseReport(BenchmarkReport report,
2828
var workloadMeasurements = entire.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).ToArray();
2929
if (workloadMeasurements.IsEmpty())
3030
yield break;
31-
var workload = workloadMeasurements.GetStatistics();
3231

32+
var workloadSample = workloadMeasurements.GetStatistics().Sample;
3333
var threshold = currentFrequency.Value.ToResolution().Nanoseconds / 2;
3434

3535
var zeroMeasurement = overheadMeasurements.Any()
36-
? ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload.WithoutOutliers(), overheadMeasurements.GetStatistics().WithoutOutliers())
37-
: ZeroMeasurementHelper.CheckZeroMeasurementOneSample(workload.WithoutOutliers(), threshold);
36+
? ZeroMeasurementHelper.AreIndistinguishable(workloadSample, overheadMeasurements.GetStatistics().Sample)
37+
: ZeroMeasurementHelper.IsNegligible(workloadSample, threshold);
3838

3939
if (zeroMeasurement)
4040
yield return CreateWarning("The method duration is indistinguishable from the empty method duration",
41-
report, false);
41+
report, false);
4242
}
4343
}
4444
}

‎src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,34 @@
1+
using BenchmarkDotNet.Mathematics;
2+
using Perfolizer;
3+
using Perfolizer.Mathematics.Common;
4+
using Perfolizer.Mathematics.GenericEstimators;
15
using Perfolizer.Mathematics.SignificanceTesting;
2-
using Perfolizer.Mathematics.Thresholds;
6+
using Perfolizer.Mathematics.SignificanceTesting.MannWhitney;
7+
using Perfolizer.Metrology;
38

49
namespace BenchmarkDotNet.Analysers
510
{
6-
public static class ZeroMeasurementHelper
11+
internal static class ZeroMeasurementHelper
712
{
8-
/// <summary>
9-
/// Checks distribution against Zero Measurement hypothesis in case of known threshold
10-
/// </summary>
11-
/// <returns>True if measurement is ZeroMeasurement</returns>
12-
public static bool CheckZeroMeasurementOneSample(double[] results, double threshold)
13-
{
14-
if (results.Length < 3)
15-
return false;
16-
return !StudentTest.Instance.IsGreater(results, threshold).NullHypothesisIsRejected;
17-
}
13+
public static bool IsNegligible(Sample results, double threshold) => HodgesLehmannEstimator.Instance.Median(results) < threshold;
14+
public static bool IsNoticeable(Sample results, double threshold) => !IsNegligible(results, threshold);
1815

19-
/// <summary>
20-
/// Checks distribution against Zero Measurement hypothesis in case of two samples
21-
/// </summary>
22-
/// <returns>True if measurement is ZeroMeasurement</returns>
23-
public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold? threshold = null)
16+
public static bool AreIndistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) =>
17+
AreIndistinguishable(new Sample(workload), new Sample(overhead), threshold);
18+
19+
public static bool AreIndistinguishable(Sample workload, Sample overhead, Threshold? threshold = null)
2420
{
25-
if (workload.Length < 3 || overhead.Length < 3)
21+
threshold ??= MathHelper.DefaultThreshold;
22+
var tost = new SimpleEquivalenceTest(MannWhitneyTest.Instance);
23+
if (workload.Size == 1 || overhead.Size == 1)
2624
return false;
27-
return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected;
25+
return tost.Perform(workload, overhead, threshold, SignificanceLevel.P1E5) == ComparisonResult.Indistinguishable;
2826
}
27+
28+
public static bool AreDistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) =>
29+
!AreIndistinguishable(workload, overhead, threshold);
30+
31+
public static bool AreDistinguishable(Sample workload, Sample overhead, Threshold? threshold = null) =>
32+
!AreIndistinguishable(workload, overhead, threshold);
2933
}
3034
}

‎src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ namespace BenchmarkDotNet.Attributes
77
[PublicAPI]
88
public class ConfidenceIntervalErrorColumnAttribute : ColumnConfigBaseAttribute
99
{
10-
public ConfidenceIntervalErrorColumnAttribute(ConfidenceLevel level = ConfidenceLevel.L999) : base(StatisticColumn.CiError(level))
11-
{
12-
}
10+
public ConfidenceIntervalErrorColumnAttribute() : base(StatisticColumn.CiError(ConfidenceLevel.L999)) { }
11+
public ConfidenceIntervalErrorColumnAttribute(ConfidenceLevel level) : base(StatisticColumn.CiError(level)) { }
1312
}
1413
}

0 commit comments

Comments
 (0)