Skip to content

Commit 349e900

Browse files
AndreyAkinshinAndreyAkinshin
committed
Introduce RatioStyle, fix #721 
1 parent cd0bda4 commit 349e900

File tree

7 files changed

+292
-8
lines changed

7 files changed

+292
-8
lines changed

‎docs/articles/samples/IntroRatioStyle.md

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
---
2+
uid: BenchmarkDotNet.Samples.IntroRatioStyle
3+
---
4+
5+
## Sample: IntroRatioStyle
6+
7+
Using `RatioStyle`, we can override the style of the "Ratio" column in `SummaryStyle`.
8+
Here are the possible values:
9+
10+
* `Value`: default value that shows the ration value between the current benchmark and the baseline benchmark (e.g., `0.15` or `1.15`)
11+
* `Percentage`: express the ration in percentage (e.g., `-85%` or `+15%`)
12+
* `Trend`: shows how much the current benchmark is faster or slower than the base benchmark (e.g., `6.63x faster` or `1.15x slower`)
13+
14+
### Source code
15+
16+
[!code-csharp[IntroRatioStyle.cs](../../../samples/BenchmarkDotNet.Samples/IntroRatioStyle.cs)]
17+
18+
### Output
19+
20+
With the given `RatioStyle.Trend`, we have the following table:
21+
22+
```markdown
23+
| Method | Mean | Error | StdDev | Ratio | RatioSD |
24+
|--------- |-----------:|--------:|--------:|-------------:|--------:|
25+
| Baseline | 1,000.6 ms | 2.48 ms | 0.14 ms | baseline | |
26+
| Bar | 150.9 ms | 1.30 ms | 0.07 ms | 6.63x faster | 0.00x |
27+
| Foo | 1,150.4 ms | 5.17 ms | 0.28 ms | 1.15x slower | 0.00x |
28+
```
29+
30+
With the default `RatioStyle.Value`, we get the following table:
31+
32+
```markdown
33+
| Method | Mean | Error | StdDev | Ratio |
34+
|--------- |-----------:|--------:|--------:|------:|
35+
| Baseline | 1,000.3 ms | 2.71 ms | 0.15 ms | 1.00 |
36+
| Bar | 150.6 ms | 1.67 ms | 0.09 ms | 0.15 |
37+
| Foo | 1,150.6 ms | 7.41 ms | 0.41 ms | 1.15 |
38+
```
39+
40+
If we use `RatioStyle.Percentage`, we get the following table:
41+
42+
```markdown
43+
| Method | Mean | Error | StdDev | Ratio | RatioSD |
44+
|--------- |-----------:|--------:|--------:|---------:|--------:|
45+
| Baseline | 1,000.3 ms | 4.69 ms | 0.26 ms | baseline | |
46+
| Bar | 150.7 ms | 1.42 ms | 0.08 ms | -85% | 0.1% |
47+
| Foo | 1,150.3 ms | 6.13 ms | 0.34 ms | +15% | 0.0% |
48+
```
49+
50+
### Links
51+
52+
* @docs.baselines
53+
* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroRatioStyle
54+
55+
---

‎docs/articles/samples/toc.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@
9292
href: IntroRankColumn.md
9393
- name: IntroRatioSD
9494
href: IntroRatioSD.md
95+
- name: IntroRatioStyle
96+
href: IntroRatioStyle.md
9597
- name: IntroSetupCleanupGlobal
9698
href: IntroSetupCleanupGlobal.md
9799
- name: IntroSetupCleanupIteration

‎samples/BenchmarkDotNet.Samples/IntroRatioStyle.cs

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System.Threading;
2+
using BenchmarkDotNet.Attributes;
3+
using BenchmarkDotNet.Columns;
4+
using BenchmarkDotNet.Configs;
5+
using BenchmarkDotNet.Reports;
6+
7+
namespace BenchmarkDotNet.Samples
8+
{
9+
[ShortRunJob, Config(typeof(Config))]
10+
public class IntroRatioStyle
11+
{
12+
[Benchmark(Baseline = true)]
13+
public void Baseline() => Thread.Sleep(1000);
14+
15+
[Benchmark]
16+
public void Bar() => Thread.Sleep(150);
17+
18+
[Benchmark]
19+
public void Foo() => Thread.Sleep(1150);
20+
21+
private class Config : ManualConfig
22+
{
23+
public Config()
24+
{
25+
SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend);
26+
}
27+
}
28+
}
29+
}

‎src/BenchmarkDotNet/Columns/BaselineRatioColumn.cs

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,62 @@ public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, St
5050
var ratio = GetRatioStatistics(current, baseline);
5151
if (ratio == null)
5252
return "NA";
53+
var invertedRatio = GetRatioStatistics(baseline, current);
5354

5455
var cultureInfo = summary.GetCultureInfo();
56+
var ratioStyle = summary?.Style?.RatioStyle ?? RatioStyle.Value;
57+
5558
switch (Metric)
5659
{
5760
case RatioMetric.Mean:
58-
return IsNonBaselinesPrecise(summary, baseline, benchmarkCase) ? ratio.Mean.ToString("N3", cultureInfo) : ratio.Mean.ToString("N2", cultureInfo);
61+
{
62+
bool advancedPrecision = IsNonBaselinesPrecise(summary, baseline, benchmarkCase);
63+
switch (ratioStyle)
64+
{
65+
case RatioStyle.Value:
66+
return ratio.Mean.ToString(advancedPrecision ? "N3" : "N2", cultureInfo);
67+
case RatioStyle.Percentage:
68+
return isBaseline
69+
? "baseline"
70+
: ratio.Mean >= 1.0
71+
? "+" + ((ratio.Mean - 1.0) * 100).ToString(advancedPrecision ? "N1" : "N0") + "%"
72+
: "-" + ((1.0 - ratio.Mean) * 100).ToString(advancedPrecision ? "N1" : "N0") + "%";
73+
case RatioStyle.Trend:
74+
return isBaseline
75+
? "baseline"
76+
: ratio.Mean >= 1.0
77+
? ratio.Mean.ToString(advancedPrecision ? "N3" : "N2") + "x slower"
78+
: invertedRatio == null
79+
? "NA"
80+
: invertedRatio.Mean.ToString(advancedPrecision ? "N3" : "N2") + "x faster";
81+
default:
82+
throw new ArgumentOutOfRangeException(nameof(summary), ratioStyle, "RatioStyle is not supported");
83+
}
84+
}
5985
case RatioMetric.StdDev:
60-
return ratio.StandardDeviation.ToString("N2", cultureInfo);
86+
{
87+
switch (ratioStyle)
88+
{
89+
case RatioStyle.Value:
90+
return ratio.StandardDeviation.ToString("N2", cultureInfo);
91+
case RatioStyle.Percentage:
92+
return isBaseline
93+
? ""
94+
: Math.Abs(ratio.Mean) < 1e-9
95+
? "NA"
96+
: (100 * ratio.StandardDeviation / ratio.Mean).ToString("N1", cultureInfo) + "%";
97+
case RatioStyle.Trend:
98+
return isBaseline
99+
? ""
100+
: ratio.Mean >= 1.0
101+
? ratio.StandardDeviation.ToString("N2", cultureInfo) + "x"
102+
: invertedRatio == null
103+
? "NA"
104+
: invertedRatio.StandardDeviation.ToString("N2", cultureInfo) + "x";
105+
default:
106+
throw new ArgumentOutOfRangeException(nameof(summary), ratioStyle, "RatioStyle is not supported");
107+
}
108+
}
61109
default:
62110
throw new NotSupportedException();
63111
}

‎src/BenchmarkDotNet/Columns/RatioColumnStyle.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
namespace BenchmarkDotNet.Columns
2+
{
3+
public enum RatioStyle
4+
{
5+
Value,
6+
Percentage,
7+
Trend
8+
}
9+
}

‎src/BenchmarkDotNet/Reports/SummaryStyle.cs

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,11 @@ public class SummaryStyle : IEquatable<SummaryStyle>
2222
public TimeUnit TimeUnit { get; }
2323
[NotNull]
2424
public CultureInfo CultureInfo { get; }
25+
26+
public RatioStyle RatioStyle { get; }
2527

26-
public SummaryStyle([CanBeNull] CultureInfo cultureInfo, bool printUnitsInHeader, SizeUnit sizeUnit, TimeUnit timeUnit, bool printUnitsInContent = true, bool printZeroValuesInContent = false, int maxParameterColumnWidth = DefaultMaxParameterColumnWidth)
28+
public SummaryStyle([CanBeNull] CultureInfo cultureInfo, bool printUnitsInHeader, SizeUnit sizeUnit, TimeUnit timeUnit, bool printUnitsInContent = true,
29+
bool printZeroValuesInContent = false, int maxParameterColumnWidth = DefaultMaxParameterColumnWidth, RatioStyle ratioStyle = RatioStyle.Value)
2730
{
2831
if (maxParameterColumnWidth < DefaultMaxParameterColumnWidth)
2932
throw new ArgumentOutOfRangeException(nameof(maxParameterColumnWidth), $"{DefaultMaxParameterColumnWidth} is the minimum.");
@@ -35,22 +38,26 @@ public SummaryStyle([CanBeNull] CultureInfo cultureInfo, bool printUnitsInHeader
3538
TimeUnit = timeUnit;
3639
PrintZeroValuesInContent = printZeroValuesInContent;
3740
MaxParameterColumnWidth = maxParameterColumnWidth;
41+
RatioStyle = ratioStyle;
3842
}
3943

4044
public SummaryStyle WithTimeUnit(TimeUnit timeUnit)
41-
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, SizeUnit, timeUnit, PrintUnitsInContent, PrintZeroValuesInContent);
45+
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, SizeUnit, timeUnit, PrintUnitsInContent, PrintZeroValuesInContent, MaxParameterColumnWidth, RatioStyle);
4246

4347
public SummaryStyle WithSizeUnit(SizeUnit sizeUnit)
44-
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, sizeUnit, TimeUnit, PrintUnitsInContent, PrintZeroValuesInContent);
48+
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, sizeUnit, TimeUnit, PrintUnitsInContent, PrintZeroValuesInContent, MaxParameterColumnWidth, RatioStyle);
4549

4650
public SummaryStyle WithZeroMetricValuesInContent()
47-
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, SizeUnit, TimeUnit, PrintUnitsInContent, printZeroValuesInContent: true);
51+
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, SizeUnit, TimeUnit, PrintUnitsInContent, printZeroValuesInContent: true, MaxParameterColumnWidth, RatioStyle);
4852

4953
public SummaryStyle WithMaxParameterColumnWidth(int maxParameterColumnWidth)
50-
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, SizeUnit, TimeUnit, PrintUnitsInContent, PrintZeroValuesInContent, maxParameterColumnWidth);
54+
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, SizeUnit, TimeUnit, PrintUnitsInContent, PrintZeroValuesInContent, maxParameterColumnWidth, RatioStyle);
5155

5256
public SummaryStyle WithCultureInfo(CultureInfo cultureInfo)
53-
=> new SummaryStyle(cultureInfo, PrintUnitsInHeader, SizeUnit, TimeUnit, PrintUnitsInContent, PrintZeroValuesInContent, MaxParameterColumnWidth);
57+
=> new SummaryStyle(cultureInfo, PrintUnitsInHeader, SizeUnit, TimeUnit, PrintUnitsInContent, PrintZeroValuesInContent, MaxParameterColumnWidth, RatioStyle);
58+
59+
public SummaryStyle WithRatioStyle(RatioStyle ratioStyle)
60+
=> new SummaryStyle(CultureInfo, PrintUnitsInHeader, SizeUnit, TimeUnit, PrintUnitsInContent, PrintZeroValuesInContent, MaxParameterColumnWidth, ratioStyle);
5461

5562
public bool Equals(SummaryStyle other)
5663
{

‎tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Collections.Immutable;
4+
using System.Linq;
5+
using BenchmarkDotNet.Attributes;
6+
using BenchmarkDotNet.Columns;
7+
using BenchmarkDotNet.Configs;
8+
using BenchmarkDotNet.Engines;
9+
using BenchmarkDotNet.Exporters;
10+
using BenchmarkDotNet.Loggers;
11+
using BenchmarkDotNet.Reports;
12+
using BenchmarkDotNet.Running;
13+
using BenchmarkDotNet.Tests.Builders;
14+
using BenchmarkDotNet.Toolchains;
15+
using BenchmarkDotNet.Toolchains.Results;
16+
using BenchmarkDotNet.Validators;
17+
using JetBrains.Annotations;
18+
using Xunit;
19+
using Xunit.Abstractions;
20+
21+
namespace BenchmarkDotNet.Tests.Reports
22+
{
23+
public class RatioStyleTests
24+
{
25+
private readonly ITestOutputHelper output;
26+
27+
public RatioStyleTests(ITestOutputHelper output)
28+
{
29+
this.output = output;
30+
}
31+
32+
private class TestData
33+
{
34+
public RatioStyle RatioStyle { get; }
35+
public int[] MeanValues { get; }
36+
public int Noise { get; }
37+
public string[] ExpectedRatioValues { get; }
38+
39+
public TestData(RatioStyle ratioStyle, int[] meanValues, int noise, string[] expectedRatioValues)
40+
{
41+
RatioStyle = ratioStyle;
42+
MeanValues = meanValues;
43+
ExpectedRatioValues = expectedRatioValues;
44+
Noise = noise;
45+
}
46+
}
47+
48+
private static readonly IDictionary<string, TestData> Data = new Dictionary<string, TestData>
49+
{
50+
{ "Percentage", new TestData(RatioStyle.Percentage, new[] { 100, 15, 115 }, 1, new[] { "baseline", "-85%", "+15%" }) },
51+
{ "Trend", new TestData(RatioStyle.Trend, new[] { 100, 15, 115 }, 1, new[] { "baseline", "6.83x faster", "1.15x slower" }) }
52+
};
53+
54+
[UsedImplicitly]
55+
public static TheoryData<string> DataNames = TheoryDataHelper.Create(Data.Keys);
56+
57+
[Theory]
58+
[MemberData(nameof(DataNames))]
59+
// First value is baseline, others are benchmark measurements
60+
public void RatioPrecisionTestWithBaseline([NotNull] string testDataKey)
61+
{
62+
var testData = Data[testDataKey];
63+
var summary = CreateSummary(testData.MeanValues, testData.RatioStyle, testData.Noise);
64+
int ratioIndex = Array.FindIndex(summary.Table.FullHeader, c => c == BaselineRatioColumn.RatioMean.ColumnName);
65+
66+
for (int rowIndex = 0; rowIndex < summary.Table.FullContent.Length; rowIndex++)
67+
{
68+
var row = summary.Table.FullContent[rowIndex];
69+
string actual = row[ratioIndex];
70+
string expected = testData.ExpectedRatioValues[rowIndex];
71+
Assert.Equal(expected, actual);
72+
}
73+
}
74+
75+
// TODO: Union this with MockFactory
76+
private Summary CreateSummary(int[] values, RatioStyle ratioStyle, int noise)
77+
{
78+
var logger = new AccumulationLogger();
79+
var config = DefaultConfig.Instance.WithSummaryStyle(SummaryStyle.Default.WithRatioStyle(ratioStyle: ratioStyle));
80+
var benchmarks = CreateBenchmarks(config).ToList();
81+
var benchmarkReports = new List<BenchmarkReport>();
82+
for (var x = 0; x < benchmarks.Count; x++)
83+
{
84+
var benchmark = benchmarks[x];
85+
benchmarkReports.Add(CreateReport(benchmark, values[x], noise));
86+
}
87+
88+
var summary = new Summary(
89+
"MockSummary",
90+
benchmarkReports.ToImmutableArray(),
91+
new HostEnvironmentInfoBuilder().Build(),
92+
string.Empty,
93+
string.Empty,
94+
TimeSpan.FromMinutes(1),
95+
TestCultureInfo.Instance,
96+
ImmutableArray<ValidationError>.Empty);
97+
MarkdownExporter.Default.ExportToLog(summary, logger);
98+
output.WriteLine(logger.GetLog());
99+
return summary;
100+
}
101+
102+
private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, int measurementValue, int noise)
103+
{
104+
var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty<string>()));
105+
var executeResult = new ExecuteResult(true, 0, default, Array.Empty<string>(), Array.Empty<string>());
106+
var measurements = new List<Measurement>
107+
{
108+
new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, measurementValue),
109+
new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, measurementValue + noise),
110+
new Measurement(1, IterationMode.Workload, IterationStage.Result, 3, 1, measurementValue - noise),
111+
new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, measurementValue + 2 * noise),
112+
new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, measurementValue - 3 * noise)
113+
};
114+
return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List<ExecuteResult> { executeResult }, measurements, default,
115+
Array.Empty<Metric>());
116+
}
117+
118+
private static IEnumerable<BenchmarkCase> CreateBenchmarks(IConfig config) =>
119+
BenchmarkConverter.TypeToBenchmarks(typeof(MockBenchmarkClass), config).BenchmarksCases;
120+
121+
[LongRunJob]
122+
public class MockBenchmarkClass
123+
{
124+
[Benchmark(Baseline = true)]
125+
public void Baseline() { }
126+
127+
[Benchmark]
128+
public void Bar() { }
129+
130+
[Benchmark]
131+
public void Foo() { }
132+
}
133+
}
134+
}

0 commit comments

Comments
 (0)