Skip to content

Commit d1efe85

Browse files
baywetbaywet
committed
ci: adds percentage based comparisons
Signed-off-by: Vincent Biret <[email protected]>
1 parent 0106ad7 commit d1efe85

File tree

5 files changed

+101
-16
lines changed

5 files changed

+101
-16
lines changed

‎.github/workflows/ci-cd.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,14 +76,14 @@ jobs:
7676
working-directory: ./performance/benchmark
7777

7878
- name: Run comparison tool for empty models
79-
run: dotnet run -c Release --project ./performance/resultsComparer/resultsComparer.csproj -- compare $OLD_REPORT $NEW_REPORT
79+
run: dotnet run -c Release --project ./performance/resultsComparer/resultsComparer.csproj -- compare $OLD_REPORT $NEW_REPORT -p IdenticalMemoryUsage
8080
shell: bash
8181
env:
8282
NEW_REPORT: "${{ github.workspace }}/performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.EmptyModels-report.json"
8383
OLD_REPORT: "${{ github.workspace }}/performanceResults/performance.EmptyModels-report.json"
8484

8585
- name: Run comparison tool for descriptions
86-
run: dotnet run -c Release --project ./performance/resultsComparer/resultsComparer.csproj -- compare $OLD_REPORT $NEW_REPORT
86+
run: dotnet run -c Release --project ./performance/resultsComparer/resultsComparer.csproj -- compare $OLD_REPORT $NEW_REPORT -p ZeroPointOnePercentDifferenceMemoryUsage
8787
shell: bash
8888
env:
8989
NEW_REPORT: "${{ github.workspace }}/performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report.json"

‎performance/resultsComparer/policies/BaseBenchmarkComparisonPolicy.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
using resultsComparer.Models;
2+
3+
namespace resultsComparer.Policies;
4+
5+
internal abstract class BaseBenchmarkComparisonPolicy : IBenchmarkComparisonPolicy
6+
{
7+
protected abstract string TypeName { get; }
8+
public string Name => TypeName[..^6]; // Remove "Policy" suffix
9+
10+
public abstract bool Equals(BenchmarkMemory? x, BenchmarkMemory? y);
11+
public abstract string GetErrorMessage(BenchmarkMemory? x, BenchmarkMemory? y);
12+
13+
public int GetHashCode(BenchmarkMemory obj)
14+
{
15+
throw new InvalidOperationException("This method should not be called. Use Equals instead.");
16+
}
17+
}

‎performance/resultsComparer/policies/IBenchmarkComparisonPolicy.cs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,28 @@ public static IEnumerable<IBenchmarkComparisonPolicy> GetSelectedPolicies(string
1212
{
1313
yield break;
1414
}
15+
var allPolicies = GetAllPolicies();
1516
if (names is ["all"])
1617
{
17-
foreach (var policy in GetAllPolicies())
18+
foreach (var policy in allPolicies)
1819
{
1920
yield return policy;
2021
}
2122
}
2223
var indexedNames = names.ToHashSet(StringComparer.OrdinalIgnoreCase);
23-
if (indexedNames.Contains(nameof(IdenticalMemoryUsagePolicy)))
24+
foreach (var policy in allPolicies.Where(x => indexedNames.Contains(x.Name)))
2425
{
25-
yield return IdenticalMemoryUsagePolicy.Instance;
26+
yield return policy;
2627
}
2728
}
2829
public static IBenchmarkComparisonPolicy[] GetAllPolicies()
2930
{
30-
return [IdenticalMemoryUsagePolicy.Instance];
31+
return [
32+
IdenticalMemoryUsagePolicy.Instance,
33+
ZeroPointOnePercentDifferenceMemoryUsagePolicy.Instance,
34+
OnePercentDifferenceMemoryUsagePolicy.Instance,
35+
TwoPercentDifferenceMemoryUsagePolicy.Instance,
36+
FivePercentDifferenceMemoryUsagePolicy.Instance,
37+
];
3138
}
3239
}

‎performance/resultsComparer/policies/IdenticalMemoryUsagePolicy.cs‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,17 @@
11
using resultsComparer.Models;
22

33
namespace resultsComparer.Policies;
4-
internal sealed class IdenticalMemoryUsagePolicy : IBenchmarkComparisonPolicy
4+
internal sealed class IdenticalMemoryUsagePolicy : BaseBenchmarkComparisonPolicy
55
{
66
public static IdenticalMemoryUsagePolicy Instance { get; } = new IdenticalMemoryUsagePolicy();
7-
public string Name => nameof(IdenticalMemoryUsagePolicy);
8-
public bool Equals(BenchmarkMemory? x, BenchmarkMemory? y)
7+
protected override string TypeName => nameof(IdenticalMemoryUsagePolicy);
8+
public override bool Equals(BenchmarkMemory? x, BenchmarkMemory? y)
99
{
1010
return x?.AllocatedBytes == y?.AllocatedBytes;
1111
}
1212

13-
public string GetErrorMessage(BenchmarkMemory? x, BenchmarkMemory? y)
13+
public override string GetErrorMessage(BenchmarkMemory? x, BenchmarkMemory? y)
1414
{
1515
return $"Allocated bytes differ: {x?.AllocatedBytes} != {y?.AllocatedBytes}";
16-
}
17-
18-
public int GetHashCode(BenchmarkMemory obj)
19-
{
20-
return obj.AllocatedBytes.GetHashCode();
21-
}
16+
}
2217
}

‎performance/resultsComparer/policies/PercentageMemoryUsagePolicy.cs‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
using resultsComparer.Models;
2+
3+
namespace resultsComparer.Policies;
4+
5+
internal sealed class ZeroPointOnePercentDifferenceMemoryUsagePolicy : PercentageMemoryUsagePolicy
6+
{
7+
public static ZeroPointOnePercentDifferenceMemoryUsagePolicy Instance { get; } = new ZeroPointOnePercentDifferenceMemoryUsagePolicy();
8+
protected override string TypeName => nameof(ZeroPointOnePercentDifferenceMemoryUsagePolicy);
9+
public ZeroPointOnePercentDifferenceMemoryUsagePolicy():base(0.1f) {}
10+
}
11+
12+
internal sealed class OnePercentDifferenceMemoryUsagePolicy : PercentageMemoryUsagePolicy
13+
{
14+
public static OnePercentDifferenceMemoryUsagePolicy Instance { get; } = new OnePercentDifferenceMemoryUsagePolicy();
15+
protected override string TypeName => nameof(OnePercentDifferenceMemoryUsagePolicy);
16+
public OnePercentDifferenceMemoryUsagePolicy():base(1) {}
17+
}
18+
19+
internal sealed class TwoPercentDifferenceMemoryUsagePolicy : PercentageMemoryUsagePolicy
20+
{
21+
public static TwoPercentDifferenceMemoryUsagePolicy Instance { get; } = new TwoPercentDifferenceMemoryUsagePolicy();
22+
protected override string TypeName => nameof(TwoPercentDifferenceMemoryUsagePolicy);
23+
public TwoPercentDifferenceMemoryUsagePolicy():base(2) {}
24+
}
25+
26+
internal sealed class FivePercentDifferenceMemoryUsagePolicy : PercentageMemoryUsagePolicy
27+
{
28+
public static FivePercentDifferenceMemoryUsagePolicy Instance { get; } = new FivePercentDifferenceMemoryUsagePolicy();
29+
protected override string TypeName => nameof(FivePercentDifferenceMemoryUsagePolicy);
30+
public FivePercentDifferenceMemoryUsagePolicy():base(1) {}
31+
}
32+
33+
internal abstract class PercentageMemoryUsagePolicy(float tolerancePercentagePoints) : BaseBenchmarkComparisonPolicy
34+
{
35+
private float TolerancePercentagePoints { get; } = Math.Abs(tolerancePercentagePoints);
36+
public override bool Equals(BenchmarkMemory? x, BenchmarkMemory? y)
37+
{
38+
if (x is null && y is null)
39+
{
40+
return true;
41+
}
42+
if (x is null || y is null)
43+
{
44+
return false;
45+
}
46+
var forwardRatio = GetPercentageDifference(x, y);
47+
var backwardRatio = GetPercentageDifference(y, x);
48+
return forwardRatio <= TolerancePercentagePoints && backwardRatio <= TolerancePercentagePoints;
49+
}
50+
private static double GetPercentageDifference(BenchmarkMemory x, BenchmarkMemory y)
51+
{
52+
return Math.Abs(GetAbsoluteRatio(x, y)) * 100;
53+
}
54+
private static double GetAbsoluteRatio(BenchmarkMemory x, BenchmarkMemory y)
55+
{
56+
return Math.Abs(((double)(x.AllocatedBytes - y.AllocatedBytes))/x.AllocatedBytes);
57+
}
58+
public override string GetErrorMessage(BenchmarkMemory? x, BenchmarkMemory? y)
59+
{
60+
if (x is null || y is null)
61+
{
62+
return "One of the benchmarks is null.";
63+
}
64+
return $"Allocated bytes differ: {x.AllocatedBytes} != {y.AllocatedBytes}, Ratio: {GetAbsoluteRatio(x, y)}, Allowed: {TolerancePercentagePoints}%";
65+
}
66+
}

0 commit comments

Comments
 (0)