Skip to content

Commit 1c991f6

Browse files
baywetbaywet
committed
chore: refactors to a command structure
Signed-off-by: Vincent Biret <[email protected]>
1 parent b82594f commit 1c991f6

File tree

6 files changed

+208
-105
lines changed

6 files changed

+208
-105
lines changed

‎.vscode/launch.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
// If you have changed target frameworks, make sure to update the program path.
4545
"program": "${workspaceFolder}/performance/resultsComparer/bin/Debug/net8.0/resultsComparer.dll",
4646
"cwd": "${workspaceFolder}/performance/resultsComparer",
47+
"args": ["compare"],
4748
// For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
4849
"console": "internalConsole",
4950
"stopAtEntry": false,

‎performance/resultsComparer/Logger.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license.
3+
4+
using Microsoft.Extensions.Logging;
5+
6+
namespace resultsComparer;
7+
public static class Logger
8+
{
9+
public static ILoggerFactory ConfigureLogger(LogLevel logLevel)
10+
{
11+
// Configure logger options
12+
#if DEBUG
13+
logLevel = logLevel > LogLevel.Debug ? LogLevel.Debug : logLevel;
14+
#endif
15+
16+
return LoggerFactory.Create((builder) =>
17+
{
18+
builder
19+
.AddSimpleConsole(c => c.IncludeScopes = true)
20+
#if DEBUG
21+
.AddDebug()
22+
#endif
23+
.SetMinimumLevel(logLevel);
24+
});
25+
}
26+
}

‎performance/resultsComparer/Program.cs‎

Lines changed: 26 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,118 +1,39 @@
11
// See https://aka.ms/new-console-template for more information
2-
using System.Text.Json;
2+
using System.CommandLine;
3+
using Microsoft.Extensions.Logging;
4+
using resultsComparer.Handlers;
35

46
namespace resultsComparer;
57

68
public class Program
79
{
810
public static async Task<int> Main(string[] args)
911
{
10-
var existingBenchmark = await GetBenchmarksAllocatedBytes(ExistingReportPath);
11-
if (existingBenchmark is null)
12-
{
13-
await Console.Error.WriteLineAsync("No existing benchmark data found.");
14-
return 1;
15-
}
16-
var newBenchmark = await GetBenchmarksAllocatedBytes(ExistingReportPath);
17-
if (newBenchmark is null)
18-
{
19-
await Console.Error.WriteLineAsync("No new benchmark data found.");
20-
return 1;
21-
}
22-
IBenchmarkComparisonPolicy[] comparisonPolicies = [
23-
MemoryBenchmarkResultComparer.Instance
24-
];
25-
var hasErrors = false;
26-
foreach(var existingBenchmarkResult in existingBenchmark)
27-
{
28-
if (!newBenchmark.TryGetValue(existingBenchmarkResult.Key, out var newBenchmarkResult))
29-
{
30-
await Console.Error.WriteLineAsync($"No new benchmark result found for {existingBenchmarkResult.Key}.");
31-
hasErrors = true;
32-
}
33-
foreach (var comparisonPolicy in comparisonPolicies)
34-
{
35-
if (!comparisonPolicy.Equals(existingBenchmarkResult.Value, newBenchmarkResult))
36-
{
37-
await Console.Error.WriteLineAsync($"Benchmark result for {existingBenchmarkResult.Key} does not match the existing benchmark result. {comparisonPolicy.GetErrorMessage(existingBenchmarkResult.Value, newBenchmarkResult)}");
38-
hasErrors = true;
39-
}
40-
}
41-
}
42-
43-
if (newBenchmark.Keys.Where(x => !existingBenchmark.ContainsKey(x)).ToArray() is { Length: > 0 } missingKeys)
44-
{
45-
await Console.Error.WriteLineAsync("New benchmark results found that do not exist in the existing benchmark results.");
46-
foreach (var missingKey in missingKeys)
47-
{
48-
await Console.Error.WriteLineAsync($"New benchmark result found: {missingKey}.");
49-
}
50-
hasErrors = true;
51-
}
52-
return hasErrors ? 1 : 0;
53-
}
54-
private const string ExistingReportPath = "../benchmark/BenchmarkDotNet.Artifacts/results/performance.EmptyModels-report.json";
55-
56-
private static async Task<Dictionary<string, BenchmarkResult>?> GetBenchmarksAllocatedBytes(string targetPath, CancellationToken cancellationToken = default)
57-
{
58-
if (!File.Exists(targetPath))
59-
{
60-
return null;
61-
}
62-
using var stream = new FileStream(targetPath, FileMode.Open, FileAccess.Read);
63-
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken);
64-
var rootElement = document.RootElement;
65-
if (rootElement.ValueKind is not JsonValueKind.Object ||
66-
!rootElement.TryGetProperty("Benchmarks", out var benchmarksNode) ||
67-
benchmarksNode.ValueKind is not JsonValueKind.Array)
68-
{
69-
return null;
70-
}
71-
return benchmarksNode.EnumerateArray().Select(benchmarkNode => {
72-
if (benchmarkNode.ValueKind is not JsonValueKind.Object)
73-
{
74-
return default;
75-
}
76-
if (!benchmarkNode.TryGetProperty("Memory", out var memoryNode) ||
77-
memoryNode.ValueKind is not JsonValueKind.Object ||
78-
!memoryNode.TryGetProperty("BytesAllocatedPerOperation", out var allocatedBytesNode) ||
79-
allocatedBytesNode.ValueKind is not JsonValueKind.Number ||
80-
!allocatedBytesNode.TryGetInt64(out var allocatedBytes))
81-
{
82-
return default;
83-
}
84-
if (!benchmarkNode.TryGetProperty("Method", out var nameNode) ||
85-
nameNode.ValueKind is not JsonValueKind.String ||
86-
nameNode.GetString() is not string name)
87-
{
88-
return default;
89-
}
90-
return (name, new BenchmarkResult(allocatedBytes));
91-
})
92-
.Where(x => x.name is not null && x.Item2 is not null)
93-
.ToDictionary(x => x.name!, x => x.Item2!, StringComparer.OrdinalIgnoreCase);
12+
var rootCommand = CreateRootCommand();
13+
return await rootCommand.InvokeAsync(args);
9414
}
95-
private sealed record BenchmarkResult(long AllocatedBytes);
96-
private interface IBenchmarkComparisonPolicy : IEqualityComparer<BenchmarkResult>
15+
internal static RootCommand CreateRootCommand()
9716
{
98-
string GetErrorMessage(BenchmarkResult? x, BenchmarkResult? y);
99-
}
100-
private sealed class MemoryBenchmarkResultComparer : IBenchmarkComparisonPolicy
101-
{
102-
public static MemoryBenchmarkResultComparer Instance { get; } = new MemoryBenchmarkResultComparer();
103-
public bool Equals(BenchmarkResult? x, BenchmarkResult? y)
104-
{
105-
return x?.AllocatedBytes == y?.AllocatedBytes;
106-
}
17+
var rootCommand = new RootCommand { };
10718

108-
public string GetErrorMessage(BenchmarkResult? x, BenchmarkResult? y)
109-
{
110-
return $"Allocated bytes differ: {x?.AllocatedBytes} != {y?.AllocatedBytes}";
111-
}
112-
113-
public int GetHashCode(BenchmarkResult obj)
114-
{
115-
return obj.AllocatedBytes.GetHashCode();
116-
}
19+
var compareCommand = new Command("compare")
20+
{
21+
Description = "Compare the benchmark results."
22+
};
23+
var oldResultsPathArgument = new Argument<string>("existingReportPath", () => ExistingReportPath, "The path to the existing benchmark report.");
24+
compareCommand.AddArgument(oldResultsPathArgument);
25+
var newResultsPathArgument = new Argument<string>("newReportPath", () => ExistingReportPath, "The path to the new benchmark report.");
26+
compareCommand.AddArgument(newResultsPathArgument);
27+
var logLevelOption = new Option<LogLevel>(["--log-level", "-l"], () => LogLevel.Warning, "The log level to use.");
28+
compareCommand.AddOption(logLevelOption);
29+
compareCommand.Handler = new CompareCommandHandler
30+
{
31+
OldResultsPath = oldResultsPathArgument,
32+
NewResultsPath = newResultsPathArgument,
33+
LogLevel = logLevelOption
34+
};
35+
rootCommand.Add(compareCommand);
36+
return rootCommand;
11737
}
38+
private const string ExistingReportPath = "../benchmark/BenchmarkDotNet.Artifacts/results/performance.EmptyModels-report.json";
11839
}

‎performance/resultsComparer/handlers/AsyncCommandHandler.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using System;
2+
using System.CommandLine.Invocation;
3+
using System.Threading.Tasks;
4+
5+
namespace resultsComparer.Handlers;
6+
7+
internal abstract class AsyncCommandHandler : ICommandHandler
8+
{
9+
public int Invoke(InvocationContext context)
10+
{
11+
throw new InvalidOperationException("This method should not be called");
12+
}
13+
public abstract Task<int> InvokeAsync(InvocationContext context);
14+
}

‎performance/resultsComparer/handlers/CompareCommandHandler.cs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
using System;
2+
using System.CommandLine;
3+
using System.CommandLine.Invocation;
4+
using System.Text.Json;
5+
using Microsoft.Extensions.DependencyInjection;
6+
using Microsoft.Extensions.Logging;
7+
8+
namespace resultsComparer.Handlers;
9+
10+
internal class CompareCommandHandler : AsyncCommandHandler
11+
{
12+
public required Argument<string> OldResultsPath { get; set; }
13+
public required Argument<string> NewResultsPath { get; set; }
14+
public required Option<LogLevel> LogLevel { get; set; }
15+
16+
public override Task<int> InvokeAsync(InvocationContext context)
17+
{
18+
var cancellationToken = context.BindingContext.GetRequiredService<CancellationToken>();
19+
var oldResultsPath = context.ParseResult.GetValueForArgument(OldResultsPath);
20+
var newResultsPath = context.ParseResult.GetValueForArgument(NewResultsPath);
21+
var logLevel = context.ParseResult.GetValueForOption(LogLevel);
22+
using var loggerFactory = Logger.ConfigureLogger(logLevel);
23+
var logger = loggerFactory.CreateLogger<CompareCommandHandler>();
24+
return CompareResultsAsync(oldResultsPath, newResultsPath, logger, cancellationToken);
25+
}
26+
private static async Task<int> CompareResultsAsync(string existingReportPath, string newReportPath, ILogger logger, CancellationToken cancellationToken = default) {
27+
28+
var existingBenchmark = await GetBenchmarksAllocatedBytes(existingReportPath, cancellationToken);
29+
if (existingBenchmark is null)
30+
{
31+
logger.LogError("No existing benchmark data found.");
32+
return 1;
33+
}
34+
var newBenchmark = await GetBenchmarksAllocatedBytes(newReportPath, cancellationToken);
35+
if (newBenchmark is null)
36+
{
37+
logger.LogError("No new benchmark data found.");
38+
return 1;
39+
}
40+
IBenchmarkComparisonPolicy[] comparisonPolicies = [
41+
MemoryBenchmarkResultComparer.Instance
42+
];
43+
var hasErrors = false;
44+
foreach(var existingBenchmarkResult in existingBenchmark)
45+
{
46+
if (!newBenchmark.TryGetValue(existingBenchmarkResult.Key, out var newBenchmarkResult))
47+
{
48+
logger.LogError("No new benchmark result found for {existingBenchmarkResultKey}.", existingBenchmarkResult.Key);
49+
hasErrors = true;
50+
}
51+
foreach (var comparisonPolicy in comparisonPolicies)
52+
{
53+
if (!comparisonPolicy.Equals(existingBenchmarkResult.Value, newBenchmarkResult))
54+
{
55+
logger.LogError("Benchmark result for {existingBenchmarkResultKey} does not match the existing benchmark result. {errorMessage}", existingBenchmarkResult.Key, comparisonPolicy.GetErrorMessage(existingBenchmarkResult.Value, newBenchmarkResult));
56+
hasErrors = true;
57+
}
58+
}
59+
}
60+
61+
if (newBenchmark.Keys.Where(x => !existingBenchmark.ContainsKey(x)).ToArray() is { Length: > 0 } missingKeys)
62+
{
63+
logger.LogError("New benchmark results found that do not exist in the existing benchmark results.");
64+
foreach (var missingKey in missingKeys)
65+
{
66+
logger.LogError("New benchmark result found: {missingKey}.", missingKey);
67+
}
68+
hasErrors = true;
69+
}
70+
logger.LogInformation("Benchmark comparison complete. {status}", hasErrors ? "Errors found" : "No errors found");
71+
return hasErrors ? 1 : 0;
72+
}
73+
74+
private static async Task<Dictionary<string, BenchmarkResult>?> GetBenchmarksAllocatedBytes(string targetPath, CancellationToken cancellationToken = default)
75+
{
76+
if (!File.Exists(targetPath))
77+
{
78+
return null;
79+
}
80+
using var stream = new FileStream(targetPath, FileMode.Open, FileAccess.Read);
81+
using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken);
82+
var rootElement = document.RootElement;
83+
if (rootElement.ValueKind is not JsonValueKind.Object ||
84+
!rootElement.TryGetProperty("Benchmarks", out var benchmarksNode) ||
85+
benchmarksNode.ValueKind is not JsonValueKind.Array)
86+
{
87+
return null;
88+
}
89+
return benchmarksNode.EnumerateArray().Select(benchmarkNode => {
90+
if (benchmarkNode.ValueKind is not JsonValueKind.Object)
91+
{
92+
return default;
93+
}
94+
if (!benchmarkNode.TryGetProperty("Memory", out var memoryNode) ||
95+
memoryNode.ValueKind is not JsonValueKind.Object ||
96+
!memoryNode.TryGetProperty("BytesAllocatedPerOperation", out var allocatedBytesNode) ||
97+
allocatedBytesNode.ValueKind is not JsonValueKind.Number ||
98+
!allocatedBytesNode.TryGetInt64(out var allocatedBytes))
99+
{
100+
return default;
101+
}
102+
if (!benchmarkNode.TryGetProperty("Method", out var nameNode) ||
103+
nameNode.ValueKind is not JsonValueKind.String ||
104+
nameNode.GetString() is not string name)
105+
{
106+
return default;
107+
}
108+
return (name, new BenchmarkResult(allocatedBytes));
109+
})
110+
.Where(x => x.name is not null && x.Item2 is not null)
111+
.ToDictionary(x => x.name!, x => x.Item2!, StringComparer.OrdinalIgnoreCase);
112+
}
113+
private sealed record BenchmarkResult(long AllocatedBytes);
114+
private interface IBenchmarkComparisonPolicy : IEqualityComparer<BenchmarkResult>
115+
{
116+
string GetErrorMessage(BenchmarkResult? x, BenchmarkResult? y);
117+
}
118+
private sealed class MemoryBenchmarkResultComparer : IBenchmarkComparisonPolicy
119+
{
120+
public static MemoryBenchmarkResultComparer Instance { get; } = new MemoryBenchmarkResultComparer();
121+
public bool Equals(BenchmarkResult? x, BenchmarkResult? y)
122+
{
123+
return x?.AllocatedBytes == y?.AllocatedBytes;
124+
}
125+
126+
public string GetErrorMessage(BenchmarkResult? x, BenchmarkResult? y)
127+
{
128+
return $"Allocated bytes differ: {x?.AllocatedBytes} != {y?.AllocatedBytes}";
129+
}
130+
131+
public int GetHashCode(BenchmarkResult obj)
132+
{
133+
return obj.AllocatedBytes.GetHashCode();
134+
}
135+
}
136+
}

‎performance/resultsComparer/resultsComparer.csproj‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88
</PropertyGroup>
99

1010
<ItemGroup>
11+
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.3" />
12+
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.3" />
13+
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.3" />
14+
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="9.0.3" />
15+
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
1116
<PackageReference Include="system.text.json" Version="9.0.3" />
1217
</ItemGroup>
1318

0 commit comments

Comments
 (0)