Skip to content

Commit 8c55f17

Browse files
baywetbaywet
committed
ci: adds basic infrastructure for descriptions based performance tests
Signed-off-by: Vincent Biret <[email protected]>
1 parent 4b7d820 commit 8c55f17

File tree

7 files changed

+117
-0
lines changed

7 files changed

+117
-0
lines changed

‎performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report-github.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
```
2+
3+
BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.3476)
4+
11th Gen Intel Core i7-1185G7 3.00GHz, 1 CPU, 8 logical and 4 physical cores
5+
.NET SDK 8.0.407
6+
[Host] : .NET 8.0.14 (8.0.1425.11118), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
7+
ShortRun : .NET 8.0.14 (8.0.1425.11118), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
8+
9+
Job=ShortRun IterationCount=3 LaunchCount=1
10+
WarmupCount=3
11+
12+
```
13+
| Method | Mean | Error | StdDev | Gen0 | Gen1 | Allocated |
14+
|--------- |---------:|---------:|---------:|--------:|--------:|----------:|
15+
| PetStore | 574.4 μs | 842.9 μs | 46.20 μs | 66.4063 | 15.6250 | 424.67 KB |

‎performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report.csv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev,Gen0,Gen1,Allocated
2+
PetStore,ShortRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,3,Default,1,Default,Default,Default,Default,Default,Default,16,3,574.4 μs,842.9 μs,46.20 μs,66.4063,15.6250,424.67 KB

‎performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report.html‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<!DOCTYPE html>
2+
<html lang='en'>
3+
<head>
4+
<meta charset='utf-8' />
5+
<title>performance.Descriptions-20250407-141533</title>
6+
7+
<style type="text/css">
8+
table { border-collapse: collapse; display: block; width: 100%; overflow: auto; }
9+
td, th { padding: 6px 13px; border: 1px solid #ddd; text-align: right; }
10+
tr { background-color: #fff; border-top: 1px solid #ccc; }
11+
tr:nth-child(even) { background: #f8f8f8; }
12+
</style>
13+
</head>
14+
<body>
15+
<pre><code>
16+
BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.3476)
17+
11th Gen Intel Core i7-1185G7 3.00GHz, 1 CPU, 8 logical and 4 physical cores
18+
.NET SDK 8.0.407
19+
[Host] : .NET 8.0.14 (8.0.1425.11118), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
20+
ShortRun : .NET 8.0.14 (8.0.1425.11118), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
21+
</code></pre>
22+
<pre><code>Job=ShortRun IterationCount=3 LaunchCount=1
23+
WarmupCount=3
24+
</code></pre>
25+
26+
<table>
27+
<thead><tr><th>Method</th><th>Mean</th><th>Error</th><th>StdDev</th><th>Gen0</th><th>Gen1</th><th>Allocated</th>
28+
</tr>
29+
</thead><tbody><tr><td>PetStore</td><td>574.4 &mu;s</td><td>842.9 &mu;s</td><td>46.20 &mu;s</td><td>66.4063</td><td>15.6250</td><td>424.67 KB</td>
30+
</tr></tbody></table>
31+
</body>
32+
</html>

‎performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"Title":"performance.Descriptions-20250407-141533","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.14.0","OsVersion":"Windows 11 (10.0.26100.3476)","ProcessorName":"11th Gen Intel Core i7-1185G7 3.00GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":".NET 8.0.14 (8.0.1425.11118)","Architecture":"X64","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"RELEASE","DotNetCliVersion":"8.0.407","ChronometerFrequency":{"Hertz":10000000},"HardwareTimerKind":"Unknown"},"Benchmarks":[{"DisplayInfo":"Descriptions.PetStore: ShortRun(IterationCount=3, LaunchCount=1, WarmupCount=3)","Namespace":"performance","Type":"Descriptions","Method":"PetStore","MethodTitle":"PetStore","Parameters":"","FullName":"performance.Descriptions.PetStore","HardwareIntrinsics":"AVX-512F+CD+BW+DQ+VL+VBMI,AES,BMI1,BMI2,FMA,LZCNT,PCLMUL,POPCNT VectorSize=256","Statistics":{"OriginalValues":[627671.875,549296.875,546102.734375],"N":3,"Min":546102.734375,"LowerFence":486522.94921875,"Q1":547699.8046875,"Median":549296.875,"Mean":574357.1614583334,"Q3":588484.375,"UpperFence":649661.23046875,"Max":627671.875,"InterquartileRange":40784.5703125,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":26673.299031088187,"Variance":2134394643.6055503,"StandardDeviation":46199.509127322446,"Skewness":0.3828313836915352,"Kurtosis":0.6666666666666653,"ConfidenceInterval":{"N":3,"Mean":574357.1614583334,"StandardError":26673.299031088187,"Level":12,"Margin":842851.0284063907,"Lower":-268493.8669480573,"Upper":1417208.189864724},"Percentiles":{"P0":546102.734375,"P25":547699.8046875,"P50":549296.875,"P67":575944.375,"P80":596321.875,"P85":604159.375,"P90":611996.875,"P95":619834.375,"P100":627671.875}},"Memory":{"Gen0Collections":17,"Gen1Collections":4,"Gen2Collections":0,"TotalOperations":256,"BytesAllocatedPerOperation":434867},"Measurements":[{"IterationMode":"Overhead","IterationStage":"Jitting","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":401000},{"IterationMode":"Workload","IterationStage":"Jitting","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":119853600},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":1,"Operations":2,"Nanoseconds":5232800},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":2,"Operations":3,"Nanoseconds":6876300},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":3,"Operations":4,"Nanoseconds":10323200},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":4,"Operations":5,"Nanoseconds":13859200},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":5,"Operations":6,"Nanoseconds":19375700},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":6,"Operations":7,"Nanoseconds":25025500},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":7,"Operations":8,"Nanoseconds":25797900},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":8,"Operations":9,"Nanoseconds":21831100},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":9,"Operations":10,"Nanoseconds":25960100},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":10,"Operations":11,"Nanoseconds":46019900},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":11,"Operations":12,"Nanoseconds":48740100},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":12,"Operations":13,"Nanoseconds":45952900},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":13,"Operations":14,"Nanoseconds":29883600},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":14,"Operations":15,"Nanoseconds":34838500},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":15,"Operations":16,"Nanoseconds":30708100},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":16,"Operations":32,"Nanoseconds":92722400},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":17,"Operations":64,"Nanoseconds":198261400},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":18,"Operations":128,"Nanoseconds":417534500},{"IterationMode":"Workload","IterationStage":"Pilot","LaunchIndex":1,"IterationIndex":19,"Operations":256,"Nanoseconds":569835400},{"IterationMode":"Workload","IterationStage":"Warmup","LaunchIndex":1,"IterationIndex":1,"Operations":256,"Nanoseconds":223548600},{"IterationMode":"Workload","IterationStage":"Warmup","LaunchIndex":1,"IterationIndex":2,"Operations":256,"Nanoseconds":204621200},{"IterationMode":"Workload","IterationStage":"Warmup","LaunchIndex":1,"IterationIndex":3,"Operations":256,"Nanoseconds":192027700},{"IterationMode":"Workload","IterationStage":"Actual","LaunchIndex":1,"IterationIndex":1,"Operations":256,"Nanoseconds":160684000},{"IterationMode":"Workload","IterationStage":"Actual","LaunchIndex":1,"IterationIndex":2,"Operations":256,"Nanoseconds":140620000},{"IterationMode":"Workload","IterationStage":"Actual","LaunchIndex":1,"IterationIndex":3,"Operations":256,"Nanoseconds":139802300},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":256,"Nanoseconds":160684000},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":256,"Nanoseconds":140620000},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":256,"Nanoseconds":139802300}],"Metrics":[{"Value":66.40625,"Descriptor":{"Id":"Gen0Collects","DisplayName":"Gen0","Legend":"GC Generation 0 collects per 1000 operations","NumberFormat":"#0.0000","UnitType":0,"Unit":"Count","TheGreaterTheBetter":false,"PriorityInCategory":0}},{"Value":15.625,"Descriptor":{"Id":"Gen1Collects","DisplayName":"Gen1","Legend":"GC Generation 1 collects per 1000 operations","NumberFormat":"#0.0000","UnitType":0,"Unit":"Count","TheGreaterTheBetter":false,"PriorityInCategory":1}},{"Value":0,"Descriptor":{"Id":"Gen2Collects","DisplayName":"Gen2","Legend":"GC Generation 2 collects per 1000 operations","NumberFormat":"#0.0000","UnitType":0,"Unit":"Count","TheGreaterTheBetter":false,"PriorityInCategory":2}},{"Value":434867,"Descriptor":{"Id":"Allocated Memory","DisplayName":"Allocated","Legend":"Allocated memory per single operation (managed only, inclusive, 1KB = 1024B)","NumberFormat":"0.##","UnitType":2,"Unit":"B","TheGreaterTheBetter":false,"PriorityInCategory":3}}]}]}

‎performance/benchmark/Descriptions.cs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Reflection;
5+
using System.Threading.Tasks;
6+
using BenchmarkDotNet.Attributes;
7+
using Microsoft.OpenApi;
8+
using Microsoft.OpenApi.Models;
9+
using Microsoft.OpenApi.Reader;
10+
11+
namespace performance;
12+
13+
[MemoryDiagnoser]
14+
[JsonExporter]
15+
[ShortRunJob]
16+
public class Descriptions
17+
{
18+
[Benchmark]
19+
public async Task<OpenApiDocument> PetStore()
20+
{
21+
return await ParseDocumentAsync(PetStorePath);
22+
}
23+
private readonly Dictionary<string, MemoryStream> _streams = new(StringComparer.OrdinalIgnoreCase);
24+
[GlobalSetup]
25+
public async Task GetAllDescriptions()
26+
{
27+
readerSettings = new OpenApiReaderSettings
28+
{
29+
LeaveStreamOpen = true,
30+
};
31+
readerSettings.AddYamlReader();
32+
await LoadDocumentIntoStreams(PetStorePath);
33+
}
34+
private OpenApiReaderSettings readerSettings;
35+
private const string PetStorePath = @"petStore.yaml";
36+
private async Task<OpenApiDocument> ParseDocumentAsync(string fileName)
37+
{
38+
var stream = _streams[fileName];
39+
stream.Seek(0, SeekOrigin.Begin);
40+
41+
var (document, _) = await OpenApiDocument.LoadAsync(stream, OpenApiConstants.Yaml, readerSettings).ConfigureAwait(false);
42+
return document;
43+
}
44+
private static readonly Assembly assembly = typeof(Descriptions).GetTypeInfo().Assembly;
45+
private async Task LoadDocumentIntoStreams(string fileName)
46+
{
47+
using var resource = assembly.GetManifestResourceStream($"PerformanceTests.{fileName}");
48+
var stream = new MemoryStream(); // NOT disposed on purpose
49+
await resource.CopyToAsync(stream).ConfigureAwait(false);
50+
stream.Seek(0, SeekOrigin.Begin);
51+
_streams.Add(fileName, stream);
52+
}
53+
[GlobalCleanup]
54+
public void Cleanup()
55+
{
56+
foreach (var stream in _streams.Values)
57+
{
58+
stream.Dispose();
59+
}
60+
_streams.Clear();
61+
}
62+
}

‎performance/benchmark/PerformanceTests.csproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,9 @@
1919
</ItemGroup>
2020
<ItemGroup>
2121
<ProjectReference Include="..\..\src\Microsoft.OpenApi\Microsoft.OpenApi.csproj" />
22+
<ProjectReference Include="..\..\src\Microsoft.OpenApi.YamlReader\Microsoft.OpenApi.YamlReader.csproj" />
23+
</ItemGroup>
24+
<ItemGroup>
25+
<EmbeddedResource Include="..\..\test\Microsoft.OpenApi.Readers.Tests\V3Tests\Samples\OpenApiDocument\petStore.yaml" />
2226
</ItemGroup>
2327
</Project>

‎performance/benchmark/Program.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ public class Program
77
public static void Main(string[] args)
88
{
99
var config = DefaultConfig.Instance;
10+
BenchmarkRunner.Run<Descriptions>(config, args);
1011
BenchmarkRunner.Run<EmptyModels>(config, args);
1112
}
1213
}

0 commit comments

Comments
 (0)