Skip to content

Commit 91375df

Browse files
author
Ilias Tsakiridis
committed
feat: add OpenMetrics exporter support and update related tests
1 parent e97c686 commit 91375df

14 files changed

+851
-2
lines changed

‎BenchmarkDotNet.sln.DotSettings

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@
161161
<s:Boolean x:Key="/Default/UserDictionary/Words/=NONINFRINGEMENT/@EntryIndexedValue">True</s:Boolean>
162162
<s:Boolean x:Key="/Default/UserDictionary/Words/=notcs/@EntryIndexedValue">True</s:Boolean>
163163
<s:Boolean x:Key="/Default/UserDictionary/Words/=nuget/@EntryIndexedValue">True</s:Boolean>
164+
<s:Boolean x:Key="/Default/UserDictionary/Words/=openmetrics/@EntryIndexedValue">True</s:Boolean>
164165
<s:Boolean x:Key="/Default/UserDictionary/Words/=outofproc/@EntryIndexedValue">True</s:Boolean>
165166
<s:Boolean x:Key="/Default/UserDictionary/Words/=parameterless/@EntryIndexedValue">True</s:Boolean>
166167
<s:Boolean x:Key="/Default/UserDictionary/Words/=Partitioner/@EntryIndexedValue">True</s:Boolean>
@@ -185,6 +186,7 @@
185186
<s:Boolean x:Key="/Default/UserDictionary/Words/=sitnik/@EntryIndexedValue">True</s:Boolean>
186187
<s:Boolean x:Key="/Default/UserDictionary/Words/=sproj/@EntryIndexedValue">True</s:Boolean>
187188
<s:Boolean x:Key="/Default/UserDictionary/Words/=stackoverflow/@EntryIndexedValue">True</s:Boolean>
189+
<s:Boolean x:Key="/Default/UserDictionary/Words/=stddev/@EntryIndexedValue">True</s:Boolean>
188190
<s:Boolean x:Key="/Default/UserDictionary/Words/=stloc/@EntryIndexedValue">True</s:Boolean>
189191
<s:Boolean x:Key="/Default/UserDictionary/Words/=Sturges/@EntryIndexedValue">True</s:Boolean>
190192
<s:Boolean x:Key="/Default/UserDictionary/Words/=subfolder/@EntryIndexedValue">True</s:Boolean>

‎src/BenchmarkDotNet/Attributes/Exporters/OpenMetricsExporterAttribute.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using BenchmarkDotNet.Exporters;
2+
using BenchmarkDotNet.Exporters.OpenMetrics;
3+
using JetBrains.Annotations;
4+
5+
namespace BenchmarkDotNet.Attributes
6+
{
7+
[PublicAPI]
8+
public class OpenMetricsExporterAttribute : ExporterConfigBaseAttribute
9+
{
10+
public OpenMetricsExporterAttribute() : base(OpenMetricsExporter.Default)
11+
{
12+
}
13+
}
14+
}

‎src/BenchmarkDotNet/Exporters/DefaultExporters.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using BenchmarkDotNet.Exporters.Csv;
22
using BenchmarkDotNet.Exporters.Json;
3+
using BenchmarkDotNet.Exporters.OpenMetrics;
34
using BenchmarkDotNet.Exporters.Xml;
45
using JetBrains.Annotations;
56

@@ -12,6 +13,7 @@ public static class DefaultExporters
1213
[PublicAPI] public static readonly IExporter CsvMeasurements = CsvMeasurementsExporter.Default;
1314
[PublicAPI] public static readonly IExporter Html = HtmlExporter.Default;
1415
[PublicAPI] public static readonly IExporter Markdown = MarkdownExporter.Default;
16+
[PublicAPI] public static readonly IExporter OpenMetrics = OpenMetricsExporter.Default;
1517
[PublicAPI] public static readonly IExporter Plain = PlainExporter.Default;
1618
[PublicAPI] public static readonly IExporter RPlot = RPlotExporter.Default;
1719

‎src/BenchmarkDotNet/Exporters/OpenMetrics/OpenMetricsExporter.cs

Lines changed: 305 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,305 @@
1+
using System.Collections.Generic;
2+
using System.Collections.Immutable;
3+
using System.Linq;
4+
using BenchmarkDotNet.Loggers;
5+
using BenchmarkDotNet.Parameters;
6+
using BenchmarkDotNet.Reports;
7+
using BenchmarkDotNet.Running;
8+
using System;
9+
using System.Text;
10+
using BenchmarkDotNet.Engines;
11+
using BenchmarkDotNet.Extensions;
12+
using BenchmarkDotNet.Mathematics;
13+
14+
namespace BenchmarkDotNet.Exporters.OpenMetrics;
15+
16+
public class OpenMetricsExporter : ExporterBase
17+
{
18+
private const string MetricPrefix = "benchmark_";
19+
protected override string FileExtension => "metrics";
20+
protected override string FileCaption => "openmetrics";
21+
22+
public static readonly IExporter Default = new OpenMetricsExporter();
23+
24+
public override void ExportToLog(Summary summary, ILogger logger)
25+
{
26+
var metricsSet = new HashSet<OpenMetric>();
27+
28+
foreach (var report in summary.Reports)
29+
{
30+
var benchmark = report.BenchmarkCase;
31+
var gcStats = report.GcStats;
32+
var descriptor = benchmark.Descriptor;
33+
var parameters = benchmark.Parameters;
34+
35+
var stats = report.ResultStatistics;
36+
var metrics = report.Metrics;
37+
if (stats == null)
38+
continue;
39+
40+
AddCommonMetrics(metricsSet, descriptor, parameters, stats, gcStats);
41+
AddAdditionalMetrics(metricsSet, metrics, descriptor, parameters);
42+
}
43+
44+
WriteMetricsToLogger(logger, metricsSet);
45+
}
46+
47+
private static void AddCommonMetrics(HashSet<OpenMetric> metricsSet, Descriptor descriptor, ParameterInstances parameters, Statistics stats, GcStats gcStats)
48+
{
49+
metricsSet.AddRange([
50+
// Mean
51+
OpenMetric.FromStatistics(
52+
$"{MetricPrefix}mean_nanoseconds",
53+
"Mean execution time in nanoseconds.",
54+
"gauge",
55+
descriptor,
56+
parameters,
57+
stats.Mean),
58+
// Error
59+
OpenMetric.FromStatistics(
60+
$"{MetricPrefix}error_nanoseconds",
61+
"Standard error of the mean execution time in nanoseconds.",
62+
"gauge",
63+
descriptor,
64+
parameters,
65+
stats.StandardError),
66+
// Standard Deviation
67+
OpenMetric.FromStatistics(
68+
$"{MetricPrefix}stddev_nanoseconds",
69+
"Standard deviation of execution time in nanoseconds.",
70+
"gauge",
71+
descriptor,
72+
parameters,
73+
stats.StandardDeviation),
74+
// GC Stats Gen0
75+
OpenMetric.FromStatistics(
76+
$"{MetricPrefix}gc_gen0_collections",
77+
"Number of Gen 0 garbage collections during the benchmark execution.",
78+
"gauge",
79+
descriptor,
80+
parameters,
81+
gcStats.Gen0Collections),
82+
// GC Stats Gen1
83+
OpenMetric.FromStatistics(
84+
$"{MetricPrefix}gc_gen1_collections",
85+
"Number of Gen 1 garbage collections during the benchmark execution.",
86+
"gauge",
87+
descriptor,
88+
parameters,
89+
gcStats.Gen1Collections),
90+
// GC Stats Gen2
91+
OpenMetric.FromStatistics(
92+
$"{MetricPrefix}gc_gen2_collections",
93+
"Number of Gen 2 garbage collections during the benchmark execution.",
94+
"gauge",
95+
descriptor,
96+
parameters,
97+
gcStats.Gen2Collections),
98+
// Total GC Operations
99+
OpenMetric.FromStatistics(
100+
$"{MetricPrefix}gc_total_operations",
101+
"Total number of garbage collection operations during the benchmark execution.",
102+
"gauge",
103+
descriptor,
104+
parameters,
105+
gcStats.TotalOperations),
106+
// P90
107+
OpenMetric.FromStatistics(
108+
$"{MetricPrefix}p90_nanoseconds",
109+
"90th percentile execution time in nanoseconds.",
110+
"gauge",
111+
descriptor,
112+
parameters,
113+
stats.Percentiles.P90),
114+
// P95
115+
OpenMetric.FromStatistics(
116+
$"{MetricPrefix}p95_nanoseconds",
117+
"95th percentile execution time in nanoseconds.",
118+
"gauge",
119+
descriptor,
120+
parameters,
121+
stats.Percentiles.P95)
122+
]);
123+
}
124+
125+
private static void AddAdditionalMetrics(HashSet<OpenMetric> metricsSet, IReadOnlyDictionary<string, Metric> metrics, Descriptor descriptor, ParameterInstances parameters)
126+
{
127+
var reservedMetricNames = new HashSet<string>
128+
{
129+
$"{MetricPrefix}mean_nanoseconds",
130+
$"{MetricPrefix}error_nanoseconds",
131+
$"{MetricPrefix}stddev_nanoseconds",
132+
$"{MetricPrefix}gc_gen0_collections",
133+
$"{MetricPrefix}gc_gen1_collections",
134+
$"{MetricPrefix}gc_gen2_collections",
135+
$"{MetricPrefix}gc_total_operations",
136+
$"{MetricPrefix}p90_nanoseconds",
137+
$"{MetricPrefix}p95_nanoseconds"
138+
};
139+
140+
foreach (var metric in metrics)
141+
{
142+
string metricName = SanitizeMetricName(metric.Key);
143+
string fullMetricName = $"{MetricPrefix}{metricName}";
144+
145+
if (reservedMetricNames.Contains(fullMetricName))
146+
continue;
147+
148+
metricsSet.Add(OpenMetric.FromMetric(
149+
fullMetricName,
150+
metric,
151+
"gauge", // Assuming all additional metrics are of type "gauge"
152+
descriptor,
153+
parameters));
154+
}
155+
}
156+
157+
private static void WriteMetricsToLogger(ILogger logger, HashSet<OpenMetric> metricsSet)
158+
{
159+
var emittedHelpType = new HashSet<string>();
160+
161+
foreach (var metric in metricsSet.OrderBy(m => m.Name))
162+
{
163+
if (!emittedHelpType.Contains(metric.Name))
164+
{
165+
logger.WriteLine($"# HELP {metric.Name} {metric.Help}");
166+
logger.WriteLine($"# TYPE {metric.Name} {metric.Type}");
167+
emittedHelpType.Add(metric.Name);
168+
}
169+
170+
logger.WriteLine(metric.ToString());
171+
}
172+
173+
logger.WriteLine("# EOF");
174+
}
175+
176+
private static string SanitizeMetricName(string name)
177+
{
178+
var builder = new StringBuilder();
179+
bool lastWasUnderscore = false;
180+
181+
foreach (char c in name.ToLowerInvariant())
182+
{
183+
if (char.IsLetterOrDigit(c) || c == '_')
184+
{
185+
builder.Append(c);
186+
lastWasUnderscore = false;
187+
}
188+
else if (!lastWasUnderscore)
189+
{
190+
builder.Append('_');
191+
lastWasUnderscore = true;
192+
}
193+
}
194+
195+
string? result = builder.ToString().Trim('_'); // <-- Trim here
196+
197+
if (result.Length > 0 && char.IsDigit(result[0]))
198+
result = "_" + result;
199+
200+
return result;
201+
}
202+
203+
private class OpenMetric : IEquatable<OpenMetric>
204+
{
205+
internal string Name { get; }
206+
internal string Help { get; }
207+
internal string Type { get; }
208+
private readonly ImmutableSortedDictionary<string, string> labels;
209+
private readonly double value;
210+
211+
private OpenMetric(string name, string help, string type, ImmutableSortedDictionary<string, string> labels, double value)
212+
{
213+
if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Metric name cannot be null or empty.");
214+
if (string.IsNullOrWhiteSpace(type)) throw new ArgumentException("Metric type cannot be null or empty.");
215+
216+
Name = name;
217+
Help = help;
218+
Type = type;
219+
this.labels = labels ?? throw new ArgumentNullException(nameof(labels));
220+
this.value = value;
221+
}
222+
223+
public static OpenMetric FromStatistics(string name, string help, string type, Descriptor descriptor, ParameterInstances parameters, double value)
224+
{
225+
var labels = BuildLabelDict(descriptor, parameters);
226+
return new OpenMetric(name, help, type, labels, value);
227+
}
228+
229+
public static OpenMetric FromMetric(string fullMetricName, KeyValuePair<string, Metric> metric, string type, Descriptor descriptor, ParameterInstances parameters)
230+
{
231+
string help = $"Additional metric {metric.Key}";
232+
var labels = BuildLabelDict(descriptor, parameters);
233+
return new OpenMetric(fullMetricName, help, type, labels, metric.Value.Value);
234+
}
235+
236+
private static readonly Dictionary<string, string> NormalizedLabelKeyCache = new();
237+
private static string NormalizeLabelKey(string key)
238+
{
239+
string normalized = new(key
240+
.ToLowerInvariant()
241+
.Select(c => char.IsLetterOrDigit(c) ? c : '_')
242+
.ToArray());
243+
return normalized;
244+
}
245+
246+
private static ImmutableSortedDictionary<string, string> BuildLabelDict(Descriptor descriptor, ParameterInstances parameters)
247+
{
248+
var dict = new SortedDictionary<string, string>
249+
{
250+
["method"] = descriptor.WorkloadMethod.Name,
251+
["type"] = descriptor.Type.Name
252+
};
253+
foreach (var param in parameters.Items)
254+
{
255+
string key = NormalizeLabelKey(param.Name);
256+
string value = param.Value.ToString().Replace("\\", @"\\").Replace("\"", "\\\"");
257+
dict[key] = value;
258+
}
259+
return dict.ToImmutableSortedDictionary();
260+
}
261+
262+
public override bool Equals(object? obj) => Equals(obj as OpenMetric);
263+
264+
public bool Equals(OpenMetric? other)
265+
{
266+
if (other is null)
267+
return false;
268+
269+
return Name == other.Name
270+
&& value.Equals(other.value)
271+
&& labels.Count == other.labels.Count
272+
&& labels.All(kv => other.labels.TryGetValue(kv.Key, out string? otherValue) && kv.Value == otherValue);
273+
}
274+
275+
public override int GetHashCode()
276+
{
277+
var hash = new HashCode();
278+
hash.Add(Name);
279+
hash.Add(value);
280+
281+
foreach (var kv in labels)
282+
{
283+
hash.Add(kv.Key);
284+
hash.Add(kv.Value);
285+
}
286+
287+
return hash.ToHashCode();
288+
}
289+
290+
public override string ToString()
291+
{
292+
string labelStr = labels.Count > 0
293+
? $"{{{string.Join(", ", labels.Select(kvp => $"{Escape(kvp.Key)}=\"{Escape(kvp.Value)}\""))}}}"
294+
: string.Empty;
295+
return $"{Name}{labelStr} {value}";
296+
297+
static string Escape(string s) =>
298+
s.Replace("\\", @"\\")
299+
.Replace("\"", "\\\"")
300+
.Replace("\n", "\\n")
301+
.Replace("\r", "\\r")
302+
.Replace("\t", "\\t");
303+
}
304+
}
305+
}

‎tests/BenchmarkDotNet.IntegrationTests/ValidatorsTest.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using Xunit;
99
using Xunit.Abstractions;
1010
using BenchmarkDotNet.Exporters.Json;
11+
using BenchmarkDotNet.Exporters.OpenMetrics;
1112
using BenchmarkDotNet.Exporters.Xml;
1213

1314
namespace BenchmarkDotNet.IntegrationTests
@@ -23,6 +24,7 @@ public ValidatorsTest(ITestOutputHelper output) : base(output) { }
2324
MarkdownExporter.Default,
2425
MarkdownExporter.GitHub,
2526
MarkdownExporter.StackOverflow,
27+
OpenMetricsExporter.Default,
2628
CsvExporter.Default,
2729
CsvMeasurementsExporter.Default,
2830
HtmlExporter.Default,

‎tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@
88
using BenchmarkDotNet.Diagnosers;
99
using BenchmarkDotNet.Exporters;
1010
using BenchmarkDotNet.Exporters.Json;
11+
using BenchmarkDotNet.Exporters.OpenMetrics;
1112
using BenchmarkDotNet.Exporters.Xml;
1213
using BenchmarkDotNet.Loggers;
1314
using BenchmarkDotNet.Reports;
14-
using BenchmarkDotNet.Tests.Builders;
1515
using BenchmarkDotNet.Tests.Infra;
1616
using BenchmarkDotNet.Tests.Mocks;
1717
using BenchmarkDotNet.Tests.Reports;
@@ -95,6 +95,7 @@ private static IEnumerable<IExporter> GetExporters()
9595
yield return MarkdownExporter.Console;
9696
yield return MarkdownExporter.GitHub;
9797
yield return MarkdownExporter.StackOverflow;
98+
yield return OpenMetricsExporter.Default;
9899
yield return PlainExporter.Default;
99100
yield return XmlExporter.Brief;
100101
yield return XmlExporter.BriefCompressed;

‎tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
using BenchmarkDotNet.Tests.Mocks;
1010
using BenchmarkDotNet.Attributes;
1111
using BenchmarkDotNet.Configs;
12-
using BenchmarkDotNet.Tests.Builders;
1312
using BenchmarkDotNet.Tests.Infra;
1413
using BenchmarkDotNet.Validators;
1514
using JetBrains.Annotations;

0 commit comments

Comments
 (0)