Skip to content

Commit 464bed2

Browse files
author
Ilias Tsakiridis
committed
feat: add OpenMetrics exporter support and update related tests
1 parent e97c686 commit 464bed2

14 files changed

+898
-2
lines changed

‎BenchmarkDotNet.sln.DotSettings

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@
161161
<s:Boolean x:Key="/Default/UserDictionary/Words/=NONINFRINGEMENT/@EntryIndexedValue">True</s:Boolean>
162162
<s:Boolean x:Key="/Default/UserDictionary/Words/=notcs/@EntryIndexedValue">True</s:Boolean>
163163
<s:Boolean x:Key="/Default/UserDictionary/Words/=nuget/@EntryIndexedValue">True</s:Boolean>
164+
<s:Boolean x:Key="/Default/UserDictionary/Words/=openmetrics/@EntryIndexedValue">True</s:Boolean>
164165
<s:Boolean x:Key="/Default/UserDictionary/Words/=outofproc/@EntryIndexedValue">True</s:Boolean>
165166
<s:Boolean x:Key="/Default/UserDictionary/Words/=parameterless/@EntryIndexedValue">True</s:Boolean>
166167
<s:Boolean x:Key="/Default/UserDictionary/Words/=Partitioner/@EntryIndexedValue">True</s:Boolean>
@@ -185,6 +186,7 @@
185186
<s:Boolean x:Key="/Default/UserDictionary/Words/=sitnik/@EntryIndexedValue">True</s:Boolean>
186187
<s:Boolean x:Key="/Default/UserDictionary/Words/=sproj/@EntryIndexedValue">True</s:Boolean>
187188
<s:Boolean x:Key="/Default/UserDictionary/Words/=stackoverflow/@EntryIndexedValue">True</s:Boolean>
189+
<s:Boolean x:Key="/Default/UserDictionary/Words/=stddev/@EntryIndexedValue">True</s:Boolean>
188190
<s:Boolean x:Key="/Default/UserDictionary/Words/=stloc/@EntryIndexedValue">True</s:Boolean>
189191
<s:Boolean x:Key="/Default/UserDictionary/Words/=Sturges/@EntryIndexedValue">True</s:Boolean>
190192
<s:Boolean x:Key="/Default/UserDictionary/Words/=subfolder/@EntryIndexedValue">True</s:Boolean>

‎src/BenchmarkDotNet/Attributes/Exporters/OpenMetricsExporterAttribute.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using BenchmarkDotNet.Exporters;
2+
using BenchmarkDotNet.Exporters.OpenMetrics;
3+
using JetBrains.Annotations;
4+
5+
namespace BenchmarkDotNet.Attributes
6+
{
7+
[PublicAPI]
8+
public class OpenMetricsExporterAttribute : ExporterConfigBaseAttribute
9+
{
10+
public OpenMetricsExporterAttribute() : base(OpenMetricsExporter.Default)
11+
{
12+
}
13+
}
14+
}

‎src/BenchmarkDotNet/Exporters/DefaultExporters.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using BenchmarkDotNet.Exporters.Csv;
22
using BenchmarkDotNet.Exporters.Json;
3+
using BenchmarkDotNet.Exporters.OpenMetrics;
34
using BenchmarkDotNet.Exporters.Xml;
45
using JetBrains.Annotations;
56

@@ -12,6 +13,7 @@ public static class DefaultExporters
1213
[PublicAPI] public static readonly IExporter CsvMeasurements = CsvMeasurementsExporter.Default;
1314
[PublicAPI] public static readonly IExporter Html = HtmlExporter.Default;
1415
[PublicAPI] public static readonly IExporter Markdown = MarkdownExporter.Default;
16+
[PublicAPI] public static readonly IExporter OpenMetrics = OpenMetricsExporter.Default;
1517
[PublicAPI] public static readonly IExporter Plain = PlainExporter.Default;
1618
[PublicAPI] public static readonly IExporter RPlot = RPlotExporter.Default;
1719

‎src/BenchmarkDotNet/Exporters/OpenMetrics/OpenMetricsExporter.cs

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
using System.Collections.Generic;
2+
using System.Collections.Immutable;
3+
using System.Linq;
4+
using BenchmarkDotNet.Loggers;
5+
using BenchmarkDotNet.Parameters;
6+
using BenchmarkDotNet.Reports;
7+
using BenchmarkDotNet.Running;
8+
using System;
9+
using System.Text;
10+
using BenchmarkDotNet.Engines;
11+
using BenchmarkDotNet.Extensions;
12+
using BenchmarkDotNet.Mathematics;
13+
14+
namespace BenchmarkDotNet.Exporters.OpenMetrics;
15+
16+
public class OpenMetricsExporter : ExporterBase
17+
{
18+
private const string MetricPrefix = "benchmark_";
19+
protected override string FileExtension => "metrics";
20+
protected override string FileCaption => "openmetrics";
21+
22+
public static readonly IExporter Default = new OpenMetricsExporter();
23+
24+
public override void ExportToLog(Summary summary, ILogger logger)
25+
{
26+
var metricsSet = new HashSet<OpenMetric>();
27+
28+
foreach (var report in summary.Reports)
29+
{
30+
var benchmark = report.BenchmarkCase;
31+
var gcStats = report.GcStats;
32+
var descriptor = benchmark.Descriptor;
33+
var parameters = benchmark.Parameters;
34+
35+
var stats = report.ResultStatistics;
36+
var metrics = report.Metrics;
37+
if (stats == null)
38+
continue;
39+
40+
AddCommonMetrics(metricsSet, descriptor, parameters, stats, gcStats);
41+
AddAdditionalMetrics(metricsSet, metrics, descriptor, parameters);
42+
}
43+
44+
WriteMetricsToLogger(logger, metricsSet);
45+
}
46+
47+
private static void AddCommonMetrics(HashSet<OpenMetric> metricsSet, Descriptor descriptor, ParameterInstances parameters, Statistics stats, GcStats gcStats)
48+
{
49+
metricsSet.AddRange([
50+
// Mean
51+
OpenMetric.FromStatistics(
52+
$"{MetricPrefix}execution_time_nanoseconds",
53+
"Mean execution time in nanoseconds.",
54+
"gauge",
55+
"nanoseconds",
56+
descriptor,
57+
parameters,
58+
stats.Mean),
59+
// Error
60+
OpenMetric.FromStatistics(
61+
$"{MetricPrefix}error_nanoseconds",
62+
"Standard error of the mean execution time in nanoseconds.",
63+
"gauge",
64+
"nanoseconds",
65+
descriptor,
66+
parameters,
67+
stats.StandardError),
68+
// Standard Deviation
69+
OpenMetric.FromStatistics(
70+
$"{MetricPrefix}stddev_nanoseconds",
71+
"Standard deviation of execution time in nanoseconds.",
72+
"gauge",
73+
"nanoseconds",
74+
descriptor,
75+
parameters,
76+
stats.StandardDeviation),
77+
// GC Stats Gen0 - these are counters, not gauges
78+
OpenMetric.FromStatistics(
79+
$"{MetricPrefix}gc_gen0_collections_total",
80+
"Total number of Gen 0 garbage collections during the benchmark execution.",
81+
"counter",
82+
"",
83+
descriptor,
84+
parameters,
85+
gcStats.Gen0Collections),
86+
// GC Stats Gen1
87+
OpenMetric.FromStatistics(
88+
$"{MetricPrefix}gc_gen1_collections_total",
89+
"Total number of Gen 1 garbage collections during the benchmark execution.",
90+
"counter",
91+
"",
92+
descriptor,
93+
parameters,
94+
gcStats.Gen1Collections),
95+
// GC Stats Gen2
96+
OpenMetric.FromStatistics(
97+
$"{MetricPrefix}gc_gen2_collections_total",
98+
"Total number of Gen 2 garbage collections during the benchmark execution.",
99+
"counter",
100+
"",
101+
descriptor,
102+
parameters,
103+
gcStats.Gen2Collections),
104+
// Total GC Operations
105+
OpenMetric.FromStatistics(
106+
$"{MetricPrefix}gc_total_operations_total",
107+
"Total number of garbage collection operations during the benchmark execution.",
108+
"counter",
109+
"",
110+
descriptor,
111+
parameters,
112+
gcStats.TotalOperations),
113+
// P90 - in nanoseconds
114+
OpenMetric.FromStatistics(
115+
$"{MetricPrefix}p90_nanoseconds",
116+
"90th percentile execution time in nanoseconds.",
117+
"gauge",
118+
"nanoseconds",
119+
descriptor,
120+
parameters,
121+
stats.Percentiles.P90),
122+
// P95 - in nanoseconds
123+
OpenMetric.FromStatistics(
124+
$"{MetricPrefix}p95_nanoseconds",
125+
"95th percentile execution time in nanoseconds.",
126+
"gauge",
127+
"nanoseconds",
128+
descriptor,
129+
parameters,
130+
stats.Percentiles.P95)
131+
]);
132+
}
133+
134+
private static void AddAdditionalMetrics(HashSet<OpenMetric> metricsSet, IReadOnlyDictionary<string, Metric> metrics, Descriptor descriptor, ParameterInstances parameters)
135+
{
136+
var reservedMetricNames = new HashSet<string>
137+
{
138+
$"{MetricPrefix}execution_time_nanoseconds",
139+
$"{MetricPrefix}error_nanoseconds",
140+
$"{MetricPrefix}stddev_nanoseconds",
141+
$"{MetricPrefix}gc_gen0_collections_total",
142+
$"{MetricPrefix}gc_gen1_collections_total",
143+
$"{MetricPrefix}gc_gen2_collections_total",
144+
$"{MetricPrefix}gc_total_operations_total",
145+
$"{MetricPrefix}p90_nanoseconds",
146+
$"{MetricPrefix}p95_nanoseconds"
147+
};
148+
149+
foreach (var metric in metrics)
150+
{
151+
string metricName = SanitizeMetricName(metric.Key);
152+
string fullMetricName = $"{MetricPrefix}{metricName}";
153+
154+
if (reservedMetricNames.Contains(fullMetricName))
155+
continue;
156+
157+
metricsSet.Add(OpenMetric.FromMetric(
158+
fullMetricName,
159+
metric,
160+
"gauge", // Assuming all additional metrics are of type "gauge"
161+
descriptor,
162+
parameters));
163+
}
164+
}
165+
166+
private static void WriteMetricsToLogger(ILogger logger, HashSet<OpenMetric> metricsSet)
167+
{
168+
var emittedHelpType = new HashSet<string>();
169+
170+
foreach (var metric in metricsSet.OrderBy(m => m.Name))
171+
{
172+
if (!emittedHelpType.Contains(metric.Name))
173+
{
174+
logger.WriteLine($"# HELP {metric.Name} {metric.Help}");
175+
logger.WriteLine($"# TYPE {metric.Name} {metric.Type}");
176+
if (!string.IsNullOrEmpty(metric.Unit))
177+
{
178+
logger.WriteLine($"# UNIT {metric.Name} {metric.Unit}");
179+
}
180+
emittedHelpType.Add(metric.Name);
181+
}
182+
183+
logger.WriteLine(metric.ToString());
184+
}
185+
186+
logger.WriteLine("# EOF");
187+
}
188+
189+
private static string SanitizeMetricName(string name)
190+
{
191+
var builder = new StringBuilder();
192+
bool lastWasUnderscore = false;
193+
194+
foreach (char c in name.ToLowerInvariant())
195+
{
196+
if (char.IsLetterOrDigit(c) || c == '_')
197+
{
198+
builder.Append(c);
199+
lastWasUnderscore = false;
200+
}
201+
else if (!lastWasUnderscore)
202+
{
203+
builder.Append('_');
204+
lastWasUnderscore = true;
205+
}
206+
}
207+
208+
string? result = builder.ToString().Trim('_'); // <-- Trim here
209+
210+
if (result.Length > 0 && char.IsDigit(result[0]))
211+
result = "_" + result;
212+
213+
return result;
214+
}
215+
216+
private class OpenMetric : IEquatable<OpenMetric>
217+
{
218+
internal string Name { get; }
219+
internal string Help { get; }
220+
internal string Type { get; }
221+
internal string Unit { get; }
222+
private readonly ImmutableSortedDictionary<string, string> labels;
223+
private readonly double value;
224+
225+
private OpenMetric(string name, string help, string type, string unit, ImmutableSortedDictionary<string, string> labels, double value)
226+
{
227+
if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Metric name cannot be null or empty.");
228+
if (string.IsNullOrWhiteSpace(type)) throw new ArgumentException("Metric type cannot be null or empty.");
229+
230+
Name = name;
231+
Help = help;
232+
Type = type;
233+
Unit = unit ?? "";
234+
this.labels = labels ?? throw new ArgumentNullException(nameof(labels));
235+
this.value = value;
236+
}
237+
238+
public static OpenMetric FromStatistics(string name, string help, string type, string unit, Descriptor descriptor, ParameterInstances parameters, double value)
239+
{
240+
var labels = BuildLabelDict(descriptor, parameters);
241+
return new OpenMetric(name, help, type, unit, labels, value);
242+
}
243+
244+
public static OpenMetric FromMetric(string fullMetricName, KeyValuePair<string, Metric> metric, string type, Descriptor descriptor, ParameterInstances parameters)
245+
{
246+
string help = $"Additional metric {metric.Key}";
247+
var labels = BuildLabelDict(descriptor, parameters);
248+
return new OpenMetric(fullMetricName, help, type, "", labels, metric.Value.Value);
249+
}
250+
251+
private static readonly Dictionary<string, string> NormalizedLabelKeyCache = new();
252+
private static string NormalizeLabelKey(string key)
253+
{
254+
string normalized = new(key
255+
.ToLowerInvariant()
256+
.Select(c => char.IsLetterOrDigit(c) ? c : '_')
257+
.ToArray());
258+
return normalized;
259+
}
260+
261+
private static ImmutableSortedDictionary<string, string> BuildLabelDict(Descriptor descriptor, ParameterInstances parameters)
262+
{
263+
var dict = new SortedDictionary<string, string>
264+
{
265+
["method"] = descriptor.WorkloadMethod.Name,
266+
["type"] = descriptor.TypeInfo
267+
};
268+
foreach (var param in parameters.Items)
269+
{
270+
string key = NormalizeLabelKey(param.Name);
271+
string value = EscapeLabelValue(param.Value?.ToString() ?? "");
272+
dict[key] = value;
273+
}
274+
return dict.ToImmutableSortedDictionary();
275+
}
276+
277+
private static string EscapeLabelValue(string value)
278+
{
279+
return value.Replace("\\", @"\\")
280+
.Replace("\"", "\\\"")
281+
.Replace("\n", "\\n")
282+
.Replace("\r", "\\r")
283+
.Replace("\t", "\\t");
284+
}
285+
286+
public override bool Equals(object? obj) => Equals(obj as OpenMetric);
287+
288+
public bool Equals(OpenMetric? other)
289+
{
290+
if (other is null)
291+
return false;
292+
293+
return Name == other.Name
294+
&& value.Equals(other.value)
295+
&& labels.Count == other.labels.Count
296+
&& labels.All(kv => other.labels.TryGetValue(kv.Key, out string? otherValue) && kv.Value == otherValue);
297+
}
298+
299+
public override int GetHashCode()
300+
{
301+
var hash = new HashCode();
302+
hash.Add(Name);
303+
hash.Add(value);
304+
305+
foreach (var kv in labels)
306+
{
307+
hash.Add(kv.Key);
308+
hash.Add(kv.Value);
309+
}
310+
311+
return hash.ToHashCode();
312+
}
313+
314+
public override string ToString()
315+
{
316+
string labelStr = labels.Count > 0
317+
? $"{{{string.Join(", ", labels.Select(kvp => $"{kvp.Key}=\"{kvp.Value}\""))}}}"
318+
: string.Empty;
319+
return $"{Name}{labelStr} {value}";
320+
}
321+
}
322+
}

‎tests/BenchmarkDotNet.IntegrationTests/ValidatorsTest.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using Xunit;
99
using Xunit.Abstractions;
1010
using BenchmarkDotNet.Exporters.Json;
11+
using BenchmarkDotNet.Exporters.OpenMetrics;
1112
using BenchmarkDotNet.Exporters.Xml;
1213

1314
namespace BenchmarkDotNet.IntegrationTests
@@ -23,6 +24,7 @@ public ValidatorsTest(ITestOutputHelper output) : base(output) { }
2324
MarkdownExporter.Default,
2425
MarkdownExporter.GitHub,
2526
MarkdownExporter.StackOverflow,
27+
OpenMetricsExporter.Default,
2628
CsvExporter.Default,
2729
CsvMeasurementsExporter.Default,
2830
HtmlExporter.Default,

0 commit comments

Comments
 (0)