Skip to content

Commit 8177a39

Browse files
robertcoltheartKielekvishweshbankwarCodeBlanch
authored
[prometheus] Fix issue with corrupted buffers when reading both OpenMetrics and plain text formats (#5623)
Co-authored-by: Piotr Kiełkowicz <[email protected]> Co-authored-by: Vishwesh Bankwar <[email protected]> Co-authored-by: Mikel Blanchard <[email protected]>
1 parent b444464 commit 8177a39

File tree

7 files changed

+207
-86
lines changed

7 files changed

+207
-86
lines changed

src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
## Unreleased
44

5+
* Fixed an issue with corrupted buffers when reading both OpenMetrics and
6+
plain text formats from Prometheus exporters.
7+
([#5623](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5623))
8+
59
## 1.8.0-rc.1
610

711
Released 2024-Mar-27

src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@ public async Task InvokeAsync(HttpContext httpContext)
5757

5858
try
5959
{
60-
if (collectionResponse.View.Count > 0)
60+
var dataView = openMetricsRequested ? collectionResponse.OpenMetricsView : collectionResponse.PlainTextView;
61+
62+
if (dataView.Count > 0)
6163
{
6264
response.StatusCode = 200;
6365
#if NET8_0_OR_GREATER
@@ -69,7 +71,7 @@ public async Task InvokeAsync(HttpContext httpContext)
6971
? "application/openmetrics-text; version=1.0.0; charset=utf-8"
7072
: "text/plain; charset=utf-8; version=0.0.4";
7173

72-
await response.Body.WriteAsync(collectionResponse.View.Array, 0, collectionResponse.View.Count).ConfigureAwait(false);
74+
await response.Body.WriteAsync(dataView.Array, 0, dataView.Count).ConfigureAwait(false);
7375
}
7476
else
7577
{

src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
## Unreleased
44

5+
* Fixed an issue with corrupted buffers when reading both OpenMetrics and
6+
plain text formats from Prometheus exporters.
7+
([#5623](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5623))
8+
59
## 1.8.0-rc.1
610

711
Released 2024-Mar-27

src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusCollectionManager.cs

Lines changed: 75 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,14 @@ internal sealed class PrometheusCollectionManager
1616
private readonly Dictionary<Metric, PrometheusMetric> metricsCache;
1717
private readonly HashSet<string> scopes;
1818
private int metricsCacheCount;
19-
private byte[] buffer = new byte[85000]; // encourage the object to live in LOH (large object heap)
19+
private byte[] plainTextBuffer = new byte[85000]; // encourage the object to live in LOH (large object heap)
20+
private byte[] openMetricsBuffer = new byte[85000]; // encourage the object to live in LOH (large object heap)
2021
private int targetInfoBufferLength = -1; // zero or positive when target_info has been written for the first time
22+
private ArraySegment<byte> previousPlainTextDataView;
23+
private ArraySegment<byte> previousOpenMetricsDataView;
2124
private int globalLockState;
22-
private ArraySegment<byte> previousDataView;
23-
private DateTime? previousDataViewGeneratedAtUtc;
25+
private DateTime? previousPlainTextDataViewGeneratedAtUtc;
26+
private DateTime? previousOpenMetricsDataViewGeneratedAtUtc;
2427
private int readerCount;
2528
private bool collectionRunning;
2629
private TaskCompletionSource<CollectionResponse> collectionTcs;
@@ -44,16 +47,20 @@ public Task<CollectionResponse> EnterCollect(bool openMetricsRequested)
4447

4548
// If we are within {ScrapeResponseCacheDurationMilliseconds} of the
4649
// last successful collect, return the previous view.
47-
if (this.previousDataViewGeneratedAtUtc.HasValue
50+
var previousDataViewGeneratedAtUtc = openMetricsRequested
51+
? this.previousOpenMetricsDataViewGeneratedAtUtc
52+
: this.previousPlainTextDataViewGeneratedAtUtc;
53+
54+
if (previousDataViewGeneratedAtUtc.HasValue
4855
&& this.scrapeResponseCacheDurationMilliseconds > 0
49-
&& this.previousDataViewGeneratedAtUtc.Value.AddMilliseconds(this.scrapeResponseCacheDurationMilliseconds) >= DateTime.UtcNow)
56+
&& previousDataViewGeneratedAtUtc.Value.AddMilliseconds(this.scrapeResponseCacheDurationMilliseconds) >= DateTime.UtcNow)
5057
{
5158
Interlocked.Increment(ref this.readerCount);
5259
this.ExitGlobalLock();
5360
#if NET6_0_OR_GREATER
54-
return new ValueTask<CollectionResponse>(new CollectionResponse(this.previousDataView, this.previousDataViewGeneratedAtUtc.Value, fromCache: true));
61+
return new ValueTask<CollectionResponse>(new CollectionResponse(this.previousOpenMetricsDataView, this.previousPlainTextDataView, previousDataViewGeneratedAtUtc.Value, fromCache: true));
5562
#else
56-
return Task.FromResult(new CollectionResponse(this.previousDataView, this.previousDataViewGeneratedAtUtc.Value, fromCache: true));
63+
return Task.FromResult(new CollectionResponse(this.previousOpenMetricsDataView, this.previousPlainTextDataView, previousDataViewGeneratedAtUtc.Value, fromCache: true));
5764
#endif
5865
}
5966

@@ -78,16 +85,37 @@ public Task<CollectionResponse> EnterCollect(bool openMetricsRequested)
7885

7986
// Start a collection on the current thread.
8087
this.collectionRunning = true;
81-
this.previousDataViewGeneratedAtUtc = null;
88+
89+
if (openMetricsRequested)
90+
{
91+
this.previousOpenMetricsDataViewGeneratedAtUtc = null;
92+
}
93+
else
94+
{
95+
this.previousPlainTextDataViewGeneratedAtUtc = null;
96+
}
97+
8298
Interlocked.Increment(ref this.readerCount);
8399
this.ExitGlobalLock();
84100

85101
CollectionResponse response;
86102
var result = this.ExecuteCollect(openMetricsRequested);
87103
if (result)
88104
{
89-
this.previousDataViewGeneratedAtUtc = DateTime.UtcNow;
90-
response = new CollectionResponse(this.previousDataView, this.previousDataViewGeneratedAtUtc.Value, fromCache: false);
105+
if (openMetricsRequested)
106+
{
107+
this.previousOpenMetricsDataViewGeneratedAtUtc = DateTime.UtcNow;
108+
}
109+
else
110+
{
111+
this.previousPlainTextDataViewGeneratedAtUtc = DateTime.UtcNow;
112+
}
113+
114+
previousDataViewGeneratedAtUtc = openMetricsRequested
115+
? this.previousOpenMetricsDataViewGeneratedAtUtc
116+
: this.previousPlainTextDataViewGeneratedAtUtc;
117+
118+
response = new CollectionResponse(this.previousOpenMetricsDataView, this.previousPlainTextDataView, previousDataViewGeneratedAtUtc.Value, fromCache: false);
91119
}
92120
else
93121
{
@@ -170,6 +198,7 @@ private bool ExecuteCollect(bool openMetricsRequested)
170198
private ExportResult OnCollect(Batch<Metric> metrics)
171199
{
172200
var cursor = 0;
201+
var buffer = this.exporter.OpenMetricsRequested ? this.openMetricsBuffer : this.plainTextBuffer;
173202

174203
try
175204
{
@@ -192,13 +221,13 @@ private ExportResult OnCollect(Batch<Metric> metrics)
192221
{
193222
try
194223
{
195-
cursor = PrometheusSerializer.WriteScopeInfo(this.buffer, cursor, metric.MeterName);
224+
cursor = PrometheusSerializer.WriteScopeInfo(buffer, cursor, metric.MeterName);
196225

197226
break;
198227
}
199228
catch (IndexOutOfRangeException)
200229
{
201-
if (!this.IncreaseBufferSize())
230+
if (!this.IncreaseBufferSize(ref buffer))
202231
{
203232
// there are two cases we might run into the following condition:
204233
// 1. we have many metrics to be exported - in this case we probably want
@@ -226,7 +255,7 @@ private ExportResult OnCollect(Batch<Metric> metrics)
226255
try
227256
{
228257
cursor = PrometheusSerializer.WriteMetric(
229-
this.buffer,
258+
buffer,
230259
cursor,
231260
metric,
232261
this.GetPrometheusMetric(metric),
@@ -236,7 +265,7 @@ private ExportResult OnCollect(Batch<Metric> metrics)
236265
}
237266
catch (IndexOutOfRangeException)
238267
{
239-
if (!this.IncreaseBufferSize())
268+
if (!this.IncreaseBufferSize(ref buffer))
240269
{
241270
throw;
242271
}
@@ -248,24 +277,40 @@ private ExportResult OnCollect(Batch<Metric> metrics)
248277
{
249278
try
250279
{
251-
cursor = PrometheusSerializer.WriteEof(this.buffer, cursor);
280+
cursor = PrometheusSerializer.WriteEof(buffer, cursor);
252281
break;
253282
}
254283
catch (IndexOutOfRangeException)
255284
{
256-
if (!this.IncreaseBufferSize())
285+
if (!this.IncreaseBufferSize(ref buffer))
257286
{
258287
throw;
259288
}
260289
}
261290
}
262291

263-
this.previousDataView = new ArraySegment<byte>(this.buffer, 0, cursor);
292+
if (this.exporter.OpenMetricsRequested)
293+
{
294+
this.previousOpenMetricsDataView = new ArraySegment<byte>(this.openMetricsBuffer, 0, cursor);
295+
}
296+
else
297+
{
298+
this.previousPlainTextDataView = new ArraySegment<byte>(this.plainTextBuffer, 0, cursor);
299+
}
300+
264301
return ExportResult.Success;
265302
}
266303
catch (Exception)
267304
{
268-
this.previousDataView = new ArraySegment<byte>(Array.Empty<byte>(), 0, 0);
305+
if (this.exporter.OpenMetricsRequested)
306+
{
307+
this.previousOpenMetricsDataView = new ArraySegment<byte>(Array.Empty<byte>(), 0, 0);
308+
}
309+
else
310+
{
311+
this.previousPlainTextDataView = new ArraySegment<byte>(Array.Empty<byte>(), 0, 0);
312+
}
313+
269314
return ExportResult.Failure;
270315
}
271316
}
@@ -278,13 +323,13 @@ private int WriteTargetInfo()
278323
{
279324
try
280325
{
281-
this.targetInfoBufferLength = PrometheusSerializer.WriteTargetInfo(this.buffer, 0, this.exporter.Resource);
326+
this.targetInfoBufferLength = PrometheusSerializer.WriteTargetInfo(this.openMetricsBuffer, 0, this.exporter.Resource);
282327

283328
break;
284329
}
285330
catch (IndexOutOfRangeException)
286331
{
287-
if (!this.IncreaseBufferSize())
332+
if (!this.IncreaseBufferSize(ref this.openMetricsBuffer))
288333
{
289334
throw;
290335
}
@@ -295,18 +340,18 @@ private int WriteTargetInfo()
295340
return this.targetInfoBufferLength;
296341
}
297342

298-
private bool IncreaseBufferSize()
343+
private bool IncreaseBufferSize(ref byte[] buffer)
299344
{
300-
var newBufferSize = this.buffer.Length * 2;
345+
var newBufferSize = buffer.Length * 2;
301346

302347
if (newBufferSize > 100 * 1024 * 1024)
303348
{
304349
return false;
305350
}
306351

307352
var newBuffer = new byte[newBufferSize];
308-
this.buffer.CopyTo(newBuffer, 0);
309-
this.buffer = newBuffer;
353+
buffer.CopyTo(newBuffer, 0);
354+
buffer = newBuffer;
310355

311356
return true;
312357
}
@@ -331,14 +376,17 @@ private PrometheusMetric GetPrometheusMetric(Metric metric)
331376

332377
public readonly struct CollectionResponse
333378
{
334-
public CollectionResponse(ArraySegment<byte> view, DateTime generatedAtUtc, bool fromCache)
379+
public CollectionResponse(ArraySegment<byte> openMetricsView, ArraySegment<byte> plainTextView, DateTime generatedAtUtc, bool fromCache)
335380
{
336-
this.View = view;
381+
this.OpenMetricsView = openMetricsView;
382+
this.PlainTextView = plainTextView;
337383
this.GeneratedAtUtc = generatedAtUtc;
338384
this.FromCache = fromCache;
339385
}
340386

341-
public ArraySegment<byte> View { get; }
387+
public ArraySegment<byte> OpenMetricsView { get; }
388+
389+
public ArraySegment<byte> PlainTextView { get; }
342390

343391
public DateTime GeneratedAtUtc { get; }
344392

src/OpenTelemetry.Exporter.Prometheus.HttpListener/PrometheusHttpListener.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,15 +153,18 @@ private async Task ProcessRequestAsync(HttpListenerContext context)
153153
try
154154
{
155155
context.Response.Headers.Add("Server", string.Empty);
156-
if (collectionResponse.View.Count > 0)
156+
157+
var dataView = openMetricsRequested ? collectionResponse.OpenMetricsView : collectionResponse.PlainTextView;
158+
159+
if (dataView.Count > 0)
157160
{
158161
context.Response.StatusCode = 200;
159162
context.Response.Headers.Add("Last-Modified", collectionResponse.GeneratedAtUtc.ToString("R"));
160163
context.Response.ContentType = openMetricsRequested
161164
? "application/openmetrics-text; version=1.0.0; charset=utf-8"
162165
: "text/plain; charset=utf-8; version=0.0.4";
163166

164-
await context.Response.OutputStream.WriteAsync(collectionResponse.View.Array, 0, collectionResponse.View.Count).ConfigureAwait(false);
167+
await context.Response.OutputStream.WriteAsync(dataView.Array, 0, dataView.Count).ConfigureAwait(false);
165168
}
166169
else
167170
{

0 commit comments

Comments
 (0)