Skip to content

Commit ac8d8e9

Browse files
authored
Optimize Observability return types in in Prometheus exporter (#7410)
Do not allocate a return function from `ExportMetrics`, `RecordCollectionDuration`, or `RecordOperationDuration` to the heap. Use the added `ExportOp` or `Timer` type instead. ### Benchmarks #### `prometheus` ``` goos: linux goarch: amd64 pkg: go.opentelemetry.io/otel/exporters/prometheus cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz │ main.bmark.result │ prom-optimize-observ.bmark.result │ │ sec/op │ sec/op vs base │ Collect1/ObservabilityDisabled-8 27.59µ ± 7% 27.55µ ± 2% ~ (p=0.631 n=10) Collect1/ObservabilityEnabled-8 29.23µ ± 1% 27.25µ ± 7% -6.78% (p=0.004 n=10) Collect10/ObservabilityDisabled-8 70.75µ ± 3% 66.81µ ± 4% -5.57% (p=0.003 n=10) Collect10/ObservabilityEnabled-8 75.41µ ± 5% 71.13µ ± 5% -5.68% (p=0.002 n=10) Collect100/ObservabilityDisabled-8 420.7µ ± 4% 425.4µ ± 6% ~ (p=0.912 n=10) Collect100/ObservabilityEnabled-8 432.3µ ± 3% 422.2µ ± 5% ~ (p=0.105 n=10) Collect1000/ObservabilityDisabled-8 3.929m ± 31% 3.808m ± 2% -3.09% (p=0.001 n=10) Collect1000/ObservabilityEnabled-8 4.150m ± 1% 3.964m ± 4% -4.48% (p=0.003 n=10) Collect10000/ObservabilityDisabled-8 37.64m ± 6% 37.52m ± 2% ~ (p=0.739 n=10) Collect10000/ObservabilityEnabled-8 39.46m ± 2% 39.81m ± 19% ~ (p=0.436 n=10) geomean 672.6µ 654.6µ -2.68% │ main.bmark.result │ prom-optimize-observ.bmark.result │ │ B/op │ B/op vs base │ Collect1/ObservabilityDisabled-8 34.40Ki ± 0% 34.40Ki ± 0% ~ (p=0.075 n=10) Collect1/ObservabilityEnabled-8 34.64Ki ± 0% 34.43Ki ± 0% -0.60% (p=0.000 n=10) Collect10/ObservabilityDisabled-8 47.26Ki ± 0% 47.25Ki ± 0% ~ (p=0.093 n=10) Collect10/ObservabilityEnabled-8 47.93Ki ± 0% 47.30Ki ± 0% -1.33% (p=0.000 n=10) Collect100/ObservabilityDisabled-8 191.3Ki ± 0% 191.0Ki ± 0% ~ (p=0.218 n=10) Collect100/ObservabilityEnabled-8 197.0Ki ± 0% 191.6Ki ± 0% -2.74% (p=0.000 n=10) Collect1000/ObservabilityDisabled-8 1.902Mi ± 1% 1.891Mi ± 1% ~ (p=0.353 n=10) Collect1000/ObservabilityEnabled-8 1.935Mi ± 2% 1.889Mi ± 1% -2.38% (p=0.000 n=10) Collect10000/ObservabilityDisabled-8 17.67Mi ± 4% 18.17Mi ± 5% ~ (p=0.190 n=10) Collect10000/ObservabilityEnabled-8 18.62Mi ± 4% 17.98Mi ± 6% -3.42% (p=0.035 n=10) geomean 410.4Ki 406.9Ki -0.85% │ main.bmark.result │ prom-optimize-observ.bmark.result │ │ allocs/op │ allocs/op vs base │ Collect1/ObservabilityDisabled-8 61.00 ± 0% 61.00 ± 0% ~ (p=1.000 n=10) ¹ Collect1/ObservabilityEnabled-8 64.00 ± 0% 61.00 ± 0% -4.69% (p=0.000 n=10) Collect10/ObservabilityDisabled-8 410.0 ± 0% 410.0 ± 0% ~ (p=1.000 n=10) ¹ Collect10/ObservabilityEnabled-8 423.0 ± 0% 411.0 ± 0% -2.84% (p=0.000 n=10) Collect100/ObservabilityDisabled-8 3.874k ± 0% 3.874k ± 0% ~ (p=0.272 n=10) Collect100/ObservabilityEnabled-8 3.979k ± 0% 3.876k ± 0% -2.59% (p=0.000 n=10) Collect1000/ObservabilityDisabled-8 38.90k ± 0% 38.88k ± 0% ~ (p=0.306 n=10) Collect1000/ObservabilityEnabled-8 39.88k ± 0% 38.88k ± 0% -2.51% (p=0.000 n=10) Collect10000/ObservabilityDisabled-8 387.9k ± 0% 388.9k ± 0% ~ (p=0.138 n=10) Collect10000/ObservabilityEnabled-8 399.0k ± 0% 388.6k ± 1% -2.60% (p=0.000 n=10) geomean 4.364k 4.298k -1.52% ¹ all samples are equal ``` #### `prometheus/internal/observ` ```terminal goos: linux goarch: amd64 pkg: go.opentelemetry.io/otel/exporters/prometheus/internal/observ cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz │ main.bmark.result │ prom-optimize-observ.bmark.result │ │ sec/op │ sec/op vs base │ InstrumentationExportMetrics/NoError-8 92.64n ± 7% 56.07n ± 5% -39.48% (p=0.000 n=10) InstrumentationExportMetrics/AllError-8 664.6n ± 4% 579.9n ± 4% -12.74% (p=0.000 n=10) InstrumentationExportMetrics/PartialError-8 637.5n ± 10% 579.1n ± 6% -9.15% (p=0.000 n=10) InstrumentationRecordOperationDuration/NoError-8 148.3n ± 5% 109.9n ± 3% -25.89% (p=0.000 n=10) InstrumentationRecordOperationDuration/Error-8 709.9n ± 8% 613.5n ± 3% -13.58% (p=0.000 n=10) InstrumentationRecordCollectionDuration/NoError-8 150.9n ± 11% 114.1n ± 5% -24.35% (p=0.000 n=10) InstrumentationRecordCollectionDuration/Error-8 723.5n ± 12% 629.2n ± 2% -13.04% (p=0.000 n=10) geomean 332.7n 264.8n -20.42% │ main.bmark.result │ prom-optimize-observ.bmark.result │ │ B/op │ B/op vs base │ InstrumentationExportMetrics/NoError-8 48.00 ± 0% 0.00 ± 0% -100.00% (p=0.000 n=10) InstrumentationExportMetrics/AllError-8 264.0 ± 0% 216.0 ± 0% -18.18% (p=0.000 n=10) InstrumentationExportMetrics/PartialError-8 264.0 ± 0% 216.0 ± 0% -18.18% (p=0.000 n=10) InstrumentationRecordOperationDuration/NoError-8 80.00 ± 0% 0.00 ± 0% -100.00% (p=0.000 n=10) InstrumentationRecordOperationDuration/Error-8 296.0 ± 0% 216.0 ± 0% -27.03% (p=0.000 n=10) InstrumentationRecordCollectionDuration/NoError-8 80.00 ± 0% 0.00 ± 0% -100.00% (p=0.000 n=10) InstrumentationRecordCollectionDuration/Error-8 296.0 ± 0% 216.0 ± 0% -27.03% (p=0.000 n=10) geomean 152.0 ? ¹ ² ¹ summaries must be >0 to compute geomean ² ratios must be >0 to compute geomean │ main.bmark.result │ prom-optimize-observ.bmark.result │ │ allocs/op │ allocs/op vs base │ InstrumentationExportMetrics/NoError-8 1.000 ± 0% 0.000 ± 0% -100.00% (p=0.000 n=10) InstrumentationExportMetrics/AllError-8 3.000 ± 0% 2.000 ± 0% -33.33% (p=0.000 n=10) InstrumentationExportMetrics/PartialError-8 3.000 ± 0% 2.000 ± 0% -33.33% (p=0.000 n=10) InstrumentationRecordOperationDuration/NoError-8 1.000 ± 0% 0.000 ± 0% -100.00% (p=0.000 n=10) InstrumentationRecordOperationDuration/Error-8 3.000 ± 0% 2.000 ± 0% -33.33% (p=0.000 n=10) InstrumentationRecordCollectionDuration/NoError-8 1.000 ± 0% 0.000 ± 0% -100.00% (p=0.000 n=10) InstrumentationRecordCollectionDuration/Error-8 3.000 ± 0% 2.000 ± 0% -33.33% (p=0.000 n=10) geomean 1.873 ? ¹ ² ¹ summaries must be >0 to compute geomean ² ratios must be >0 to compute geomean ```
1 parent 88d3fed commit ac8d8e9

File tree

4 files changed

+179
-133
lines changed

4 files changed

+179
-133
lines changed

exporters/prometheus/benchmark_test.go

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,26 +13,37 @@ import (
1313
"go.opentelemetry.io/otel/sdk/metric"
1414
)
1515

16-
func benchmarkCollect(b *testing.B, n int) {
17-
ctx := b.Context()
18-
registry := prometheus.NewRegistry()
19-
exporter, err := New(WithRegisterer(registry))
20-
require.NoError(b, err)
21-
provider := metric.NewMeterProvider(metric.WithReader(exporter))
22-
meter := provider.Meter("testmeter")
23-
24-
for i := range n {
25-
counter, err := meter.Float64Counter(fmt.Sprintf("foo_%d", i))
16+
func run(n int) func(b *testing.B) {
17+
return func(b *testing.B) {
18+
ctx := b.Context()
19+
registry := prometheus.NewRegistry()
20+
exporter, err := New(WithRegisterer(registry))
2621
require.NoError(b, err)
27-
counter.Add(ctx, float64(i))
22+
provider := metric.NewMeterProvider(metric.WithReader(exporter))
23+
meter := provider.Meter("testmeter")
24+
25+
for i := range n {
26+
counter, err := meter.Float64Counter(fmt.Sprintf("foo_%d", i))
27+
require.NoError(b, err)
28+
counter.Add(ctx, float64(i))
29+
}
30+
31+
b.ReportAllocs()
32+
b.ResetTimer()
33+
for i := 0; i < b.N; i++ {
34+
_, err := registry.Gather()
35+
require.NoError(b, err)
36+
}
2837
}
38+
}
2939

30-
b.ReportAllocs()
31-
b.ResetTimer()
32-
for i := 0; i < b.N; i++ {
33-
_, err := registry.Gather()
34-
require.NoError(b, err)
35-
}
40+
func benchmarkCollect(b *testing.B, n int) {
41+
b.Run("ObservabilityDisabled", run(n))
42+
b.Run("ObservabilityEnabled", func(b *testing.B) {
43+
b.Setenv("OTEL_GO_X_OBSERVABILITY", "true")
44+
bmark := run(n)
45+
bmark(b)
46+
})
3647
}
3748

3849
func BenchmarkCollect1(b *testing.B) { benchmarkCollect(b, 1) }

exporters/prometheus/exporter.go

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -166,16 +166,16 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) {
166166
ctx := context.TODO()
167167

168168
if c.inst != nil {
169-
endOp := c.inst.RecordOperationDuration(ctx)
170-
defer func() { endOp(err) }()
169+
timer := c.inst.RecordOperationDuration(ctx)
170+
defer func() { timer.Stop(err) }()
171171
}
172172

173173
metrics := metricsPool.Get().(*metricdata.ResourceMetrics)
174174
defer metricsPool.Put(metrics)
175175

176176
endCollection := func(error) {}
177177
if c.inst != nil {
178-
endCollection = c.inst.RecordCollectionDuration(ctx)
178+
endCollection = c.inst.RecordCollectionDuration(ctx).Stop
179179
}
180180
err = c.reader.Collect(ctx, metrics)
181181
endCollection(err)
@@ -366,8 +366,8 @@ func addExponentialHistogramMetric[N int64 | float64](
366366
var err error
367367
var success int64
368368
if inst != nil {
369-
end := inst.ExportMetrics(ctx, int64(len(histogram.DataPoints)))
370-
defer func() { end(success, err) }()
369+
op := inst.ExportMetrics(ctx, int64(len(histogram.DataPoints)))
370+
defer func() { op.End(success, err) }()
371371
}
372372

373373
for j, dp := range histogram.DataPoints {
@@ -467,8 +467,8 @@ func addHistogramMetric[N int64 | float64](
467467
var err error
468468
var success int64
469469
if inst != nil {
470-
end := inst.ExportMetrics(ctx, int64(len(histogram.DataPoints)))
471-
defer func() { end(success, err) }()
470+
op := inst.ExportMetrics(ctx, int64(len(histogram.DataPoints)))
471+
defer func() { op.End(success, err) }()
472472
}
473473

474474
for j, dp := range histogram.DataPoints {
@@ -515,8 +515,8 @@ func addSumMetric[N int64 | float64](
515515
var err error
516516
var success int64
517517
if inst != nil {
518-
end := inst.ExportMetrics(ctx, int64(len(sum.DataPoints)))
519-
defer func() { end(success, err) }()
518+
op := inst.ExportMetrics(ctx, int64(len(sum.DataPoints)))
519+
defer func() { op.End(success, err) }()
520520
}
521521

522522
valueType := prometheus.CounterValue
@@ -565,8 +565,8 @@ func addGaugeMetric[N int64 | float64](
565565
var err error
566566
var success int64
567567
if inst != nil {
568-
end := inst.ExportMetrics(ctx, int64(len(gauge.DataPoints)))
569-
defer func() { end(success, err) }()
568+
op := inst.ExportMetrics(ctx, int64(len(gauge.DataPoints)))
569+
defer func() { op.End(success, err) }()
570570
}
571571

572572
for i, dp := range gauge.DataPoints {

exporters/prometheus/internal/observ/instrumentation.go

Lines changed: 78 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -144,81 +144,106 @@ func NewInstrumentation(id int64) (*Instrumentation, error) {
144144
return i, err
145145
}
146146

147-
// RecordDurationDone is a function that is called when a call to an Exporters'
148-
// RecordOperationDuration or RecordCollectionDuration completes.
147+
// RecordOperationDuration starts the timing of an operation.
149148
//
150-
// Any error that is encountered is provided as err.
151-
type RecordDurationDone func(error)
152-
153-
func (i *Instrumentation) RecordOperationDuration(ctx context.Context) RecordDurationDone {
154-
return i.recordDuration(ctx, i.operationDuration)
149+
// It returns a [Timer] that tracks the operation duration. The [Timer.Stop]
150+
// method must be called when the operation completes.
151+
func (i *Instrumentation) RecordOperationDuration(ctx context.Context) Timer {
152+
return Timer{
153+
ctx: ctx,
154+
start: time.Now(),
155+
inst: i,
156+
hist: i.operationDuration,
157+
}
155158
}
156159

157-
func (i *Instrumentation) RecordCollectionDuration(ctx context.Context) RecordDurationDone {
158-
return i.recordDuration(ctx, i.collectionDuration)
160+
// RecordCollectionDuration starts the timing of a collection operation.
161+
//
162+
// It returns a [Timer] that tracks the collection duration. The [Timer.Stop]
163+
// method must be called when the collection completes.
164+
func (i *Instrumentation) RecordCollectionDuration(ctx context.Context) Timer {
165+
return Timer{
166+
ctx: ctx,
167+
start: time.Now(),
168+
inst: i,
169+
hist: i.collectionDuration,
170+
}
159171
}
160172

161-
func (i *Instrumentation) recordDuration(
162-
ctx context.Context,
163-
h metric.Float64Histogram,
164-
) RecordDurationDone {
165-
start := time.Now()
173+
// Timer tracks the duration of an operation.
174+
type Timer struct {
175+
ctx context.Context
176+
start time.Time
166177

167-
return func(err error) {
168-
recordOpt := get[metric.RecordOption](recordOptPool)
169-
defer put(recordOptPool, recordOpt)
170-
*recordOpt = append(*recordOpt, i.setOpt)
171-
172-
if err != nil {
173-
attrs := get[attribute.KeyValue](measureAttrsPool)
174-
defer put(measureAttrsPool, attrs)
175-
*attrs = append(*attrs, i.attrs...)
176-
*attrs = append(*attrs, semconv.ErrorType(err))
177-
178-
set := attribute.NewSet(*attrs...)
179-
*recordOpt = append((*recordOpt)[:0], metric.WithAttributeSet(set))
180-
}
178+
inst *Instrumentation
179+
hist metric.Float64Histogram
180+
}
181181

182-
h.Record(ctx, time.Since(start).Seconds(), *recordOpt...)
182+
// Stop ends the timing operation and records the elapsed duration.
183+
//
184+
// If err is non-nil, an appropriate error type attribute will be included.
185+
func (t Timer) Stop(err error) {
186+
recordOpt := get[metric.RecordOption](recordOptPool)
187+
defer put(recordOptPool, recordOpt)
188+
*recordOpt = append(*recordOpt, t.inst.setOpt)
189+
190+
if err != nil {
191+
attrs := get[attribute.KeyValue](measureAttrsPool)
192+
defer put(measureAttrsPool, attrs)
193+
*attrs = append(*attrs, t.inst.attrs...)
194+
*attrs = append(*attrs, semconv.ErrorType(err))
195+
196+
set := attribute.NewSet(*attrs...)
197+
*recordOpt = append((*recordOpt)[:0], metric.WithAttributeSet(set))
183198
}
199+
200+
t.hist.Record(t.ctx, time.Since(t.start).Seconds(), *recordOpt...)
184201
}
185202

186-
// ExportMetricsDone is a function that is called when a call to an Exporter's
187-
// export methods completes.
203+
// ExportMetrics starts the observation of a metric export operation.
188204
//
189-
// The number of successful exports is provided as success. Any error that is
190-
// encountered is provided as err.
191-
type ExportMetricsDone func(success int64, err error)
192-
193-
func (i *Instrumentation) ExportMetrics(ctx context.Context, n int64) ExportMetricsDone {
205+
// It returns an [ExportOp] that tracks the export operation. The
206+
// [ExportOp.End] method must be called when the export completes.
207+
func (i *Instrumentation) ExportMetrics(ctx context.Context, n int64) ExportOp {
194208
addOpt := get[metric.AddOption](addOptPool)
195209
defer put(addOptPool, addOpt)
196210
*addOpt = append(*addOpt, i.setOpt)
197211

198212
i.inflightMetric.Add(ctx, n, *addOpt...)
199213

200-
return i.end(ctx, n)
214+
return ExportOp{ctx: ctx, nMetrics: n, inst: i}
215+
}
216+
217+
// ExportOp tracks a metric export operation.
218+
type ExportOp struct {
219+
ctx context.Context
220+
nMetrics int64
221+
222+
inst *Instrumentation
201223
}
202224

203-
func (i *Instrumentation) end(ctx context.Context, n int64) ExportMetricsDone {
204-
return func(success int64, err error) {
205-
addOpt := get[metric.AddOption](addOptPool)
206-
defer put(addOptPool, addOpt)
207-
*addOpt = append(*addOpt, i.setOpt)
225+
// End ends the observation of a metric export operation.
226+
//
227+
// The success parameter is the number of metrics that were successfully
228+
// exported. If a non-nil error is provided, the number of failed metrics will
229+
// be recorded with the error type attribute.
230+
func (e ExportOp) End(success int64, err error) {
231+
addOpt := get[metric.AddOption](addOptPool)
232+
defer put(addOptPool, addOpt)
233+
*addOpt = append(*addOpt, e.inst.setOpt)
208234

209-
i.inflightMetric.Add(ctx, -n, *addOpt...)
210-
i.exportedMetric.Add(ctx, success, *addOpt...)
235+
e.inst.inflightMetric.Add(e.ctx, -e.nMetrics, *addOpt...)
236+
e.inst.exportedMetric.Add(e.ctx, success, *addOpt...)
211237

212-
if err != nil {
213-
attrs := get[attribute.KeyValue](measureAttrsPool)
214-
defer put(measureAttrsPool, attrs)
215-
*attrs = append(*attrs, i.attrs...)
216-
*attrs = append(*attrs, semconv.ErrorType(err))
238+
if err != nil {
239+
attrs := get[attribute.KeyValue](measureAttrsPool)
240+
defer put(measureAttrsPool, attrs)
241+
*attrs = append(*attrs, e.inst.attrs...)
242+
*attrs = append(*attrs, semconv.ErrorType(err))
217243

218-
set := attribute.NewSet(*attrs...)
244+
set := attribute.NewSet(*attrs...)
219245

220-
*addOpt = append((*addOpt)[:0], metric.WithAttributeSet(set))
221-
i.exportedMetric.Add(ctx, n-success, *addOpt...)
222-
}
246+
*addOpt = append((*addOpt)[:0], metric.WithAttributeSet(set))
247+
e.inst.exportedMetric.Add(e.ctx, e.nMetrics-success, *addOpt...)
223248
}
224249
}

0 commit comments

Comments
 (0)