Skip to content

Commit 57d4125

Browse files
committed
Add exemplars to counter and histogram
Signed-off-by: beorn7 <[email protected]>
1 parent 803ef2a commit 57d4125

File tree

8 files changed

+227
-60
lines changed

8 files changed

+227
-60
lines changed

examples/random/main.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package main
1818

1919
import (
2020
"flag"
21+
"fmt"
2122
"log"
2223
"math"
2324
"math/rand"
@@ -89,7 +90,11 @@ func main() {
8990
for {
9091
v := (rand.NormFloat64() * *normDomain) + *normMean
9192
rpcDurations.WithLabelValues("normal").Observe(v)
92-
rpcDurationsHistogram.Observe(v)
93+
rpcDurationsHistogram.ObserveWithExemplar(
94+
// Demonstrate exemplar support with a dummy ID. This would be
95+
// something like a trace ID in a real application.
96+
v, prometheus.Labels{"dummyID": fmt.Sprint(rand.Intn(100000))},
97+
)
9398
time.Sleep(time.Duration(75*oscillationFactor()) * time.Millisecond)
9499
}
95100
}()
@@ -103,6 +108,12 @@ func main() {
103108
}()
104109

105110
// Expose the registered metrics via HTTP.
106-
http.Handle("/metrics", promhttp.Handler())
111+
http.Handle("/metrics", promhttp.HandlerFor(
112+
prometheus.DefaultGatherer,
113+
promhttp.HandlerOpts{
114+
// Opt into OpenMetrics to support exemplars.
115+
EnableOpenMetrics: true,
116+
},
117+
))
107118
log.Fatal(http.ListenAndServe(*addr, nil))
108119
}

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ require (
55
github.com/cespare/xxhash/v2 v2.1.1
66
github.com/golang/protobuf v1.3.2
77
github.com/json-iterator/go v1.1.8
8-
github.com/prometheus/client_model v0.1.0
9-
github.com/prometheus/common v0.7.0
8+
github.com/prometheus/client_model v0.2.0
9+
github.com/prometheus/common v0.9.0
1010
github.com/prometheus/procfs v0.0.8
1111
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f
1212
)

go.sum

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,12 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f
5858
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
5959
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
6060
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
61-
github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
62-
github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
61+
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
62+
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
6363
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
6464
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
65-
github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
66-
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
65+
github.com/prometheus/common v0.9.0 h1:yg//x/8DqN+PxXTBFMwVCopGqDn3wSxmbF/3PCuu1bk=
66+
github.com/prometheus/common v0.9.0/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
6767
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
6868
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
6969
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -97,4 +97,4 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
9797
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
9898
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
9999
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
100-
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
100+
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

prometheus/counter.go

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"errors"
1818
"math"
1919
"sync/atomic"
20+
"time"
2021

2122
dto "github.com/prometheus/client_model/go"
2223
)
@@ -40,6 +41,14 @@ type Counter interface {
4041
// Add adds the given value to the counter. It panics if the value is <
4142
// 0.
4243
Add(float64)
44+
// AddWithExemplar works like Add but also replaces the currently saved
45+
// exemplar (if any) with a new one, created from the provided value,
46+
// the current time as timestamp, and the provided labels. Empty Labels
47+
// will lead to a valid (label-less) exemplar. But if Labels is nil, the
48+
// current exemplar is left in place. This method panics if the value is
49+
// < 0, if any of the provided labels are invalid, or if the provided
50+
// labels contain more than 64 runes in total.
51+
AddWithExemplar(value float64, exemplar Labels)
4352
}
4453

4554
// CounterOpts is an alias for Opts. See there for doc comments.
@@ -78,6 +87,7 @@ type counter struct {
7887
desc *Desc
7988

8089
labelPairs []*dto.LabelPair
90+
exemplar atomic.Value // Containing nil or a *dto.Exemplar.
8191
}
8292

8393
func (c *counter) Desc() *Desc {
@@ -88,6 +98,7 @@ func (c *counter) Add(v float64) {
8898
if v < 0 {
8999
panic(errors.New("counter cannot decrease in value"))
90100
}
101+
91102
ival := uint64(v)
92103
if float64(ival) == v {
93104
atomic.AddUint64(&c.valInt, ival)
@@ -103,6 +114,11 @@ func (c *counter) Add(v float64) {
103114
}
104115
}
105116

117+
func (c *counter) AddWithExemplar(v float64, e Labels) {
118+
c.Add(v)
119+
c.updateExemplar(v, e)
120+
}
121+
106122
func (c *counter) Inc() {
107123
atomic.AddUint64(&c.valInt, 1)
108124
}
@@ -112,7 +128,23 @@ func (c *counter) Write(out *dto.Metric) error {
112128
ival := atomic.LoadUint64(&c.valInt)
113129
val := fval + float64(ival)
114130

115-
return populateMetric(CounterValue, val, c.labelPairs, out)
131+
var exemplar *dto.Exemplar
132+
if e := c.exemplar.Load(); e != nil {
133+
exemplar = e.(*dto.Exemplar)
134+
}
135+
136+
return populateMetric(CounterValue, val, c.labelPairs, exemplar, out)
137+
}
138+
139+
func (c *counter) updateExemplar(v float64, l Labels) {
140+
if l == nil {
141+
return
142+
}
143+
e, err := newExemplar(v, time.Now(), l)
144+
if err != nil {
145+
panic(err)
146+
}
147+
c.exemplar.Store(e)
116148
}
117149

118150
// CounterVec is a Collector that bundles a set of Counters that all share the

prometheus/gauge.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func (g *gauge) Sub(val float64) {
123123

124124
func (g *gauge) Write(out *dto.Metric) error {
125125
val := math.Float64frombits(atomic.LoadUint64(&g.valBits))
126-
return populateMetric(GaugeValue, val, g.labelPairs, out)
126+
return populateMetric(GaugeValue, val, g.labelPairs, nil, out)
127127
}
128128

129129
// GaugeVec is a Collector that bundles a set of Gauges that all share the same

prometheus/histogram.go

Lines changed: 82 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"sort"
2121
"sync"
2222
"sync/atomic"
23+
"time"
2324

2425
"github.com/golang/protobuf/proto"
2526

@@ -47,6 +48,15 @@ type Histogram interface {
4748

4849
// Observe adds a single observation to the histogram.
4950
Observe(float64)
51+
// ObserveWithExemplar works like Observe but also replaces the
52+
// currently saved exemplar for the relevant bucket (possibly none) with
53+
// a new one, created from the provided value, the current time as
54+
// timestamp, and the provided Labels. Empty Labels will lead to a valid
55+
// (label-less) exemplar. But if Labels is nil, the current exemplar in
56+
// the relevant bucket is left in place. This method panics if any of
57+
// the provided labels are invalid or if the provided labels contain
58+
// more than 64 runes in total.
59+
ObserveWithExemplar(value float64, exemplar Labels)
5060
}
5161

5262
// bucketLabel is used for the label that defines the upper bound of a
@@ -205,9 +215,10 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
205215
}
206216
}
207217
// Finally we know the final length of h.upperBounds and can make buckets
208-
// for both counts:
218+
// for both counts as well as exemplars:
209219
h.counts[0].buckets = make([]uint64, len(h.upperBounds))
210220
h.counts[1].buckets = make([]uint64, len(h.upperBounds))
221+
h.exemplars = make([]atomic.Value, len(h.upperBounds)+1)
211222

212223
h.init(h) // Init self-collection.
213224
return h
@@ -254,43 +265,21 @@ type histogram struct {
254265

255266
upperBounds []float64
256267
labelPairs []*dto.LabelPair
268+
exemplars []atomic.Value // One more than buckets (to include +Inf), each a *dto.Exemplar.
257269
}
258270

259271
func (h *histogram) Desc() *Desc {
260272
return h.desc
261273
}
262274

263275
func (h *histogram) Observe(v float64) {
264-
// TODO(beorn7): For small numbers of buckets (<30), a linear search is
265-
// slightly faster than the binary search. If we really care, we could
266-
// switch from one search strategy to the other depending on the number
267-
// of buckets.
268-
//
269-
// Microbenchmarks (BenchmarkHistogramNoLabels):
270-
// 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op
271-
// 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op
272-
// 300 buckets: 154 ns/op linear - binary 61.6 ns/op
273-
i := sort.SearchFloat64s(h.upperBounds, v)
274-
275-
// We increment h.countAndHotIdx so that the counter in the lower
276-
// 63 bits gets incremented. At the same time, we get the new value
277-
// back, which we can use to find the currently-hot counts.
278-
n := atomic.AddUint64(&h.countAndHotIdx, 1)
279-
hotCounts := h.counts[n>>63]
276+
h.observe(v, h.findBucket(v))
277+
}
280278

281-
if i < len(h.upperBounds) {
282-
atomic.AddUint64(&hotCounts.buckets[i], 1)
283-
}
284-
for {
285-
oldBits := atomic.LoadUint64(&hotCounts.sumBits)
286-
newBits := math.Float64bits(math.Float64frombits(oldBits) + v)
287-
if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) {
288-
break
289-
}
290-
}
291-
// Increment count last as we take it as a signal that the observation
292-
// is complete.
293-
atomic.AddUint64(&hotCounts.count, 1)
279+
func (h *histogram) ObserveWithExemplar(v float64, e Labels) {
280+
i := h.findBucket(v)
281+
h.observe(v, i)
282+
h.updateExemplar(v, i, e)
294283
}
295284

296285
func (h *histogram) Write(out *dto.Metric) error {
@@ -329,6 +318,18 @@ func (h *histogram) Write(out *dto.Metric) error {
329318
CumulativeCount: proto.Uint64(cumCount),
330319
UpperBound: proto.Float64(upperBound),
331320
}
321+
if e := h.exemplars[i].Load(); e != nil {
322+
his.Bucket[i].Exemplar = e.(*dto.Exemplar)
323+
}
324+
}
325+
// If there is an exemplar for the +Inf bucket, we have to add that bucket explicitly.
326+
if e := h.exemplars[len(h.upperBounds)].Load(); e != nil {
327+
b := &dto.Bucket{
328+
CumulativeCount: proto.Uint64(count),
329+
UpperBound: proto.Float64(math.Inf(1)),
330+
Exemplar: e.(*dto.Exemplar),
331+
}
332+
his.Bucket = append(his.Bucket, b)
332333
}
333334

334335
out.Histogram = his
@@ -352,6 +353,57 @@ func (h *histogram) Write(out *dto.Metric) error {
352353
return nil
353354
}
354355

356+
// findBucket returns the index of the bucket for the provided value, or
357+
// len(h.upperBounds) for the +Inf bucket.
358+
func (h *histogram) findBucket(v float64) int {
359+
// TODO(beorn7): For small numbers of buckets (<30), a linear search is
360+
// slightly faster than the binary search. If we really care, we could
361+
// switch from one search strategy to the other depending on the number
362+
// of buckets.
363+
//
364+
// Microbenchmarks (BenchmarkHistogramNoLabels):
365+
// 11 buckets: 38.3 ns/op linear - binary 48.7 ns/op
366+
// 100 buckets: 78.1 ns/op linear - binary 54.9 ns/op
367+
// 300 buckets: 154 ns/op linear - binary 61.6 ns/op
368+
return sort.SearchFloat64s(h.upperBounds, v)
369+
}
370+
371+
// observe is the implementation for Observe without the findBucket part.
372+
func (h *histogram) observe(v float64, bucket int) {
373+
// We increment h.countAndHotIdx so that the counter in the lower
374+
// 63 bits gets incremented. At the same time, we get the new value
375+
// back, which we can use to find the currently-hot counts.
376+
n := atomic.AddUint64(&h.countAndHotIdx, 1)
377+
hotCounts := h.counts[n>>63]
378+
379+
if bucket < len(h.upperBounds) {
380+
atomic.AddUint64(&hotCounts.buckets[bucket], 1)
381+
}
382+
for {
383+
oldBits := atomic.LoadUint64(&hotCounts.sumBits)
384+
newBits := math.Float64bits(math.Float64frombits(oldBits) + v)
385+
if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) {
386+
break
387+
}
388+
}
389+
// Increment count last as we take it as a signal that the observation
390+
// is complete.
391+
atomic.AddUint64(&hotCounts.count, 1)
392+
}
393+
394+
// updateExemplar replaces the exemplar for the provided bucket. With empty
395+
// labels, it's a no-op. It panics if any of the labels is invalid.
396+
func (h *histogram) updateExemplar(v float64, bucket int, l Labels) {
397+
if l == nil {
398+
return
399+
}
400+
e, err := newExemplar(v, time.Now(), l)
401+
if err != nil {
402+
panic(err)
403+
}
404+
h.exemplars[bucket].Store(e)
405+
}
406+
355407
// HistogramVec is a Collector that bundles a set of Histograms that all share the
356408
// same Desc, but have different values for their variable labels. This is used
357409
// if you want to count the same thing partitioned by various dimensions

0 commit comments

Comments
 (0)