Skip to content

Commit 542f7e6

Browse files
authored
Merge pull request #1471 from fatsheep9146/native-histogram-exemplar
add native histogram exemplar support
2 parents cb57abb + 2754a4c commit 542f7e6

File tree

2 files changed

+335
-4
lines changed

2 files changed

+335
-4
lines changed

prometheus/histogram.go

Lines changed: 180 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -440,7 +440,7 @@ type HistogramOpts struct {
440440
// constant (or any negative float value).
441441
NativeHistogramZeroThreshold float64
442442

443-
// The remaining fields define a strategy to limit the number of
443+
// The next three fields define a strategy to limit the number of
444444
// populated sparse buckets. If NativeHistogramMaxBucketNumber is left
445445
// at zero, the number of buckets is not limited. (Note that this might
446446
// lead to unbounded memory consumption if the values observed by the
@@ -473,6 +473,22 @@ type HistogramOpts struct {
473473
NativeHistogramMinResetDuration time.Duration
474474
NativeHistogramMaxZeroThreshold float64
475475

476+
// NativeHistogramMaxExemplars limits the number of exemplars
477+
// that are kept in memory for each native histogram. If you leave it at
478+
// zero, a default value of 10 is used. If no exemplars should be kept specifically
479+
// for native histograms, set it to a negative value. (Scrapers can
480+
// still use the exemplars exposed for classic buckets, which are managed
481+
// independently.)
482+
NativeHistogramMaxExemplars int
483+
// NativeHistogramExemplarTTL is only checked once
484+
// NativeHistogramMaxExemplars is exceeded. In that case, the
485+
// oldest exemplar is removed if it is older than NativeHistogramExemplarTTL.
486+
// Otherwise, the older exemplar in the pair of exemplars that are closest
487+
// together (on an exponential scale) is removed.
488+
// If NativeHistogramExemplarTTL is left at its zero value, a default value of
489+
// 5m is used. To always delete the oldest exemplar, set it to a negative value.
490+
NativeHistogramExemplarTTL time.Duration
491+
476492
// now is for testing purposes, by default it's time.Now.
477493
now func() time.Time
478494

@@ -532,6 +548,7 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
532548
if opts.afterFunc == nil {
533549
opts.afterFunc = time.AfterFunc
534550
}
551+
535552
h := &histogram{
536553
desc: desc,
537554
upperBounds: opts.Buckets,
@@ -556,6 +573,7 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
556573
h.nativeHistogramZeroThreshold = DefNativeHistogramZeroThreshold
557574
} // Leave h.nativeHistogramZeroThreshold at 0 otherwise.
558575
h.nativeHistogramSchema = pickSchema(opts.NativeHistogramBucketFactor)
576+
h.nativeExemplars = makeNativeExemplars(opts.NativeHistogramExemplarTTL, opts.NativeHistogramMaxExemplars)
559577
}
560578
for i, upperBound := range h.upperBounds {
561579
if i < len(h.upperBounds)-1 {
@@ -725,7 +743,8 @@ type histogram struct {
725743
// resetScheduled is protected by mtx. It is true if a reset is
726744
// scheduled for a later time (when nativeHistogramMinResetDuration has
727745
// passed).
728-
resetScheduled bool
746+
resetScheduled bool
747+
nativeExemplars nativeExemplars
729748

730749
// now is for testing purposes, by default it's time.Now.
731750
now func() time.Time
@@ -742,6 +761,9 @@ func (h *histogram) Observe(v float64) {
742761
h.observe(v, h.findBucket(v))
743762
}
744763

764+
// ObserveWithExemplar should not be called in a high-frequency setting
765+
// for a native histogram with configured exemplars. For this case,
766+
// the implementation isn't lock-free and might suffer from lock contention.
745767
func (h *histogram) ObserveWithExemplar(v float64, e Labels) {
746768
i := h.findBucket(v)
747769
h.observe(v, i)
@@ -821,6 +843,15 @@ func (h *histogram) Write(out *dto.Metric) error {
821843
Length: proto.Uint32(0),
822844
}}
823845
}
846+
847+
// If exemplars are not configured, the cap will be 0.
848+
// So append is not needed in this case.
849+
if cap(h.nativeExemplars.exemplars) > 0 {
850+
h.nativeExemplars.Lock()
851+
his.Exemplars = append(his.Exemplars, h.nativeExemplars.exemplars...)
852+
h.nativeExemplars.Unlock()
853+
}
854+
824855
}
825856
addAndResetCounts(hotCounts, coldCounts)
826857
return nil
@@ -1091,8 +1122,10 @@ func (h *histogram) resetCounts(counts *histogramCounts) {
10911122
deleteSyncMap(&counts.nativeHistogramBucketsPositive)
10921123
}
10931124

1094-
// updateExemplar replaces the exemplar for the provided bucket. With empty
1095-
// labels, it's a no-op. It panics if any of the labels is invalid.
1125+
// updateExemplar replaces the exemplar for the provided classic bucket.
1126+
// With empty labels, it's a no-op. It panics if any of the labels is invalid.
1127+
// If histogram is native, the exemplar will be cached into nativeExemplars,
1128+
// which has a limit, and will remove one exemplar when limit is reached.
10961129
func (h *histogram) updateExemplar(v float64, bucket int, l Labels) {
10971130
if l == nil {
10981131
return
@@ -1102,6 +1135,10 @@ func (h *histogram) updateExemplar(v float64, bucket int, l Labels) {
11021135
panic(err)
11031136
}
11041137
h.exemplars[bucket].Store(e)
1138+
doSparse := h.nativeHistogramSchema > math.MinInt32 && !math.IsNaN(v)
1139+
if doSparse {
1140+
h.nativeExemplars.addExemplar(e)
1141+
}
11051142
}
11061143

11071144
// HistogramVec is a Collector that bundles a set of Histograms that all share the
@@ -1575,3 +1612,142 @@ func addAndResetCounts(hot, cold *histogramCounts) {
15751612
atomic.AddUint64(&hot.nativeHistogramZeroBucket, atomic.LoadUint64(&cold.nativeHistogramZeroBucket))
15761613
atomic.StoreUint64(&cold.nativeHistogramZeroBucket, 0)
15771614
}
1615+
1616+
type nativeExemplars struct {
1617+
sync.Mutex
1618+
1619+
ttl time.Duration
1620+
exemplars []*dto.Exemplar
1621+
}
1622+
1623+
func makeNativeExemplars(ttl time.Duration, maxCount int) nativeExemplars {
1624+
if ttl == 0 {
1625+
ttl = 5 * time.Minute
1626+
}
1627+
1628+
if maxCount == 0 {
1629+
maxCount = 10
1630+
}
1631+
1632+
if maxCount < 0 {
1633+
maxCount = 0
1634+
}
1635+
1636+
return nativeExemplars{
1637+
ttl: ttl,
1638+
exemplars: make([]*dto.Exemplar, 0, maxCount),
1639+
}
1640+
}
1641+
1642+
func (n *nativeExemplars) addExemplar(e *dto.Exemplar) {
1643+
if cap(n.exemplars) == 0 {
1644+
return
1645+
}
1646+
1647+
n.Lock()
1648+
defer n.Unlock()
1649+
1650+
// The index where to insert the new exemplar.
1651+
var nIdx int = -1
1652+
1653+
// When the number of exemplars has not yet exceeded or
1654+
// is equal to cap(n.exemplars), then
1655+
// insert the new exemplar directly.
1656+
if len(n.exemplars) < cap(n.exemplars) {
1657+
for nIdx = 0; nIdx < len(n.exemplars); nIdx++ {
1658+
if *e.Value < *n.exemplars[nIdx].Value {
1659+
break
1660+
}
1661+
}
1662+
n.exemplars = append(n.exemplars[:nIdx], append([]*dto.Exemplar{e}, n.exemplars[nIdx:]...)...)
1663+
return
1664+
}
1665+
1666+
// When the number of exemplars exceeds the limit, remove one exemplar.
1667+
var (
1668+
rIdx int // The index where to remove the old exemplar.
1669+
1670+
ot = time.Now() // Oldest timestamp seen.
1671+
otIdx = -1 // Index of the exemplar with the oldest timestamp.
1672+
1673+
md = -1.0 // Logarithm of the delta of the closest pair of exemplars.
1674+
mdIdx = -1 // Index of the older exemplar within the closest pair.
1675+
cLog float64 // Logarithm of the current exemplar.
1676+
pLog float64 // Logarithm of the previous exemplar.
1677+
)
1678+
1679+
for i, exemplar := range n.exemplars {
1680+
// Find the exemplar with the oldest timestamp.
1681+
if otIdx == -1 || exemplar.Timestamp.AsTime().Before(ot) {
1682+
ot = exemplar.Timestamp.AsTime()
1683+
otIdx = i
1684+
}
1685+
1686+
// Find the index at which to insert new the exemplar.
1687+
if *e.Value <= *exemplar.Value && nIdx == -1 {
1688+
nIdx = i
1689+
}
1690+
1691+
// Find the two closest exemplars and pick the one the with older timestamp.
1692+
pLog = cLog
1693+
cLog = math.Log(exemplar.GetValue())
1694+
if i == 0 {
1695+
continue
1696+
}
1697+
diff := math.Abs(cLog - pLog)
1698+
if md == -1 || diff < md {
1699+
md = diff
1700+
if n.exemplars[i].Timestamp.AsTime().Before(n.exemplars[i-1].Timestamp.AsTime()) {
1701+
mdIdx = i
1702+
} else {
1703+
mdIdx = i - 1
1704+
}
1705+
}
1706+
1707+
}
1708+
1709+
// If all existing exemplar are smaller than new exemplar,
1710+
// then the exemplar should be inserted at the end.
1711+
if nIdx == -1 {
1712+
nIdx = len(n.exemplars)
1713+
}
1714+
1715+
if otIdx != -1 && e.Timestamp.AsTime().Sub(ot) > n.ttl {
1716+
rIdx = otIdx
1717+
} else {
1718+
// In the previous for loop, when calculating the closest pair of exemplars,
1719+
// we did not take into account the newly inserted exemplar.
1720+
// So we need to calculate with the newly inserted exemplar again.
1721+
elog := math.Log(e.GetValue())
1722+
if nIdx > 0 {
1723+
diff := math.Abs(elog - math.Log(n.exemplars[nIdx-1].GetValue()))
1724+
if diff < md {
1725+
md = diff
1726+
mdIdx = nIdx
1727+
if n.exemplars[nIdx-1].Timestamp.AsTime().Before(e.Timestamp.AsTime()) {
1728+
mdIdx = nIdx - 1
1729+
}
1730+
}
1731+
}
1732+
if nIdx < len(n.exemplars) {
1733+
diff := math.Abs(math.Log(n.exemplars[nIdx].GetValue()) - elog)
1734+
if diff < md {
1735+
mdIdx = nIdx
1736+
if n.exemplars[nIdx].Timestamp.AsTime().Before(e.Timestamp.AsTime()) {
1737+
mdIdx = nIdx
1738+
}
1739+
}
1740+
}
1741+
rIdx = mdIdx
1742+
}
1743+
1744+
// Adjust the slice according to rIdx and nIdx.
1745+
switch {
1746+
case rIdx == nIdx:
1747+
n.exemplars[nIdx] = e
1748+
case rIdx < nIdx:
1749+
n.exemplars = append(n.exemplars[:rIdx], append(n.exemplars[rIdx+1:nIdx], append([]*dto.Exemplar{e}, n.exemplars[nIdx:]...)...)...)
1750+
case rIdx > nIdx:
1751+
n.exemplars = append(n.exemplars[:nIdx], append([]*dto.Exemplar{e}, append(n.exemplars[nIdx:rIdx], n.exemplars[rIdx+1:]...)...)...)
1752+
}
1753+
}

0 commit comments

Comments
 (0)