@@ -440,7 +440,7 @@ type HistogramOpts struct {
440
440
// constant (or any negative float value).
441
441
NativeHistogramZeroThreshold float64
442
442
443
- // The remaining fields define a strategy to limit the number of
443
+ // The next three fields define a strategy to limit the number of
444
444
// populated sparse buckets. If NativeHistogramMaxBucketNumber is left
445
445
// at zero, the number of buckets is not limited. (Note that this might
446
446
// lead to unbounded memory consumption if the values observed by the
@@ -473,6 +473,22 @@ type HistogramOpts struct {
473
473
NativeHistogramMinResetDuration time.Duration
474
474
NativeHistogramMaxZeroThreshold float64
475
475
476
+ // NativeHistogramMaxExemplars limits the number of exemplars
477
+ // that are kept in memory for each native histogram. If you leave it at
478
+ // zero, a default value of 10 is used. If no exemplars should be kept specifically
479
+ // for native histograms, set it to a negative value. (Scrapers can
480
+ // still use the exemplars exposed for classic buckets, which are managed
481
+ // independently.)
482
+ NativeHistogramMaxExemplars int
483
+ // NativeHistogramExemplarTTL is only checked once
484
+ // NativeHistogramMaxExemplars is exceeded. In that case, the
485
+ // oldest exemplar is removed if it is older than NativeHistogramExemplarTTL.
486
+ // Otherwise, the older exemplar in the pair of exemplars that are closest
487
+ // together (on an exponential scale) is removed.
488
+ // If NativeHistogramExemplarTTL is left at its zero value, a default value of
489
+ // 5m is used. To always delete the oldest exemplar, set it to a negative value.
490
+ NativeHistogramExemplarTTL time.Duration
491
+
476
492
// now is for testing purposes, by default it's time.Now.
477
493
now func () time.Time
478
494
@@ -532,6 +548,7 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
532
548
if opts .afterFunc == nil {
533
549
opts .afterFunc = time .AfterFunc
534
550
}
551
+
535
552
h := & histogram {
536
553
desc : desc ,
537
554
upperBounds : opts .Buckets ,
@@ -556,6 +573,7 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr
556
573
h .nativeHistogramZeroThreshold = DefNativeHistogramZeroThreshold
557
574
} // Leave h.nativeHistogramZeroThreshold at 0 otherwise.
558
575
h .nativeHistogramSchema = pickSchema (opts .NativeHistogramBucketFactor )
576
+ h .nativeExemplars = makeNativeExemplars (opts .NativeHistogramExemplarTTL , opts .NativeHistogramMaxExemplars )
559
577
}
560
578
for i , upperBound := range h .upperBounds {
561
579
if i < len (h .upperBounds )- 1 {
@@ -725,7 +743,8 @@ type histogram struct {
725
743
// resetScheduled is protected by mtx. It is true if a reset is
726
744
// scheduled for a later time (when nativeHistogramMinResetDuration has
727
745
// passed).
728
- resetScheduled bool
746
+ resetScheduled bool
747
+ nativeExemplars nativeExemplars
729
748
730
749
// now is for testing purposes, by default it's time.Now.
731
750
now func () time.Time
@@ -742,6 +761,9 @@ func (h *histogram) Observe(v float64) {
742
761
h .observe (v , h .findBucket (v ))
743
762
}
744
763
764
+ // ObserveWithExemplar should not be called in a high-frequency setting
765
+ // for a native histogram with configured exemplars. For this case,
766
+ // the implementation isn't lock-free and might suffer from lock contention.
745
767
func (h * histogram ) ObserveWithExemplar (v float64 , e Labels ) {
746
768
i := h .findBucket (v )
747
769
h .observe (v , i )
@@ -821,6 +843,15 @@ func (h *histogram) Write(out *dto.Metric) error {
821
843
Length : proto .Uint32 (0 ),
822
844
}}
823
845
}
846
+
847
+ // If exemplars are not configured, the cap will be 0.
848
+ // So append is not needed in this case.
849
+ if cap (h .nativeExemplars .exemplars ) > 0 {
850
+ h .nativeExemplars .Lock ()
851
+ his .Exemplars = append (his .Exemplars , h .nativeExemplars .exemplars ... )
852
+ h .nativeExemplars .Unlock ()
853
+ }
854
+
824
855
}
825
856
addAndResetCounts (hotCounts , coldCounts )
826
857
return nil
@@ -1091,8 +1122,10 @@ func (h *histogram) resetCounts(counts *histogramCounts) {
1091
1122
deleteSyncMap (& counts .nativeHistogramBucketsPositive )
1092
1123
}
1093
1124
1094
- // updateExemplar replaces the exemplar for the provided bucket. With empty
1095
- // labels, it's a no-op. It panics if any of the labels is invalid.
1125
+ // updateExemplar replaces the exemplar for the provided classic bucket.
1126
+ // With empty labels, it's a no-op. It panics if any of the labels is invalid.
1127
+ // If histogram is native, the exemplar will be cached into nativeExemplars,
1128
+ // which has a limit, and will remove one exemplar when limit is reached.
1096
1129
func (h * histogram ) updateExemplar (v float64 , bucket int , l Labels ) {
1097
1130
if l == nil {
1098
1131
return
@@ -1102,6 +1135,10 @@ func (h *histogram) updateExemplar(v float64, bucket int, l Labels) {
1102
1135
panic (err )
1103
1136
}
1104
1137
h .exemplars [bucket ].Store (e )
1138
+ doSparse := h .nativeHistogramSchema > math .MinInt32 && ! math .IsNaN (v )
1139
+ if doSparse {
1140
+ h .nativeExemplars .addExemplar (e )
1141
+ }
1105
1142
}
1106
1143
1107
1144
// HistogramVec is a Collector that bundles a set of Histograms that all share the
@@ -1575,3 +1612,142 @@ func addAndResetCounts(hot, cold *histogramCounts) {
1575
1612
atomic .AddUint64 (& hot .nativeHistogramZeroBucket , atomic .LoadUint64 (& cold .nativeHistogramZeroBucket ))
1576
1613
atomic .StoreUint64 (& cold .nativeHistogramZeroBucket , 0 )
1577
1614
}
1615
+
1616
+ type nativeExemplars struct {
1617
+ sync.Mutex
1618
+
1619
+ ttl time.Duration
1620
+ exemplars []* dto.Exemplar
1621
+ }
1622
+
1623
+ func makeNativeExemplars (ttl time.Duration , maxCount int ) nativeExemplars {
1624
+ if ttl == 0 {
1625
+ ttl = 5 * time .Minute
1626
+ }
1627
+
1628
+ if maxCount == 0 {
1629
+ maxCount = 10
1630
+ }
1631
+
1632
+ if maxCount < 0 {
1633
+ maxCount = 0
1634
+ }
1635
+
1636
+ return nativeExemplars {
1637
+ ttl : ttl ,
1638
+ exemplars : make ([]* dto.Exemplar , 0 , maxCount ),
1639
+ }
1640
+ }
1641
+
1642
+ func (n * nativeExemplars ) addExemplar (e * dto.Exemplar ) {
1643
+ if cap (n .exemplars ) == 0 {
1644
+ return
1645
+ }
1646
+
1647
+ n .Lock ()
1648
+ defer n .Unlock ()
1649
+
1650
+ // The index where to insert the new exemplar.
1651
+ var nIdx int = - 1
1652
+
1653
+ // When the number of exemplars has not yet exceeded or
1654
+ // is equal to cap(n.exemplars), then
1655
+ // insert the new exemplar directly.
1656
+ if len (n .exemplars ) < cap (n .exemplars ) {
1657
+ for nIdx = 0 ; nIdx < len (n .exemplars ); nIdx ++ {
1658
+ if * e .Value < * n .exemplars [nIdx ].Value {
1659
+ break
1660
+ }
1661
+ }
1662
+ n .exemplars = append (n .exemplars [:nIdx ], append ([]* dto.Exemplar {e }, n .exemplars [nIdx :]... )... )
1663
+ return
1664
+ }
1665
+
1666
+ // When the number of exemplars exceeds the limit, remove one exemplar.
1667
+ var (
1668
+ rIdx int // The index where to remove the old exemplar.
1669
+
1670
+ ot = time .Now () // Oldest timestamp seen.
1671
+ otIdx = - 1 // Index of the exemplar with the oldest timestamp.
1672
+
1673
+ md = - 1.0 // Logarithm of the delta of the closest pair of exemplars.
1674
+ mdIdx = - 1 // Index of the older exemplar within the closest pair.
1675
+ cLog float64 // Logarithm of the current exemplar.
1676
+ pLog float64 // Logarithm of the previous exemplar.
1677
+ )
1678
+
1679
+ for i , exemplar := range n .exemplars {
1680
+ // Find the exemplar with the oldest timestamp.
1681
+ if otIdx == - 1 || exemplar .Timestamp .AsTime ().Before (ot ) {
1682
+ ot = exemplar .Timestamp .AsTime ()
1683
+ otIdx = i
1684
+ }
1685
+
1686
+ // Find the index at which to insert new the exemplar.
1687
+ if * e .Value <= * exemplar .Value && nIdx == - 1 {
1688
+ nIdx = i
1689
+ }
1690
+
1691
+ // Find the two closest exemplars and pick the one the with older timestamp.
1692
+ pLog = cLog
1693
+ cLog = math .Log (exemplar .GetValue ())
1694
+ if i == 0 {
1695
+ continue
1696
+ }
1697
+ diff := math .Abs (cLog - pLog )
1698
+ if md == - 1 || diff < md {
1699
+ md = diff
1700
+ if n .exemplars [i ].Timestamp .AsTime ().Before (n .exemplars [i - 1 ].Timestamp .AsTime ()) {
1701
+ mdIdx = i
1702
+ } else {
1703
+ mdIdx = i - 1
1704
+ }
1705
+ }
1706
+
1707
+ }
1708
+
1709
+ // If all existing exemplar are smaller than new exemplar,
1710
+ // then the exemplar should be inserted at the end.
1711
+ if nIdx == - 1 {
1712
+ nIdx = len (n .exemplars )
1713
+ }
1714
+
1715
+ if otIdx != - 1 && e .Timestamp .AsTime ().Sub (ot ) > n .ttl {
1716
+ rIdx = otIdx
1717
+ } else {
1718
+ // In the previous for loop, when calculating the closest pair of exemplars,
1719
+ // we did not take into account the newly inserted exemplar.
1720
+ // So we need to calculate with the newly inserted exemplar again.
1721
+ elog := math .Log (e .GetValue ())
1722
+ if nIdx > 0 {
1723
+ diff := math .Abs (elog - math .Log (n .exemplars [nIdx - 1 ].GetValue ()))
1724
+ if diff < md {
1725
+ md = diff
1726
+ mdIdx = nIdx
1727
+ if n .exemplars [nIdx - 1 ].Timestamp .AsTime ().Before (e .Timestamp .AsTime ()) {
1728
+ mdIdx = nIdx - 1
1729
+ }
1730
+ }
1731
+ }
1732
+ if nIdx < len (n .exemplars ) {
1733
+ diff := math .Abs (math .Log (n .exemplars [nIdx ].GetValue ()) - elog )
1734
+ if diff < md {
1735
+ mdIdx = nIdx
1736
+ if n .exemplars [nIdx ].Timestamp .AsTime ().Before (e .Timestamp .AsTime ()) {
1737
+ mdIdx = nIdx
1738
+ }
1739
+ }
1740
+ }
1741
+ rIdx = mdIdx
1742
+ }
1743
+
1744
+ // Adjust the slice according to rIdx and nIdx.
1745
+ switch {
1746
+ case rIdx == nIdx :
1747
+ n .exemplars [nIdx ] = e
1748
+ case rIdx < nIdx :
1749
+ n .exemplars = append (n .exemplars [:rIdx ], append (n .exemplars [rIdx + 1 :nIdx ], append ([]* dto.Exemplar {e }, n .exemplars [nIdx :]... )... )... )
1750
+ case rIdx > nIdx :
1751
+ n .exemplars = append (n .exemplars [:nIdx ], append ([]* dto.Exemplar {e }, append (n .exemplars [nIdx :rIdx ], n .exemplars [rIdx + 1 :]... )... )... )
1752
+ }
1753
+ }
0 commit comments