Optimize the new inhibitor implementation for ~2.5x performance improvement (#4668)

* remove metric selection from inhibitor hot-path

Signed-off-by: Ethan Hunter <[email protected]>

* add optimizations to improve inhibitor performance

Signed-off-by: Ethan Hunter <[email protected]>

* fix duration calculation

Co-authored-by: Siavash Safi <[email protected]>
Signed-off-by: Ethan Hunter <[email protected]>

---------

Signed-off-by: Ethan Hunter <[email protected]>
Signed-off-by: Ethan Hunter <[email protected]>
Co-authored-by: Siavash Safi <[email protected]>

Author: Spaceman1701

inhibit/inhibit.go

@@ -163,22 +163,25 @@ func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
 		ruleStart := time.Now()
 		if !r.TargetMatchers.Matches(lset) {
 			// If target side of rule doesn't match, we don't need to look any further.
-			r.metrics.matchesDuration.With(prometheus.Labels{"rule": r.Name, "matched": "false"}).Observe(time.Since(ruleStart).Seconds())
+			r.metrics.matchesDurationNotMatched.Observe(time.Since(ruleStart).Seconds())
 			continue
 		}
-		r.metrics.matchesDuration.With(prometheus.Labels{"rule": r.Name, "matched": "true"}).Observe(time.Since(ruleStart).Seconds())
+		r.metrics.matchesDurationMatched.Observe(time.Since(ruleStart).Seconds())
 		// If we are here, the target side matches. If the source side matches, too, we
 		// need to exclude inhibiting alerts for which the same is true.
-		if inhibitedByFP, eq := r.hasEqual(lset, r.SourceMatchers.Matches(lset)); eq {
+		if inhibitedByFP, eq := r.hasEqual(lset, r.SourceMatchers.Matches(lset), ruleStart); eq {
 			ih.marker.SetInhibited(fp, inhibitedByFP.String())
-			ih.metrics.mutesDuration.With(prometheus.Labels{"muted": "true"}).Observe(time.Since(start).Seconds())
-			r.metrics.mutesDuration.With(prometheus.Labels{"rule": r.Name, "muted": "true"}).Observe(time.Since(ruleStart).Seconds())
+			now := time.Now()
+			sinceStart := now.Sub(start)
+			sinceRuleStart := now.Sub(ruleStart)
+			ih.metrics.mutesDurationMuted.Observe(sinceStart.Seconds())
+			r.metrics.mutesDurationMuted.Observe(sinceRuleStart.Seconds())
 			return true
 		}
-		r.metrics.mutesDuration.With(prometheus.Labels{"rule": r.Name, "muted": "false"}).Observe(time.Since(ruleStart).Seconds())
+		r.metrics.mutesDurationNotMuted.Observe(time.Since(ruleStart).Seconds())
 	}
 	ih.marker.SetInhibited(fp)
-	ih.metrics.mutesDuration.With(prometheus.Labels{"muted": "false"}).Observe(time.Since(start).Seconds())
+	ih.metrics.mutesDurationNotMuted.Observe(time.Since(start).Seconds())
 
 	return false
 }
@@ -360,8 +363,7 @@ func (r *InhibitRule) gcCallback(alerts []types.Alert) {
 // labels for the given label set. If so, the fingerprint of one of those alerts
 // is returned. If excludeTwoSidedMatch is true, alerts that match both the
 // source and the target side of the rule are disregarded.
-func (r *InhibitRule) hasEqual(lset model.LabelSet, excludeTwoSidedMatch bool) (model.Fingerprint, bool) {
-	now := time.Now()
+func (r *InhibitRule) hasEqual(lset model.LabelSet, excludeTwoSidedMatch bool, now time.Time) (model.Fingerprint, bool) {
 	equal, found := r.findEqualSourceAlert(lset, now)
 	if found {
 		if excludeTwoSidedMatch && r.TargetMatchers.Matches(equal.Labels) {

inhibit/inhibit_test.go

@@ -138,7 +138,7 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 				r.updateIndex(v)
 			}
 
-			if _, have := r.hasEqual(c.input, false); have != c.result {
+			if _, have := r.hasEqual(c.input, false, time.Now()); have != c.result {
 				t.Errorf("Unexpected result %t, expected %t", have, c.result)
 			}
 		})

inhibit/metric.go

@@ -23,6 +23,8 @@ type InhibitorMetrics struct {
 	sourceAlertsCacheItems prometheus.Gauge
 	sourceAlertsIndexItems prometheus.Gauge
 	mutesDuration          *prometheus.SummaryVec
+	mutesDurationMuted     prometheus.Observer
+	mutesDurationNotMuted  prometheus.Observer
 
 	// Rule metrics
 	ruleSourceAlertsCacheItems *prometheus.GaugeVec
@@ -53,6 +55,7 @@ func NewInhibitorMetrics(reg prometheus.Registerer) *InhibitorMetrics {
 			},
 			[]string{"muted"},
 		),
+
 		ruleSourceAlertsCacheItems: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Name: "alertmanager_inhibit_rule_source_alerts_cache_items",
@@ -82,6 +85,10 @@ func NewInhibitorMetrics(reg prometheus.Registerer) *InhibitorMetrics {
 			[]string{"rule", "muted"},
 		),
 	}
+
+	metrics.mutesDurationMuted = metrics.mutesDuration.With(prometheus.Labels{"muted": "true"})
+	metrics.mutesDurationNotMuted = metrics.mutesDuration.With(prometheus.Labels{"muted": "false"})
+
 	if reg != nil {
 		reg.MustRegister(
 			metrics.sourceAlertsCacheItems,
@@ -101,20 +108,26 @@ func NewInhibitorMetrics(reg prometheus.Registerer) *InhibitorMetrics {
 }
 
 type RuleMetrics struct {
-	ruleName               string
-	matchesDuration        *prometheus.SummaryVec
-	mutesDuration          *prometheus.SummaryVec
+	ruleName                  string
+	matchesDurationMatched    prometheus.Observer
+	matchesDurationNotMatched prometheus.Observer
+
+	mutesDurationMuted    prometheus.Observer
+	mutesDurationNotMuted prometheus.Observer
+
 	sourceAlertsCacheItems *prometheus.GaugeVec
 	sourceAlertsIndexItems *prometheus.GaugeVec
 }
 
 func NewRuleMetrics(name string, metrics *InhibitorMetrics) *RuleMetrics {
 	rm := &RuleMetrics{
-		ruleName:               name,
-		matchesDuration:        metrics.ruleMatchesDuration,
-		mutesDuration:          metrics.ruleMutesDuration,
-		sourceAlertsCacheItems: metrics.ruleSourceAlertsCacheItems,
-		sourceAlertsIndexItems: metrics.ruleSourceAlertsIndexItems,
+		ruleName:                  name,
+		matchesDurationMatched:    metrics.ruleMatchesDuration.With(prometheus.Labels{"rule": name, "matched": "true"}),
+		matchesDurationNotMatched: metrics.ruleMatchesDuration.With(prometheus.Labels{"rule": name, "matched": "false"}),
+		mutesDurationMuted:        metrics.ruleMutesDuration.With(prometheus.Labels{"rule": name, "muted": "true"}),
+		mutesDurationNotMuted:     metrics.ruleMutesDuration.With(prometheus.Labels{"rule": name, "muted": "false"}),
+		sourceAlertsCacheItems:    metrics.ruleSourceAlertsCacheItems,
+		sourceAlertsIndexItems:    metrics.ruleSourceAlertsIndexItems,
 	}
 
 	rm.sourceAlertsCacheItems.With(prometheus.Labels{"rule": rm.ruleName}).Set(0)