Add update_mode label to VPA updater metrics

Author: laoj2

vertical-pod-autoscaler/go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

vertical-pod-autoscaler/pkg/updater/logic/updater.go

@@ -230,7 +230,8 @@ func (u *updater) RunOnce(ctx context.Context) {
 	// to contain only Pods controlled by a VPA in auto, recreate, or inPlaceOrRecreate mode
 	for vpa, livePods := range controlledPods {
 		vpaSize := len(livePods)
-		controlledPodsCounter.Add(vpaSize, vpaSize)
+		updateMode := vpa_api_util.GetUpdateMode(vpa)
+		controlledPodsCounter.Add(vpaSize, updateMode, vpaSize)
 		creatorToSingleGroupStatsMap, podToReplicaCreatorMap, err := u.restrictionFactory.GetCreatorMaps(livePods, vpa)
 		if err != nil {
 			klog.ErrorS(err, "Failed to get creator maps")
@@ -242,7 +243,6 @@ func (u *updater) RunOnce(ctx context.Context) {
 
 		podsForInPlace := make([]*apiv1.Pod, 0)
 		podsForEviction := make([]*apiv1.Pod, 0)
-		updateMode := vpa_api_util.GetUpdateMode(vpa)
 
 		if updateMode == vpa_types.UpdateModeInPlaceOrRecreate && features.Enabled(features.InPlaceOrRecreate) {
 			podsForInPlace = u.getPodsUpdateOrder(filterNonInPlaceUpdatablePods(livePods, inPlaceLimiter), vpa)
@@ -253,7 +253,7 @@ func (u *updater) RunOnce(ctx context.Context) {
 				klog.InfoS("Warning: feature gate is not enabled for this updateMode", "featuregate", features.InPlaceOrRecreate, "updateMode", vpa_types.UpdateModeInPlaceOrRecreate)
 			}
 			podsForEviction = u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), vpa)
-			evictablePodsCounter.Add(vpaSize, len(podsForEviction))
+			evictablePodsCounter.Add(vpaSize, updateMode, len(podsForEviction))
 		}
 
 		withInPlaceUpdatable := false
@@ -304,7 +304,7 @@ func (u *updater) RunOnce(ctx context.Context) {
 				klog.V(0).InfoS("Eviction failed", "error", evictErr, "pod", klog.KObj(pod))
 			} else {
 				withEvicted = true
-				metrics_updater.AddEvictedPod(vpaSize)
+				metrics_updater.AddEvictedPod(vpaSize, updateMode)
 			}
 		}
 
@@ -315,10 +315,10 @@ func (u *updater) RunOnce(ctx context.Context) {
 			vpasWithInPlaceUpdatedPodsCounter.Add(vpaSize, 1)
 		}
 		if withEvictable {
-			vpasWithEvictablePodsCounter.Add(vpaSize, 1)
+			vpasWithEvictablePodsCounter.Add(vpaSize, updateMode, 1)
 		}
 		if withEvicted {
-			vpasWithEvictedPodsCounter.Add(vpaSize, 1)
+			vpasWithEvictedPodsCounter.Add(vpaSize, updateMode, 1)
 		}
 	}
 	timer.ObserveStep("EvictPods")

vertical-pod-autoscaler/pkg/utils/metrics/updater/updater.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics"
 )
 
@@ -35,45 +36,52 @@ type SizeBasedGauge struct {
 	gauge  *prometheus.GaugeVec
 }
 
+// UpdateModeAndSizeBasedGauge is a wrapper for incrementally recording values
+// indexed by log2(VPA size) and update mode
+type UpdateModeAndSizeBasedGauge struct {
+	values [metrics.MaxVpaSizeLog]map[vpa_types.UpdateMode]int
+	gauge  *prometheus.GaugeVec
+}
+
 var (
 	controlledCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: metricsNamespace,
 			Name:      "controlled_pods_total",
 			Help:      "Number of Pods controlled by VPA updater.",
-		}, []string{"vpa_size_log2"},
+		}, []string{"vpa_size_log2", "update_mode"},
 	)
 
 	evictableCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: metricsNamespace,
 			Name:      "evictable_pods_total",
 			Help:      "Number of Pods matching evicition criteria.",
-		}, []string{"vpa_size_log2"},
+		}, []string{"vpa_size_log2", "update_mode"},
 	)
 
 	evictedCount = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: metricsNamespace,
 			Name:      "evicted_pods_total",
 			Help:      "Number of Pods evicted by Updater to apply a new recommendation.",
-		}, []string{"vpa_size_log2"},
+		}, []string{"vpa_size_log2", "update_mode"},
 	)
 
 	vpasWithEvictablePodsCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: metricsNamespace,
 			Name:      "vpas_with_evictable_pods_total",
 			Help:      "Number of VPA objects with at least one Pod matching evicition criteria.",
-		}, []string{"vpa_size_log2"},
+		}, []string{"vpa_size_log2", "update_mode"},
 	)
 
 	vpasWithEvictedPodsCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: metricsNamespace,
 			Name:      "vpas_with_evicted_pods_total",
 			Help:      "Number of VPA objects with at least one evicted Pod.",
-		}, []string{"vpa_size_log2"},
+		}, []string{"vpa_size_log2", "update_mode"},
 	)
 
 	inPlaceUpdatableCount = prometheus.NewGaugeVec(
@@ -138,30 +146,41 @@ func newSizeBasedGauge(gauge *prometheus.GaugeVec) *SizeBasedGauge {
 	}
 }
 
+// newModeAndSizeBasedGauge provides a wrapper for counting items in a loop
+func newModeAndSizeBasedGauge(gauge *prometheus.GaugeVec) *UpdateModeAndSizeBasedGauge {
+	g := &UpdateModeAndSizeBasedGauge{
+		gauge: gauge,
+	}
+	for i := range g.values {
+		g.values[i] = make(map[vpa_types.UpdateMode]int)
+	}
+	return g
+}
+
 // NewControlledPodsCounter returns a wrapper for counting Pods controlled by Updater
-func NewControlledPodsCounter() *SizeBasedGauge {
-	return newSizeBasedGauge(controlledCount)
+func NewControlledPodsCounter() *UpdateModeAndSizeBasedGauge {
+	return newModeAndSizeBasedGauge(controlledCount)
 }
 
 // NewEvictablePodsCounter returns a wrapper for counting Pods which are matching eviction criteria
-func NewEvictablePodsCounter() *SizeBasedGauge {
-	return newSizeBasedGauge(evictableCount)
+func NewEvictablePodsCounter() *UpdateModeAndSizeBasedGauge {
+	return newModeAndSizeBasedGauge(evictableCount)
 }
 
 // NewVpasWithEvictablePodsCounter returns a wrapper for counting VPA objects with Pods matching eviction criteria
-func NewVpasWithEvictablePodsCounter() *SizeBasedGauge {
-	return newSizeBasedGauge(vpasWithEvictablePodsCount)
+func NewVpasWithEvictablePodsCounter() *UpdateModeAndSizeBasedGauge {
+	return newModeAndSizeBasedGauge(vpasWithEvictablePodsCount)
 }
 
 // NewVpasWithEvictedPodsCounter returns a wrapper for counting VPA objects with evicted Pods
-func NewVpasWithEvictedPodsCounter() *SizeBasedGauge {
-	return newSizeBasedGauge(vpasWithEvictedPodsCount)
+func NewVpasWithEvictedPodsCounter() *UpdateModeAndSizeBasedGauge {
+	return newModeAndSizeBasedGauge(vpasWithEvictedPodsCount)
 }
 
 // AddEvictedPod increases the counter of pods evicted by Updater, by given VPA size
-func AddEvictedPod(vpaSize int) {
+func AddEvictedPod(vpaSize int, mode vpa_types.UpdateMode) {
 	log2 := metrics.GetVpaSizeLog2(vpaSize)
-	evictedCount.WithLabelValues(strconv.Itoa(log2)).Inc()
+	evictedCount.WithLabelValues(strconv.Itoa(log2), string(mode)).Inc()
 }
 
 // NewInPlaceUpdatablePodsCounter returns a wrapper for counting Pods which are matching in-place update criteria
@@ -203,3 +222,19 @@ func (g *SizeBasedGauge) Observe() {
 		g.gauge.WithLabelValues(strconv.Itoa(log2)).Set(float64(value))
 	}
 }
+
+// Add increases the counter for the given VPA size and VPA update mode.
+func (g *UpdateModeAndSizeBasedGauge) Add(vpaSize int, vpaUpdateMode vpa_types.UpdateMode, value int) {
+	log2 := metrics.GetVpaSizeLog2(vpaSize)
+	g.values[log2][vpaUpdateMode] += value
+}
+
+// Observe stores the recorded values into metrics object associated with the
+// wrapper
+func (g *UpdateModeAndSizeBasedGauge) Observe() {
+	for log2, valueMap := range g.values {
+		for vpaMode, value := range valueMap {
+			g.gauge.WithLabelValues(strconv.Itoa(log2), string(vpaMode)).Set(float64(value))
+		}
+	}
+}