Merge pull request kubernetes#3129 from kgolab/vpa-updater-metrics

Add bunch of metrics to monitor VPA Updater impact in a cluster

Author: k8s-ci-robot

vertical-pod-autoscaler/pkg/updater/README.md

@@ -29,4 +29,3 @@ before pod with 20% memory increase and no change in cpu).
 
 # Missing parts
 * Recommendation API for fetching data from Vertical Pod Autoscaler Recommender.
-* Monitoring.

vertical-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction.go

@@ -24,7 +24,6 @@ import (
 	apiv1 "k8s.io/api/core/v1"
 	policyv1 "k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater"
 	appsinformer "k8s.io/client-go/informers/apps/v1"
 	coreinformer "k8s.io/client-go/informers/core/v1"
 	kube_client "k8s.io/client-go/kubernetes"
@@ -141,7 +140,6 @@ func (e *podsEvictionRestrictionImpl) Evict(podToEvict *apiv1.Pod, eventRecorder
 	}
 	eventRecorder.Event(podToEvict, apiv1.EventTypeNormal, "EvictedByVPA",
 		"Pod was evicted by VPA Updater to apply resource recommendation.")
-	metrics_updater.AddEvictedPod()
 
 	if podToEvict.Status.Phase != apiv1.PodPending {
 		singleGroupStats, present := e.creatorToSingleGroupStatsMap[cr]

vertical-pod-autoscaler/pkg/updater/logic/updater.go

@@ -177,11 +177,31 @@ func (u *updater) RunOnce(ctx context.Context) {
 	}
 	timer.ObserveStep("AdmissionInit")
 
+	// wrappers for metrics which are computed every loop run
+	controlledPodsCounter := metrics_updater.NewControlledPodsCounter()
+	evictablePodsCounter := metrics_updater.NewEvictablePodsCounter()
+	vpasWithEvictablePodsCounter := metrics_updater.NewVpasWithEvictablePodsCounter()
+	vpasWithEvictedPodsCounter := metrics_updater.NewVpasWithEvictedPodsCounter()
+
+	// using defer to protect against 'return' after evictionRateLimiter.Wait
+	defer controlledPodsCounter.Observe()
+	defer evictablePodsCounter.Observe()
+	defer vpasWithEvictablePodsCounter.Observe()
+	defer vpasWithEvictedPodsCounter.Observe()
+
+	// NOTE: this loop assumes that controlledPods are filtered
+	// to contain only Pods controlled by a VPA in auto or recreate mode
 	for vpa, livePods := range controlledPods {
+		vpaSize := len(livePods)
+		controlledPodsCounter.Add(vpaSize, vpaSize)
 		evictionLimiter := u.evictionFactory.NewPodsEvictionRestriction(livePods)
 		podsForUpdate := u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), vpa)
+		evictablePodsCounter.Add(vpaSize, len(podsForUpdate))
 
+		withEvictable := false
+		withEvicted := false
 		for _, pod := range podsForUpdate {
+			withEvictable = true
 			if !evictionLimiter.CanEvict(pod) {
 				continue
 			}
@@ -194,8 +214,18 @@ func (u *updater) RunOnce(ctx context.Context) {
 			evictErr := evictionLimiter.Evict(pod, u.eventRecorder)
 			if evictErr != nil {
 				klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr)
+			} else {
+				withEvicted = true
+				metrics_updater.AddEvictedPod(vpaSize)
 			}
 		}
+
+		if withEvictable {
+			vpasWithEvictablePodsCounter.Add(vpaSize, 1)
+		}
+		if withEvicted {
+			vpasWithEvictedPodsCounter.Add(vpaSize, 1)
+		}
 	}
 	timer.ObserveStep("EvictPods")
 }

vertical-pod-autoscaler/pkg/utils/metrics/metrics.go

@@ -18,6 +18,7 @@ limitations under the License.
 package metrics
 
 import (
+	"math"
 	"net/http"
 	"time"
 
@@ -39,6 +40,10 @@ type ExecutionTimer struct {
 const (
 	// TopMetricsNamespace is a prefix for all VPA-related metrics namespaces
 	TopMetricsNamespace = "vpa_"
+
+	// The metrics will distinguish VPA sizes up to 2^MaxVpaSizeLog (~1M)
+	// Anything above that size will be reported in the top bucket.
+	MaxVpaSizeLog = 20
 )
 
 // Initialize sets up Prometheus to expose metrics & (optionally) health-check on the given address
@@ -87,3 +92,17 @@ func CreateExecutionTimeMetric(namespace string, help string) *prometheus.Histog
 		}, []string{"step"},
 	)
 }
+
+// GetVpaSizeLog2 returns a bucket number for a metric labelled with number of Pods under a given VPA.
+// It is basically log2(vpaSize), capped to MaxVpaSizeLog
+func GetVpaSizeLog2(vpaSize int) int {
+	if vpaSize == 0 {
+		return 0
+	}
+
+	ret := int(math.Log2(float64(vpaSize)))
+	if ret > MaxVpaSizeLog {
+		return MaxVpaSizeLog
+	}
+	return ret
+}

vertical-pod-autoscaler/pkg/utils/metrics/updater/updater.go

@@ -18,6 +18,8 @@ limitations under the License.
 package updater
 
 import (
+	"strconv"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics"
 )
@@ -26,13 +28,51 @@ const (
 	metricsNamespace = metrics.TopMetricsNamespace + "updater"
 )
 
+// SizeBasedGauge is a wrapper for incrementally recording values indexed by log2(VPA size)
+type SizeBasedGauge struct {
+	values [metrics.MaxVpaSizeLog]int
+	gauge  *prometheus.GaugeVec
+}
+
 var (
-	evictedCount = prometheus.NewCounter(
+	controlledCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: metricsNamespace,
+			Name:      "controlled_pods_total",
+			Help:      "Number of Pods controlled by VPA updater.",
+		}, []string{"vpa_size_log2"},
+	)
+
+	evictableCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: metricsNamespace,
+			Name:      "evictable_pods_total",
+			Help:      "Number of Pods matching evicition criteria.",
+		}, []string{"vpa_size_log2"},
+	)
+
+	evictedCount = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: metricsNamespace,
 			Name:      "evicted_pods_total",
 			Help:      "Number of Pods evicted by Updater to apply a new recommendation.",
-		},
+		}, []string{"vpa_size_log2"},
+	)
+
+	vpasWithEvictablePodsCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: metricsNamespace,
+			Name:      "vpas_with_evictable_pods_total",
+			Help:      "Number of VPA objects with at least one Pod matching evicition criteria.",
+		}, []string{"vpa_size_log2"},
+	)
+
+	vpasWithEvictedPodsCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: metricsNamespace,
+			Name:      "vpas_with_evicted_pods_total",
+			Help:      "Number of VPA objects with at least one evicted Pod.",
+		}, []string{"vpa_size_log2"},
 	)
 
 	functionLatency = metrics.CreateExecutionTimeMetric(metricsNamespace,
@@ -41,16 +81,57 @@ var (
 
 // Register initializes all metrics for VPA Updater
 func Register() {
-	prometheus.MustRegister(evictedCount)
-	prometheus.MustRegister(functionLatency)
+	prometheus.MustRegister(controlledCount, evictableCount, evictedCount, vpasWithEvictablePodsCount, vpasWithEvictedPodsCount, functionLatency)
 }
 
 // NewExecutionTimer provides a timer for Updater's RunOnce execution
 func NewExecutionTimer() *metrics.ExecutionTimer {
 	return metrics.NewExecutionTimer(functionLatency)
 }
 
-// AddEvictedPod increases the counter of pods evicted by VPA
-func AddEvictedPod() {
-	evictedCount.Add(1)
+// newSizeBasedGauge provides a wrapper for counting items in a loop
+func newSizeBasedGauge(gauge *prometheus.GaugeVec) *SizeBasedGauge {
+	return &SizeBasedGauge{
+		values: [metrics.MaxVpaSizeLog]int{},
+		gauge:  gauge,
+	}
+}
+
+// NewControlledPodsCounter returns a wrapper for counting Pods controlled by Updater
+func NewControlledPodsCounter() *SizeBasedGauge {
+	return newSizeBasedGauge(controlledCount)
+}
+
+// NewEvictablePodsCounter returns a wrapper for counting Pods which are matching eviction criteria
+func NewEvictablePodsCounter() *SizeBasedGauge {
+	return newSizeBasedGauge(evictableCount)
+}
+
+// NewVpasWithEvictablePodsCounter returns a wrapper for counting VPA objects with Pods matching eviction criteria
+func NewVpasWithEvictablePodsCounter() *SizeBasedGauge {
+	return newSizeBasedGauge(vpasWithEvictablePodsCount)
+}
+
+// NewVpasWithEvictedPodsCounter returns a wrapper for counting VPA objects with evicted Pods
+func NewVpasWithEvictedPodsCounter() *SizeBasedGauge {
+	return newSizeBasedGauge(vpasWithEvictedPodsCount)
+}
+
+// AddEvictedPod increases the counter of pods evicted by Updater, by given VPA size
+func AddEvictedPod(vpaSize int) {
+	log2 := metrics.GetVpaSizeLog2(vpaSize)
+	evictedCount.WithLabelValues(strconv.Itoa(log2)).Inc()
+}
+
+// Add increases the counter for the given VPA size
+func (g *SizeBasedGauge) Add(vpaSize int, value int) {
+	log2 := metrics.GetVpaSizeLog2(vpaSize)
+	g.values[log2] += value
+}
+
+// Observe stores the recorded values into metrics object associated with the wrapper
+func (g *SizeBasedGauge) Observe() {
+	for log2, value := range g.values {
+		g.gauge.WithLabelValues(strconv.Itoa(log2)).Set(float64(value))
+	}
 }