Add lease list metrics

Author: carreter

pkg/server/leases/gc_controller.go

@@ -54,11 +54,25 @@ func (c *GarbageCollectionController) Run(ctx context.Context) {
 }
 
 func (c *GarbageCollectionController) gc(ctx context.Context) {
+	start := time.Now()
 	leases, err := c.leaseInterface.List(ctx, metav1.ListOptions{LabelSelector: c.labelSelector})
+	latency := time.Now().Sub(start)
 	if err != nil {
 		klog.Errorf("Could not list leases to garbage collect: %v", err)
+
+		var apiStatus apierrors.APIStatus
+		if errors.As(err, &apiStatus) {
+			status := apiStatus.Status()
+			metrics.Metrics.ObserveLeaseList(int(status.Code), string(status.Reason))
+			metrics.Metrics.ObserveLeaseListLatency(int(status.Code), latency)
+		} else {
+			klog.Errorf("Lease list error could not be logged to metrics as it is not an APIStatus: %v", err)
+		}
+
 		return
 	}
+	metrics.Metrics.ObserveLeaseList(200, "")
+	metrics.Metrics.ObserveLeaseListLatency(200, latency)
 
 	for _, lease := range leases.Items {
 		if util.IsLeaseValid(c.pc, lease) {

pkg/server/metrics/metrics.go

@@ -60,6 +60,8 @@ type ServerMetrics struct {
 	culledLeases         prometheus.Counter
 	leaseDeleteLatencies *prometheus.HistogramVec
 	leaseDeletes         *prometheus.CounterVec
+	leaseListLatencies   *prometheus.HistogramVec
+	leaseLists           *prometheus.CounterVec
 }
 
 // newServerMetrics create a new ServerMetrics, configured with default metric names.
@@ -176,6 +178,24 @@ func newServerMetrics() *ServerMetrics {
 		},
 		[]string{"http_status_code", "reason"},
 	)
+	leaseListLatencies := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "lease_list_latency_seconds",
+			Help:      "Latency of lease list calls by the garbage collection controller in seconds.",
+		},
+		[]string{"http_status_code"},
+	)
+	leaseLists := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "lease_list_total",
+			Help:      "Count of lease deletion calls by the garbage collection controller. Labeled by HTTP status code and reason.",
+		},
+		[]string{"http_status_code", "reason"},
+	)
 	streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem)
 	streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem)
 	prometheus.MustRegister(endpointLatencies)
@@ -192,6 +212,8 @@ func newServerMetrics() *ServerMetrics {
 	prometheus.MustRegister(culledLeases)
 	prometheus.MustRegister(leaseDeleteLatencies)
 	prometheus.MustRegister(leaseDeletes)
+	prometheus.MustRegister(leaseListLatencies)
+	prometheus.MustRegister(leaseLists)
 	return &ServerMetrics{
 		endpointLatencies:    endpointLatencies,
 		frontendLatencies:    frontendLatencies,
@@ -207,6 +229,8 @@ func newServerMetrics() *ServerMetrics {
 		culledLeases:         culledLeases,
 		leaseDeleteLatencies: leaseDeleteLatencies,
 		leaseDeletes:         leaseDeletes,
+		leaseListLatencies:   leaseListLatencies,
+		leaseLists:           leaseLists,
 	}
 }
 
@@ -309,3 +333,11 @@ func (s *ServerMetrics) ObserveLeaseDeleteLatency(httpCode int, latency time.Dur
 func (s *ServerMetrics) ObserveLeaseDelete(httpCode int, reason string) {
 	s.leaseDeletes.WithLabelValues(strconv.Itoa(httpCode), reason).Inc()
 }
+
+func (s *ServerMetrics) ObserveLeaseListLatency(httpCode int, latency time.Duration) {
+	s.leaseListLatencies.WithLabelValues(strconv.Itoa(httpCode)).Observe(latency.Seconds())
+}
+
+func (s *ServerMetrics) ObserveLeaseList(httpCode int, reason string) {
+	s.leaseLists.WithLabelValues(strconv.Itoa(httpCode), reason).Inc()
+}