Add lease deletion HTTP call metrics

Author: carreter

pkg/server/leases/gc_controller.go

@@ -17,9 +17,10 @@ package leases
 
 import (
 	"context"
+	"errors"
 	"time"
 
-	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -66,13 +67,30 @@ func (c *GarbageCollectionController) gc(ctx context.Context) {
 
 		// Optimistic concurrency: if a lease has a different resourceVersion than
 		// when we got it, it may have been renewed.
+		start := time.Now()
 		err := c.leaseInterface.Delete(ctx, lease.Name, *metav1.NewRVDeletionPrecondition(lease.ResourceVersion))
-		if errors.IsNotFound(err) {
+		if apierrors.IsNotFound(err) {
 			klog.V(4).Infof("Lease %v was already deleted", lease.Name)
 		} else if err != nil {
 			klog.Errorf("Could not delete lease %v: %v", lease.Name, err)
 		} else {
 			metrics.Metrics.CulledLeasesInc()
 		}
+
+		// Log metrics for the deletion call.
+		latency := time.Now().Sub(start)
+		if err != nil {
+			var apiStatus apierrors.APIStatus
+			if errors.As(err, &apiStatus) {
+				status := apiStatus.Status()
+				metrics.Metrics.ObserveLeaseDelete(int(status.Code), string(status.Reason))
+				metrics.Metrics.ObserveLeaseDeleteLatency(int(status.Code), latency)
+			} else {
+				klog.Errorf("Lease delete error could not be logged to metrics as it is not an APIStatus: %v", err)
+			}
+		} else {
+			metrics.Metrics.ObserveLeaseDelete(200, "")
+			metrics.Metrics.ObserveLeaseDeleteLatency(200, latency)
+		}
 	}
 }

pkg/server/metrics/metrics.go

@@ -17,6 +17,7 @@ limitations under the License.
 package metrics
 
 import (
+	"strconv"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -45,18 +46,20 @@ var (
 
 // ServerMetrics includes all the metrics of the proxy server.
 type ServerMetrics struct {
-	endpointLatencies *prometheus.HistogramVec
-	frontendLatencies *prometheus.HistogramVec
-	grpcConnections   *prometheus.GaugeVec
-	httpConnections   prometheus.Gauge
-	backend           *prometheus.GaugeVec
-	pendingDials      *prometheus.GaugeVec
-	establishedConns  *prometheus.GaugeVec
-	fullRecvChannels  *prometheus.GaugeVec
-	dialFailures      *prometheus.CounterVec
-	streamPackets     *prometheus.CounterVec
-	streamErrors      *prometheus.CounterVec
-	culledLeases      prometheus.Counter
+	endpointLatencies    *prometheus.HistogramVec
+	frontendLatencies    *prometheus.HistogramVec
+	grpcConnections      *prometheus.GaugeVec
+	httpConnections      prometheus.Gauge
+	backend              *prometheus.GaugeVec
+	pendingDials         *prometheus.GaugeVec
+	establishedConns     *prometheus.GaugeVec
+	fullRecvChannels     *prometheus.GaugeVec
+	dialFailures         *prometheus.CounterVec
+	streamPackets        *prometheus.CounterVec
+	streamErrors         *prometheus.CounterVec
+	culledLeases         prometheus.Counter
+	leaseDeleteLatencies *prometheus.HistogramVec
+	leaseDeletes         *prometheus.CounterVec
 }
 
 // newServerMetrics create a new ServerMetrics, configured with default metric names.
@@ -155,6 +158,24 @@ func newServerMetrics() *ServerMetrics {
 		Name:      "culled_leases_count",
 		Help:      "Count of expired leases that the lease garbage collection controller has culled.",
 	})
+	leaseDeleteLatencies := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "lease_delete_latency_seconds",
+			Help:      "Latency of lease deletion calls by the garbage collection controller in seconds.",
+		},
+		[]string{"http_status_code"},
+	)
+	leaseDeletes := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "lease_delete_total",
+			Help:      "Count of lease delection calls by the garbage collection controller. Labeled by HTTP status code and reason.",
+		},
+		[]string{"http_status_code", "reason"},
+	)
 	streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem)
 	streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem)
 	prometheus.MustRegister(endpointLatencies)
@@ -169,19 +190,23 @@ func newServerMetrics() *ServerMetrics {
 	prometheus.MustRegister(streamPackets)
 	prometheus.MustRegister(streamErrors)
 	prometheus.MustRegister(culledLeases)
+	prometheus.MustRegister(leaseDeleteLatencies)
+	prometheus.MustRegister(leaseDeletes)
 	return &ServerMetrics{
-		endpointLatencies: endpointLatencies,
-		frontendLatencies: frontendLatencies,
-		grpcConnections:   grpcConnections,
-		httpConnections:   httpConnections,
-		backend:           backend,
-		pendingDials:      pendingDials,
-		establishedConns:  establishedConns,
-		fullRecvChannels:  fullRecvChannels,
-		dialFailures:      dialFailures,
-		streamPackets:     streamPackets,
-		streamErrors:      streamErrors,
-		culledLeases:      culledLeases,
+		endpointLatencies:    endpointLatencies,
+		frontendLatencies:    frontendLatencies,
+		grpcConnections:      grpcConnections,
+		httpConnections:      httpConnections,
+		backend:              backend,
+		pendingDials:         pendingDials,
+		establishedConns:     establishedConns,
+		fullRecvChannels:     fullRecvChannels,
+		dialFailures:         dialFailures,
+		streamPackets:        streamPackets,
+		streamErrors:         streamErrors,
+		culledLeases:         culledLeases,
+		leaseDeleteLatencies: leaseDeleteLatencies,
+		leaseDeletes:         leaseDeletes,
 	}
 }
 
@@ -276,3 +301,11 @@ func (s *ServerMetrics) ObserveStreamErrorNoPacket(segment commonmetrics.Segment
 func (s *ServerMetrics) ObserveStreamError(segment commonmetrics.Segment, err error, packetType client.PacketType) {
 	commonmetrics.ObserveStreamError(s.streamErrors, segment, err, packetType)
 }
+
+func (s *ServerMetrics) ObserveLeaseDeleteLatency(httpCode int, latency time.Duration) {
+	s.leaseDeleteLatencies.WithLabelValues(strconv.Itoa(httpCode)).Observe(latency.Seconds())
+}
+
+func (s *ServerMetrics) ObserveLeaseDelete(httpCode int, reason string) {
+	s.leaseDeletes.WithLabelValues(strconv.Itoa(httpCode), reason).Inc()
+}