Add http status code to lease list latency

Author: carreter

pkg/agent/lease_counter.go

@@ -95,9 +95,10 @@ func NewLeaseInformerWithMetrics(client kubernetes.Interface, namespace string,
 		&cache.ListWatch{
 			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
 				start := time.Now()
+				httpCode := 200
 				defer func() {
 					latency := time.Now().Sub(start)
-					metrics.Metrics.ObserveLeaseListLatency(latency)
+					metrics.Metrics.ObserveLeaseListLatency(latency, httpCode)
 				}()
 				obj, err := client.CoordinationV1().Leases(namespace).List(context.TODO(), options)
 				if err != nil {
@@ -106,6 +107,7 @@ func NewLeaseInformerWithMetrics(client kubernetes.Interface, namespace string,
 					var apiStatus apierrors.APIStatus
 					if errors.As(err, &apiStatus) {
 						status := apiStatus.Status()
+						httpCode = int(status.Code)
 						metrics.Metrics.ObserveLeaseList(int(status.Code), string(status.Reason))
 					} else {
 						klog.Errorf("Lease list error could not be logged to metrics as it is not an APIStatus: %v", err)

pkg/agent/metrics/metrics.go

@@ -60,7 +60,7 @@ type AgentMetrics struct {
 	streamErrors        *prometheus.CounterVec
 	leaseLists          *prometheus.CounterVec
 	leaseWatches        *prometheus.CounterVec
-	leaseListLatencies  prometheus.Histogram
+	leaseListLatencies  *prometheus.HistogramVec
 }
 
 // newAgentMetrics create a new AgentMetrics, configured with default metric names.
@@ -137,14 +137,15 @@ func newAgentMetrics() *AgentMetrics {
 		},
 		[]string{"http_response_code", "reason"},
 	)
-	leaseListLatencies := prometheus.NewHistogram(
+	leaseListLatencies := prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: Namespace,
 			Subsystem: Subsystem,
 			Name:      "lease_list_latency_seconds",
 			Help:      "Latency of server lease listing in seconds",
 			Buckets:   latencyBuckets,
 		},
+		[]string{"http_response_code"},
 	)
 	streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem)
 	streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem)
@@ -226,8 +227,8 @@ func (a *AgentMetrics) ObserveLeaseWatch(httpCode int, reason string) {
 	a.leaseLists.WithLabelValues(strconv.Itoa(httpCode), reason).Inc()
 }
 
-func (a *AgentMetrics) ObserveLeaseListLatency(latency time.Duration) {
-	a.leaseListLatencies.Observe(latency.Seconds())
+func (a *AgentMetrics) ObserveLeaseListLatency(latency time.Duration, httpCode int) {
+	a.leaseListLatencies.WithLabelValues(strconv.Itoa(httpCode)).Observe(latency.Seconds())
 }
 
 // EndpointConnectionInc increments a new endpoint connection.