Wire in metrics

Author: carreter

pkg/agent/lease_counter.go

@@ -2,13 +2,16 @@ package agent
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/apiserver-network-proxy/pkg/agent/metrics"
 
 	coordinationv1api "k8s.io/api/coordination/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
 )
@@ -42,12 +45,28 @@ func NewServerLeaseCounter(k8sClient kubernetes.Interface, labelSelector labels.
 func (lc *ServerLeaseCounter) Count(ctx context.Context) int {
 	// Since the number of proxy servers is generally small (1-10), we opted against
 	// using a LIST and WATCH pattern and instead list all leases on each call.
+	start := timeNow()
+	defer func() {
+		latency := timeNow().Sub(start)
+		metrics.Metrics.ObserveLeaseListLatency(latency)
+	}()
 	leases, err := lc.leaseClient.List(ctx, metav1.ListOptions{LabelSelector: lc.selector.String()})
 	if err != nil {
 		klog.Errorf("could not list leases to update server count, using fallback count (%v): %v", lc.fallbackCount, err)
+
+		apiStatus, ok := err.(apierrors.APIStatus)
+		if ok || errors.As(err, &apiStatus) {
+			status := apiStatus.Status()
+			metrics.Metrics.ObserveLeaseList(int(status.Code), string(status.Reason))
+		} else {
+			klog.Errorf("error could not be logged to metrics as it is not an APIStatus: %v", err)
+		}
+
 		return lc.fallbackCount
 	}
 
+	metrics.Metrics.ObserveLeaseList(200, "")
+
 	count := 0
 	for _, lease := range leases.Items {
 		if isLeaseValid(lease) {

pkg/agent/metrics/metrics.go

@@ -17,6 +17,7 @@ limitations under the License.
 package metrics
 
 import (
+	"strconv"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -57,6 +58,8 @@ type AgentMetrics struct {
 	endpointConnections *prometheus.GaugeVec
 	streamPackets       *prometheus.CounterVec
 	streamErrors        *prometheus.CounterVec
+	leaseLists          *prometheus.CounterVec
+	leaseListLatencies  *prometheus.HistogramVec
 }
 
 // newAgentMetrics create a new AgentMetrics, configured with default metric names.
@@ -116,6 +119,25 @@ func newAgentMetrics() *AgentMetrics {
 		},
 		[]string{},
 	)
+	leaseLists := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "lease_lists_total",
+			Help:      "Count of server lease list calls made by the agent to the k8s apiserver, labeled by HTTP response code and reason",
+		},
+		[]string{"http_response_code", "reason"},
+	)
+	leaseListLatencies := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "lease_list_latency_seconds",
+			Help:      "Latency of server lease listing in seconds",
+			Buckets:   latencyBuckets,
+		},
+		[]string{},
+	)
 	streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem)
 	streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem)
 	prometheus.MustRegister(dialLatencies)
@@ -126,6 +148,8 @@ func newAgentMetrics() *AgentMetrics {
 	prometheus.MustRegister(streamPackets)
 	prometheus.MustRegister(streamErrors)
 	prometheus.MustRegister(serverCount)
+	prometheus.MustRegister(leaseLists)
+	prometheus.MustRegister(leaseListLatencies)
 	return &AgentMetrics{
 		dialLatencies:       dialLatencies,
 		serverFailures:      serverFailures,
@@ -135,6 +159,8 @@ func newAgentMetrics() *AgentMetrics {
 		streamPackets:       streamPackets,
 		streamErrors:        streamErrors,
 		serverCount:         serverCount,
+		leaseLists:          leaseLists,
+		leaseListLatencies:  leaseListLatencies,
 	}
 
 }
@@ -148,6 +174,9 @@ func (a *AgentMetrics) Reset() {
 	a.endpointConnections.Reset()
 	a.streamPackets.Reset()
 	a.streamErrors.Reset()
+	a.serverCount.Reset()
+	a.leaseLists.Reset()
+	a.leaseListLatencies.Reset()
 }
 
 // ObserveServerFailure records a failure to send to or receive from the proxy
@@ -181,6 +210,14 @@ func (a *AgentMetrics) SetServerCount(count int) {
 	a.serverCount.WithLabelValues().Set(float64(count))
 }
 
+func (a *AgentMetrics) ObserveLeaseList(httpCode int, reason string) {
+	a.leaseLists.WithLabelValues(strconv.Itoa(httpCode), reason).Inc()
+}
+
+func (a *AgentMetrics) ObserveLeaseListLatency(latency time.Duration) {
+	a.leaseListLatencies.WithLabelValues().Observe(latency.Seconds())
+}
+
 // EndpointConnectionInc increments a new endpoint connection.
 func (a *AgentMetrics) EndpointConnectionInc() {
 	a.endpointConnections.WithLabelValues().Inc()