Fix the proxy server backend metric

Author: maqiuyujoyce

pkg/server/backend_manager.go

@@ -271,24 +271,30 @@ type DefaultBackendStorage struct {
 	// e.g., when associating to the DestHostBackendManager, it can only use the
 	// identifiers of types, IPv4, IPv6 and Host.
 	idTypes []header.IdentifierType
+	// managerName is the kind of backend manager this storage belongs to.
+	// It is used to record metrics.
+	managerName string
 }
 
 // NewDefaultBackendManager returns a DefaultBackendManager.
 func NewDefaultBackendManager() *DefaultBackendManager {
 	return &DefaultBackendManager{
 		DefaultBackendStorage: NewDefaultBackendStorage(
-			[]header.IdentifierType{header.UID})}
+			[]header.IdentifierType{header.UID}, "DefaultBackendManager")}
 }
 
 // NewDefaultBackendStorage returns a DefaultBackendStorage
-func NewDefaultBackendStorage(idTypes []header.IdentifierType) *DefaultBackendStorage {
+func NewDefaultBackendStorage(idTypes []header.IdentifierType, managerName string) *DefaultBackendStorage {
 	// Set an explicit value, so that the metric is emitted even when
 	// no agent ever successfully connects.
 	metrics.Metrics.SetBackendCount(0)
+	metrics.Metrics.SetTotalBackendCount(managerName, 0)
+
 	return &DefaultBackendStorage{
-		backends: make(map[string][]*Backend),
-		random:   rand.New(rand.NewSource(time.Now().UnixNano())), /* #nosec G404 */
-		idTypes:  idTypes,
+		backends:    make(map[string][]*Backend),
+		random:      rand.New(rand.NewSource(time.Now().UnixNano())), /* #nosec G404 */
+		idTypes:     idTypes,
+		managerName: managerName,
 	}
 }
 
@@ -318,6 +324,7 @@ func (s *DefaultBackendStorage) addBackend(identifier string, idType header.Iden
 	}
 	s.backends[identifier] = []*Backend{backend}
 	metrics.Metrics.SetBackendCount(len(s.backends))
+	metrics.Metrics.SetTotalBackendCount(s.managerName, len(s.backends))
 	s.agentIDs = append(s.agentIDs, identifier)
 }
 
@@ -359,6 +366,7 @@ func (s *DefaultBackendStorage) removeBackend(identifier string, idType header.I
 		klog.V(1).InfoS("Could not find connection matching identifier to remove", "agentID", identifier, "idType", idType)
 	}
 	metrics.Metrics.SetBackendCount(len(s.backends))
+	metrics.Metrics.SetTotalBackendCount(s.managerName, len(s.backends))
 }
 
 // NumBackends resturns the number of available backends

pkg/server/default_route_backend_manager.go

@@ -32,7 +32,7 @@ var _ BackendManager = &DefaultRouteBackendManager{}
 func NewDefaultRouteBackendManager() *DefaultRouteBackendManager {
 	return &DefaultRouteBackendManager{
 		DefaultBackendStorage: NewDefaultBackendStorage(
-			[]header.IdentifierType{header.DefaultRoute})}
+			[]header.IdentifierType{header.DefaultRoute}, "DefaultRouteBackendManager")}
 }
 
 // Backend tries to get a backend that advertises default route, with random selection.

pkg/server/desthost_backend_manager.go

@@ -32,7 +32,7 @@ var _ BackendManager = &DestHostBackendManager{}
 func NewDestHostBackendManager() *DestHostBackendManager {
 	return &DestHostBackendManager{
 		DefaultBackendStorage: NewDefaultBackendStorage(
-			[]header.IdentifierType{header.IPv4, header.IPv6, header.Host})}
+			[]header.IdentifierType{header.IPv4, header.IPv6, header.Host}, "DestHostBackendManager")}
 }
 
 func (dibm *DestHostBackendManager) AddBackend(backend *Backend) {

pkg/server/metrics/metrics.go

@@ -51,6 +51,7 @@ type ServerMetrics struct {
 	grpcConnections      *prometheus.GaugeVec
 	httpConnections      prometheus.Gauge
 	backend              *prometheus.GaugeVec
+	totalBackendCount    *prometheus.GaugeVec
 	pendingDials         *prometheus.GaugeVec
 	establishedConns     *prometheus.GaugeVec
 	fullRecvChannels     *prometheus.GaugeVec
@@ -114,6 +115,17 @@ func newServerMetrics() *ServerMetrics {
 		},
 		[]string{},
 	)
+	totalBackendCount := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "ready_backend_connections_total",
+			Help:      "Total number of konnectivity agent connected to the proxy server",
+		},
+		[]string{
+			"manager",
+		},
+	)
 	pendingDials := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: Namespace,
@@ -203,6 +215,7 @@ func newServerMetrics() *ServerMetrics {
 	prometheus.MustRegister(grpcConnections)
 	prometheus.MustRegister(httpConnections)
 	prometheus.MustRegister(backend)
+	prometheus.MustRegister(totalBackendCount)
 	prometheus.MustRegister(pendingDials)
 	prometheus.MustRegister(establishedConns)
 	prometheus.MustRegister(fullRecvChannels)
@@ -220,6 +233,7 @@ func newServerMetrics() *ServerMetrics {
 		grpcConnections:      grpcConnections,
 		httpConnections:      httpConnections,
 		backend:              backend,
+		totalBackendCount:    totalBackendCount,
 		pendingDials:         pendingDials,
 		establishedConns:     establishedConns,
 		fullRecvChannels:     fullRecvChannels,
@@ -240,6 +254,7 @@ func (s *ServerMetrics) Reset() {
 	s.frontendLatencies.Reset()
 	s.grpcConnections.Reset()
 	s.backend.Reset()
+	s.totalBackendCount.Reset()
 	s.pendingDials.Reset()
 	s.establishedConns.Reset()
 	s.fullRecvChannels.Reset()
@@ -284,6 +299,11 @@ func (s *ServerMetrics) SetBackendCount(count int) {
 	s.backend.WithLabelValues().Set(float64(count))
 }
 
+// SetTotalBackendCount sets the total number of backend connection.
+func (s *ServerMetrics) SetTotalBackendCount(managerName string, count int) {
+	s.totalBackendCount.WithLabelValues(managerName).Set(float64(count))
+}
+
 // SetPendingDialCount sets the number of pending dials.
 func (s *ServerMetrics) SetPendingDialCount(count int) {
 	s.pendingDials.WithLabelValues().Set(float64(count))