🌱 Add clustercache metrics (#11789)

* add healthcheck metric

* add connection metric

* comments

* improve help text

* address comments  + add labels

* update label naming

Author: cahillsf

controllers/clustercache/cluster_accessor.go

@@ -260,7 +260,10 @@ func (ca *clusterAccessor) Connect(ctx context.Context) (retErr error) {
 	defer func() {
 		if retErr != nil {
 			log.Error(retErr, "Connect failed")
+			connectionUp.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace).Set(0)
 			ca.lockedState.lastConnectionCreationErrorTimestamp = time.Now()
+		} else {
+			connectionUp.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace).Set(1)
 		}
 	}()
 
@@ -303,15 +306,17 @@ func (ca *clusterAccessor) Connect(ctx context.Context) (retErr error) {
 // Disconnect disconnects a connection to the workload cluster.
 func (ca *clusterAccessor) Disconnect(ctx context.Context) {
 	log := ctrl.LoggerFrom(ctx)
-
 	if !ca.Connected(ctx) {
 		log.V(6).Info("Skipping disconnect, already disconnected")
 		return
 	}
 
 	ca.lock(ctx)
-	defer ca.unlock(ctx)
 
+	defer func() {
+		ca.unlock(ctx)
+		connectionUp.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace).Set(0)
+	}()
 	log.Info("Disconnecting")
 
 	// Stopping the cache is non-blocking, so it's okay to do it while holding the lock.
@@ -356,14 +361,20 @@ func (ca *clusterAccessor) HealthCheck(ctx context.Context) (bool, bool) {
 		unauthorizedErrorOccurred = true
 		ca.lockedState.healthChecking.consecutiveFailures++
 		log.V(6).Info(fmt.Sprintf("Health probe failed (unauthorized error occurred): %v", err))
+		healthCheck.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace).Set(0)
+		healthChecksTotal.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace, "error").Inc()
 	case err != nil:
 		ca.lockedState.healthChecking.consecutiveFailures++
 		log.V(6).Info(fmt.Sprintf("Health probe failed (%d/%d): %v",
 			ca.lockedState.healthChecking.consecutiveFailures, ca.config.HealthProbe.FailureThreshold, err))
+		healthCheck.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace).Set(0)
+		healthChecksTotal.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace, "error").Inc()
 	default:
 		ca.lockedState.healthChecking.consecutiveFailures = 0
 		ca.lockedState.healthChecking.lastProbeSuccessTimestamp = ca.lockedState.healthChecking.lastProbeTimestamp
 		log.V(6).Info("Health probe succeeded")
+		healthCheck.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace).Set(1)
+		healthChecksTotal.WithLabelValues(ca.cluster.Name, ca.cluster.Namespace, "success").Inc()
 	}
 
 	tooManyConsecutiveFailures := ca.lockedState.healthChecking.consecutiveFailures >= ca.config.HealthProbe.FailureThreshold

controllers/clustercache/metrics.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clustercache
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+func init() {
+	// Register the metrics at the controller-runtime metrics registry.
+	ctrlmetrics.Registry.MustRegister(healthCheck)
+	ctrlmetrics.Registry.MustRegister(connectionUp)
+	ctrlmetrics.Registry.MustRegister(healthChecksTotal)
+}
+
+var (
+	healthCheck = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "capi_cluster_cache_healthcheck",
+			Help: "Result of the last clustercache healthcheck for a cluster.",
+		}, []string{
+			"cluster_name", "cluster_namespace",
+		},
+	)
+	healthChecksTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "capi_cluster_cache_healthchecks_total",
+			Help: "Results of all clustercache healthchecks.",
+		}, []string{
+			"cluster_name", "cluster_namespace", "status",
+		},
+	)
+	connectionUp = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "capi_cluster_cache_connection_up",
+			Help: "Whether the connection to the cluster is up.",
+		}, []string{
+			"cluster_name", "cluster_namespace",
+		},
+	)
+)