feat: enable RP metrics (#195)

jwtty · web-flow · commit d8d26187bc18 · 2025-08-18T13:02:53.000-07:00
diff --git a/cmd/hubagent/main.go b/cmd/hubagent/main.go
@@ -85,9 +85,16 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 	klog.InitFlags(nil)
 
-	metrics.Registry.MustRegister(fleetmetrics.JoinResultMetrics, fleetmetrics.LeaveResultMetrics,
-		fleetmetrics.PlacementApplyFailedCount, fleetmetrics.PlacementApplySucceedCount,
-		fleetmetrics.SchedulingCycleDurationMilliseconds, fleetmetrics.SchedulerActiveWorkers)
+	metrics.Registry.MustRegister(
+		fleetmetrics.JoinResultMetrics,
+		fleetmetrics.LeaveResultMetrics,
+		fleetmetrics.PlacementApplyFailedCount,
+		fleetmetrics.PlacementApplySucceedCount,
+		fleetmetrics.SchedulingCycleDurationMilliseconds,
+		fleetmetrics.SchedulerActiveWorkers,
+		fleetmetrics.FleetPlacementStatusLastTimeStampSeconds,
+		fleetmetrics.FleetEvictionStatus,
+	)
 }
 
 func main() {
diff --git a/pkg/controllers/clusterresourceplacement/controller.go b/pkg/controllers/clusterresourceplacement/controller.go
@@ -37,11 +37,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/scheduler/queue"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/annotations"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/defaulter"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/labels"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/resource"
@@ -112,7 +112,7 @@ func (r *Reconciler) handleDelete(ctx context.Context, placementObj fleetv1beta1
 		return ctrl.Result{}, err
 	}
 	// change the metrics to add nameplace of namespace
-	metrics.FleetPlacementStatusLastTimeStampSeconds.DeletePartialMatch(prometheus.Labels{"name": placementObj.GetName()})
+	metrics.FleetPlacementStatusLastTimeStampSeconds.DeletePartialMatch(prometheus.Labels{"namespace": placementObj.GetNamespace(), "name": placementObj.GetName()})
 	controllerutil.RemoveFinalizer(placementObj, fleetv1beta1.PlacementCleanupFinalizer)
 	if err := r.Client.Update(ctx, placementObj); err != nil {
 		klog.ErrorS(err, "Failed to remove placement finalizer", "placement", placementKObj)
@@ -1238,7 +1238,7 @@ func emitPlacementStatusMetric(placementObj fleetv1beta1.PlacementObj) {
 			status = string(cond.Status)
 			reason = cond.Reason
 		}
-		metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), scheduledConditionType, status, reason).SetToCurrentTime()
+		metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), scheduledConditionType, status, reason).SetToCurrentTime()
 		return
 	}
 
@@ -1252,12 +1252,12 @@ func emitPlacementStatusMetric(placementObj fleetv1beta1.PlacementObj) {
 				status = string(cond.Status)
 				reason = cond.Reason
 			}
-			metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), conditionType, status, reason).SetToCurrentTime()
+			metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), conditionType, status, reason).SetToCurrentTime()
 			return
 		}
 	}
 
 	// Emit the "Completed" condition metric to indicate that the placement has completed.
 	// This condition is used solely for metric reporting purposes.
-	metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), "Completed", string(metav1.ConditionTrue), "Completed").SetToCurrentTime()
+	metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), "Completed", string(metav1.ConditionTrue), "Completed").SetToCurrentTime()
 }
diff --git a/pkg/controllers/clusterresourceplacement/controller_integration_test.go b/pkg/controllers/clusterresourceplacement/controller_integration_test.go
diff --git a/pkg/controllers/clusterresourceplacement/suite_test.go b/pkg/controllers/clusterresourceplacement/suite_test.go
@@ -35,13 +35,15 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
 	"github.com/kubefleet-dev/kubefleet/cmd/hubagent/options"
 	"github.com/kubefleet-dev/kubefleet/pkg/controllers/clusterresourcebindingwatcher"
 	"github.com/kubefleet-dev/kubefleet/pkg/controllers/clusterresourceplacementwatcher"
 	"github.com/kubefleet-dev/kubefleet/pkg/controllers/clusterschedulingpolicysnapshot"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/informer"
@@ -150,6 +152,9 @@ var _ = BeforeSuite(func() {
 	}).SetupWithManagerForClusterResourceBinding(mgr)
 	Expect(err).Should(Succeed(), "failed to create clusterResourceBinding watcher")
 
+	// Register metrics.
+	ctrlmetrics.Registry.MustRegister(metrics.FleetPlacementStatusLastTimeStampSeconds)
+
 	ctx, cancel = context.WithCancel(context.TODO())
 	// Run the controller manager
 	go func() {
diff --git a/pkg/controllers/clusterresourceplacementeviction/controller.go b/pkg/controllers/clusterresourceplacementeviction/controller.go
@@ -35,10 +35,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 	bindingutils "github.com/kubefleet-dev/kubefleet/pkg/utils/binding"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/defaulter"
 	evictionutils "github.com/kubefleet-dev/kubefleet/pkg/utils/eviction"
 )
diff --git a/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go b/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go
@@ -33,8 +33,8 @@ import (
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"
 	testutilseviction "github.com/kubefleet-dev/kubefleet/test/utils/eviction"
 )
 
diff --git a/pkg/controllers/clusterresourceplacementeviction/controller_test.go b/pkg/controllers/clusterresourceplacementeviction/controller_test.go
@@ -38,8 +38,8 @@ import (
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/defaulter"
 )
 
diff --git a/pkg/controllers/clusterresourceplacementeviction/suite_test.go b/pkg/controllers/clusterresourceplacementeviction/suite_test.go
@@ -19,6 +19,7 @@ package clusterresourceplacementeviction
 import (
 	"context"
 	"flag"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -33,9 +34,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 )
 
 var (
@@ -47,6 +50,13 @@ var (
 	cancel    context.CancelFunc
 )
 
+func TestMain(m *testing.M) {
+	// Register here as the metric is both tested in ginkgo tests and go unit tests.
+	ctrlmetrics.Registry.MustRegister(metrics.FleetEvictionStatus)
+
+	os.Exit(m.Run())
+}
+
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
 
diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go
@@ -41,10 +41,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/informer"
 )
 
diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go
@@ -43,9 +43,9 @@ import (
 
 	clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1"
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"
-	"github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"
 	metricsutils "github.com/kubefleet-dev/kubefleet/test/utils/metrics"
 )
 
diff --git a/pkg/controllers/updaterun/suite_test.go b/pkg/controllers/updaterun/suite_test.go
@@ -35,11 +35,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1"
 	placementv1alpha1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1alpha1"
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
+	"github.com/kubefleet-dev/kubefleet/pkg/metrics"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/informer"
 )
@@ -96,11 +98,11 @@ var _ = BeforeSuite(func() {
 	})
 	Expect(err).Should(Succeed())
 
-	// make sure the k8s client is same as the controller client, or we can have cache delay
+	// Make sure the k8s client is same as the controller client, or we can have cache delay.
 	By("set k8s client same as the controller manager")
 	k8sClient = mgr.GetClient()
 
-	// setup informer manager for the reconciler
+	// Setup informer manager for the reconciler.
 	dynamicClient, err := dynamic.NewForConfig(cfg)
 	Expect(err).Should(Succeed())
 	dynamicInformerManager := informer.NewInformerManager(dynamicClient, 0, ctx.Done())
@@ -110,13 +112,16 @@ var _ = BeforeSuite(func() {
 		IsClusterScoped:      true,
 	}, nil)
 
-	// setup our main reconciler
+	// Setup our main reconciler.
 	err = (&Reconciler{
 		Client:          k8sClient,
 		InformerManager: dynamicInformerManager,
 	}).SetupWithManager(mgr)
 	Expect(err).Should(Succeed())
 
+	// Register metrics.
+	ctrlmetrics.Registry.MustRegister(metrics.FleetUpdateRunStatusLastTimestampSeconds)
+
 	go func() {
 		defer GinkgoRecover()
 		err = mgr.Start(ctx)
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -43,6 +43,26 @@ var (
 		Name: "placement_apply_succeed_counter",
 		Help: "Number of successfully applied cluster resource placement",
 	}, []string{"name"})
+
+	// FleetPlacementStatusLastTimeStampSeconds is a prometheus metric which keeps track of the last placement status.
+	FleetPlacementStatusLastTimeStampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "fleet_workload_placement_status_last_timestamp_seconds",
+		Help: "Last update timestamp of placement status in seconds",
+	}, []string{"namespace", "name", "generation", "conditionType", "status", "reason"})
+
+	// FleetEvictionStatus is prometheus metrics which holds the
+	// status of eviction completion.
+	FleetEvictionStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "fleet_workload_eviction_complete",
+		Help: "Last update timestamp of eviction complete status in seconds",
+	}, []string{"name", "isCompleted", "isValid"})
+
+	// FleetUpdateRunStatusLastTimestampSeconds is a prometheus metric which holds the
+	// last update timestamp of update run status in seconds.
+	FleetUpdateRunStatusLastTimestampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "fleet_workload_update_run_status_last_timestamp_seconds",
+		Help: "Last update timestamp of update run status in seconds",
+	}, []string{"name", "generation", "condition", "status", "reason"})
 )
 
 var (
diff --git a/pkg/utils/controller/metrics/metrics.go b/pkg/utils/controller/metrics/metrics.go
@@ -60,26 +60,6 @@ var (
 		Name: "fleet_workload_active_workers",
 		Help: "Number of currently used workers per controller",
 	}, []string{"controller"})
-
-	// FleetPlacementStatusLastTimeStampSeconds is a prometheus metric which keeps track of the last placement status.
-	FleetPlacementStatusLastTimeStampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "fleet_workload_placement_status_last_timestamp_seconds",
-		Help: "Timestamp in seconds of the last current placement status condition of crp.",
-	}, []string{"name", "generation", "conditionType", "status", "reason"})
-
-	// FleetEvictionStatus is prometheus metrics which holds the
-	// status of eviction completion.
-	FleetEvictionStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "fleet_workload_eviction_complete",
-		Help: "Eviction complete status ",
-	}, []string{"name", "isCompleted", "isValid"})
-
-	// FleetUpdateRunStatusLastTimestampSeconds is a prometheus metric which holds the
-	// last update timestamp of update run status in seconds.
-	FleetUpdateRunStatusLastTimestampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "fleet_workload_update_run_status_last_timestamp_seconds",
-		Help: "Last update timestamp of update run status in seconds",
-	}, []string{"name", "generation", "condition", "status", "reason"})
 )
 
 func init() {
@@ -89,8 +69,5 @@ func init() {
 		FleetReconcileTime,
 		FleetWorkerCount,
 		FleetActiveWorkers,
-		FleetPlacementStatusLastTimeStampSeconds,
-		FleetEvictionStatus,
-		FleetUpdateRunStatusLastTimestampSeconds,
 	)
 }

Original file line number	Diff line number	Diff line change
`@@ -33,8 +33,8 @@ import (`
`33`	`33`	`ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"`
`34`	`34`
`35`	`35`	`placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"`
	`36`	`+ "github.com/kubefleet-dev/kubefleet/pkg/metrics"`
`36`	`37`	`"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"`
`37`		`- "github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"`
`38`	`38`	`testutilseviction "github.com/kubefleet-dev/kubefleet/test/utils/eviction"`
`39`	`39`	`)`
`40`	`40`
Original file line number	Diff line number	Diff line change
`@@ -38,8 +38,8 @@ import (`
`38`	`38`	`ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"`
`39`	`39`
`40`	`40`	`placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"`
	`41`	`+ "github.com/kubefleet-dev/kubefleet/pkg/metrics"`
`41`	`42`	`"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"`
`42`		`- "github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"`
`43`	`43`	`"github.com/kubefleet-dev/kubefleet/pkg/utils/defaulter"`
`44`	`44`	`)`
`45`	`45`
Original file line number	Diff line number	Diff line change
`@@ -43,9 +43,9 @@ import (`
`43`	`43`
`44`	`44`	`clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1"`
`45`	`45`	`placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"`
	`46`	`+ "github.com/kubefleet-dev/kubefleet/pkg/metrics"`
`46`	`47`	`"github.com/kubefleet-dev/kubefleet/pkg/utils"`
`47`	`48`	`"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"`
`48`		`- "github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics"`
`49`	`49`	`metricsutils "github.com/kubefleet-dev/kubefleet/test/utils/metrics"`
`50`	`50`	`)`
`51`	`51`