Add metrics for controller

Author: filiptronicek

components/node-labeler/cmd/metrics.go

@@ -26,4 +26,21 @@ var (
 		Help:      "time it took for a pods to reach the running phase and the ready label was applied to the node",
 		Buckets:   []float64{5, 10, 15, 20, 25, 30, 45, 60, 75},
 	}, []string{"component"})
+
+	// Track reconciliation durations for the NodeScaledownAnnotationController
+	NodeScaledownAnnotationReconcileDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsWorkspaceSubsystem,
+		Name:      "node_scaledown_annotation_reconcile_duration_seconds",
+		Help:      "Duration of NodeScaledownAnnotationController reconciliations",
+		Buckets:   []float64{0.1, 0.5, 1, 2.5, 5, 10, 30},
+	}, []string{"operation"})
+
+	// Track queue size for the NodeScaledownAnnotationController
+	NodeScaledownAnnotationReconciliationQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsWorkspaceSubsystem,
+		Name:      "node_scaledown_annotation_reconciliation_queue_size",
+		Help:      "Current size of the NodeScaledownAnnotationController reconciliation queue",
+	})
 )

components/node-labeler/cmd/run.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/bombsimon/logrusr/v2"
 	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -143,6 +144,8 @@ var runCmd = &cobra.Command{
 
 		metrics.Registry.MustRegister(NodeLabelerCounterVec)
 		metrics.Registry.MustRegister(NodeLabelerTimeHistVec)
+		metrics.Registry.MustRegister(NodeScaledownAnnotationReconcileDuration)
+		metrics.Registry.MustRegister(NodeScaledownAnnotationReconciliationQueueSize)
 
 		err = mgr.AddHealthzCheck("healthz", healthz.Ping)
 		if err != nil {
@@ -356,6 +359,7 @@ func (c *NodeScaledownAnnotationController) workspaceFilter() predicate.Predicat
 				default:
 					log.WithField("node", ws.Status.Runtime.NodeName).Warn("reconciliation queue full")
 				}
+				NodeScaledownAnnotationReconciliationQueueSize.Set(float64(len(c.nodesToReconcile)))
 				return true
 			}
 			return false
@@ -402,6 +406,9 @@ func (wc *NodeScaledownAnnotationController) Stop() {
 
 // reconcileAllNodes lists all nodes and reconciles each one
 func (wc *NodeScaledownAnnotationController) reconcileAllNodes(ctx context.Context) (ctrl.Result, error) {
+	timer := prometheus.NewTimer(NodeScaledownAnnotationReconcileDuration.WithLabelValues("all_nodes"))
+	defer timer.ObserveDuration()
+
 	var nodes corev1.NodeList
 	if err := wc.List(ctx, &nodes); err != nil {
 		log.WithError(err).Error("failed to list nodes")
@@ -420,6 +427,9 @@ func (wc *NodeScaledownAnnotationController) reconcileAllNodes(ctx context.Conte
 
 // reconcileNode counts the workspaces running on a node and updates the autoscaler annotation accordingly
 func (c *NodeScaledownAnnotationController) reconcileNode(ctx context.Context, nodeName string) error {
+	timer := prometheus.NewTimer(NodeScaledownAnnotationReconcileDuration.WithLabelValues("node"))
+	defer timer.ObserveDuration()
+
 	var workspaceList workspacev1.WorkspaceList
 	if err := c.List(ctx, &workspaceList, client.MatchingFields{
 		"status.runtime.nodeName": nodeName,