feat: add metrics for reconciliation and status updates

Signed-off-by: Kevin Conner <[email protected]>

Author: knrc

go.mod

@@ -8,6 +8,9 @@ require (
 	github.com/go-logr/logr v1.4.2
 	github.com/onsi/ginkgo/v2 v2.22.0
 	github.com/onsi/gomega v1.36.1
+	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/client_model v0.6.1
+	github.com/stretchr/testify v1.9.0
 	golang.org/x/time v0.7.0
 	k8s.io/api v0.32.1
 	k8s.io/apimachinery v0.32.1
@@ -54,8 +57,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/spf13/cobra v1.8.1 // indirect

internal/metrics/metrics.go

@@ -0,0 +1,171 @@
+// Package metrics provides Prometheus metrics for the model validation operator.
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const (
+	// Metric label names
+	labelNamespace          = "namespace"
+	labelModelValidation    = "model_validation"
+	labelPodState           = "pod_state"
+	labelStatusUpdateResult = "result"
+	labelDriftType          = "drift_type"
+
+	// PodStateInjected represents pods with model validation finalizers
+	PodStateInjected = "injected"
+	// PodStateUninjected represents pods without model validation finalizers
+	PodStateUninjected = "uninjected"
+	// PodStateOrphaned represents pods with configuration drift
+	PodStateOrphaned = "orphaned"
+
+	// StatusUpdateSuccess indicates a successful status update
+	StatusUpdateSuccess = "success"
+	// StatusUpdateFailure indicates a failed status update
+	StatusUpdateFailure = "failure"
+)
+
+var (
+	// ModelValidationPodCounts tracks the current number of pods in each state per ModelValidation
+	ModelValidationPodCounts = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "model_validation_operator",
+			Name:      "modelvalidation_pod_count",
+			Help:      "Current number of pods tracked per ModelValidation by state",
+		},
+		[]string{labelNamespace, labelModelValidation, labelPodState},
+	)
+
+	// PodStateTransitionsTotal tracks pod state transitions
+	PodStateTransitionsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "model_validation_operator",
+			Name:      "pod_state_transitions_total",
+			Help:      "Total number of pod state transitions",
+		},
+		[]string{labelNamespace, labelModelValidation, "from_state", "to_state"},
+	)
+
+	// StatusUpdatesTotal tracks ModelValidation status updates
+	StatusUpdatesTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "model_validation_operator",
+			Name:      "status_updates_total",
+			Help:      "Total number of ModelValidation status updates",
+		},
+		[]string{labelNamespace, labelModelValidation, labelStatusUpdateResult},
+	)
+
+	// ConfigurationDriftEventsTotal tracks configuration drift events
+	ConfigurationDriftEventsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "model_validation_operator",
+			Name:      "configuration_drift_events_total",
+			Help:      "Total number of configuration drift events detected",
+		},
+		[]string{labelNamespace, labelModelValidation, labelDriftType},
+	)
+
+	// ModelValidationCRsTotal tracks total number of ModelValidation CRs per namespace
+	// Does not include authMethod for namespace-level tracking
+	ModelValidationCRsTotal = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "model_validation_operator",
+			Name:      "modelvalidation_crs_total",
+			Help:      "Total number of ModelValidation CRs being tracked per namespace",
+		},
+		[]string{labelNamespace},
+	)
+
+	// StatusUpdateDuration tracks the duration of status update operations
+	StatusUpdateDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "model_validation_operator",
+			Name:      "status_update_duration_seconds",
+			Help:      "Duration of ModelValidation status update operations",
+			Buckets:   prometheus.DefBuckets,
+		},
+		[]string{labelNamespace, labelModelValidation, labelStatusUpdateResult},
+	)
+
+	// QueueSize tracks the current size of the status update queue
+	QueueSize = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "model_validation_operator",
+			Name:      "status_update_queue_size",
+			Help:      "Current size of the status update queue",
+		},
+	)
+
+	// RetryAttemptsTotal tracks retry attempts for status updates
+	RetryAttemptsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "model_validation_operator",
+			Name:      "status_update_retry_attempts_total",
+			Help:      "Total number of status update retry attempts",
+		},
+		[]string{labelNamespace, labelModelValidation},
+	)
+)
+
+func init() {
+	metrics.Registry.MustRegister(
+		ModelValidationPodCounts,
+		PodStateTransitionsTotal,
+		StatusUpdatesTotal,
+		ConfigurationDriftEventsTotal,
+		ModelValidationCRsTotal,
+		StatusUpdateDuration,
+		QueueSize,
+		RetryAttemptsTotal,
+	)
+}
+
+// RecordPodCount records the current pod count for a ModelValidation
+func RecordPodCount(namespace, modelValidation, podState string, count float64) {
+	ModelValidationPodCounts.WithLabelValues(namespace, modelValidation, podState).Set(count)
+}
+
+// RecordPodStateTransition records a pod state transition
+func RecordPodStateTransition(namespace, modelValidation, fromState, toState string) {
+	PodStateTransitionsTotal.WithLabelValues(namespace, modelValidation, fromState, toState).Inc()
+}
+
+// RecordStatusUpdate records a status update result
+func RecordStatusUpdate(namespace, modelValidation, result string) {
+	StatusUpdatesTotal.WithLabelValues(namespace, modelValidation, result).Inc()
+}
+
+// RecordConfigurationDrift records a configuration drift event
+func RecordConfigurationDrift(namespace, modelValidation, driftType string) {
+	ConfigurationDriftEventsTotal.WithLabelValues(namespace, modelValidation, driftType).Inc()
+}
+
+// RecordModelValidationCR records the current number of ModelValidation CRs per namespace
+func RecordModelValidationCR(namespace string, count float64) {
+	ModelValidationCRsTotal.WithLabelValues(namespace).Set(count)
+}
+
+// RecordStatusUpdateDuration records the duration of a status update
+func RecordStatusUpdateDuration(namespace, modelValidation, result string, duration float64) {
+	StatusUpdateDuration.WithLabelValues(namespace, modelValidation, result).Observe(duration)
+}
+
+// SetQueueSize sets the current queue size
+func SetQueueSize(size float64) {
+	QueueSize.Set(size)
+}
+
+// RecordRetryAttempt records a retry attempt
+func RecordRetryAttempt(namespace, modelValidation string) {
+	RetryAttemptsTotal.WithLabelValues(namespace, modelValidation).Inc()
+}
+
+// RecordMultiplePodStateTransitions records multiple identical pod state transitions
+func RecordMultiplePodStateTransitions(namespace, modelValidation, fromState, toState string, count int) {
+	if count > 0 {
+		PodStateTransitionsTotal.WithLabelValues(namespace, modelValidation, fromState, toState).Add(float64(count))
+	}
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,144 @@
+package metrics
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const (
+	testNamespace       = "test-namespace"
+	testModelValidation = "test-mv"
+)
+
+func TestMetricsDefinition(t *testing.T) {
+	// Test that all metric variables are defined
+	assert.NotNil(t, ModelValidationPodCounts)
+	assert.NotNil(t, PodStateTransitionsTotal)
+	assert.NotNil(t, StatusUpdatesTotal)
+	assert.NotNil(t, ConfigurationDriftEventsTotal)
+	assert.NotNil(t, ModelValidationCRsTotal)
+	assert.NotNil(t, StatusUpdateDuration)
+	assert.NotNil(t, QueueSize)
+	assert.NotNil(t, RetryAttemptsTotal)
+}
+
+// Test helper to verify gauge metrics
+func verifyGaugeMetric(t *testing.T, gauge *prometheus.GaugeVec, labels []string, expectedValue float64) {
+	metric := gauge.WithLabelValues(labels...)
+	metricDto := &dto.Metric{}
+	err := metric.Write(metricDto)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedValue, metricDto.GetGauge().GetValue())
+}
+
+// Test helper to verify counter metrics
+func verifyCounterIncrement(t *testing.T, counter *prometheus.CounterVec, labels []string) float64 {
+	metric := counter.WithLabelValues(labels...)
+	metricDto := &dto.Metric{}
+	err := metric.Write(metricDto)
+	assert.NoError(t, err)
+	return metricDto.GetCounter().GetValue()
+}
+
+func TestRecordPodCount(t *testing.T) {
+	podState := PodStateInjected
+	count := float64(5)
+
+	RecordPodCount(testNamespace, testModelValidation, podState, count)
+
+	verifyGaugeMetric(t, ModelValidationPodCounts, []string{testNamespace, testModelValidation, podState}, count)
+}
+
+func TestRecordPodStateTransition(t *testing.T) {
+	fromState := PodStateUninjected
+	toState := PodStateInjected
+
+	labels := []string{testNamespace, testModelValidation, fromState, toState}
+	initialValue := verifyCounterIncrement(t, PodStateTransitionsTotal, labels)
+	RecordPodStateTransition(testNamespace, testModelValidation, fromState, toState)
+	finalValue := verifyCounterIncrement(t, PodStateTransitionsTotal, labels)
+
+	assert.Equal(t, initialValue+1, finalValue)
+}
+
+func TestRecordStatusUpdate(t *testing.T) {
+	result := StatusUpdateSuccess
+
+	labels := []string{testNamespace, testModelValidation, result}
+	initialValue := verifyCounterIncrement(t, StatusUpdatesTotal, labels)
+	RecordStatusUpdate(testNamespace, testModelValidation, result)
+	finalValue := verifyCounterIncrement(t, StatusUpdatesTotal, labels)
+
+	assert.Equal(t, initialValue+1, finalValue)
+}
+
+func TestRecordConfigurationDrift(t *testing.T) {
+	driftType := "config_hash"
+
+	labels := []string{testNamespace, testModelValidation, driftType}
+	initialValue := verifyCounterIncrement(t, ConfigurationDriftEventsTotal, labels)
+	RecordConfigurationDrift(testNamespace, testModelValidation, driftType)
+	finalValue := verifyCounterIncrement(t, ConfigurationDriftEventsTotal, labels)
+
+	assert.Equal(t, initialValue+1, finalValue)
+}
+
+func TestRecordModelValidationCR(t *testing.T) {
+	count := float64(3)
+
+	RecordModelValidationCR(testNamespace, count)
+
+	verifyGaugeMetric(t, ModelValidationCRsTotal, []string{testNamespace}, count)
+}
+
+func TestSetQueueSize(t *testing.T) {
+	size := float64(10)
+
+	SetQueueSize(size)
+
+	metricDto := &dto.Metric{}
+	err := QueueSize.Write(metricDto)
+	assert.NoError(t, err)
+	assert.Equal(t, size, metricDto.GetGauge().GetValue())
+}
+
+func TestRecordStatusUpdateDuration(t *testing.T) {
+	result := StatusUpdateSuccess
+	duration := 0.5 // 500ms
+
+	RecordStatusUpdateDuration(testNamespace, testModelValidation, result, duration)
+
+	// Verify the histogram was recorded by checking the metric family
+	metricFamilies, err := metrics.Registry.Gather()
+	assert.NoError(t, err)
+
+	var found bool
+	for _, mf := range metricFamilies {
+		if mf.GetName() == "model_validation_operator_status_update_duration_seconds" {
+			for _, metric := range mf.GetMetric() {
+				if metric.GetHistogram().GetSampleCount() > 0 {
+					found = true
+					break
+				}
+			}
+		}
+	}
+	assert.True(t, found, "Expected histogram metric to be recorded")
+}
+
+func TestRecordMultiplePodStateTransitions(t *testing.T) {
+	fromState := PodStateUninjected
+	toState := PodStateInjected
+	count := 3
+
+	labels := []string{testNamespace, testModelValidation, fromState, toState}
+	initialValue := verifyCounterIncrement(t, PodStateTransitionsTotal, labels)
+	RecordMultiplePodStateTransitions(testNamespace, testModelValidation, fromState, toState, count)
+	finalValue := verifyCounterIncrement(t, PodStateTransitionsTotal, labels)
+
+	assert.Equal(t, initialValue+float64(count), finalValue)
+}