add option to configure histogram buckets (#164)

Author: aidan-canva

status/controller.go

@@ -62,6 +62,7 @@ type Option struct {
 	GaugeMetricLabels     []string
 	MetricFields          map[string]string
 	GaugeMetricFields     map[string]string
+	HistogramBuckets      []float64
 }
 
 func EmitDeprecatedMetrics(o *Option) {
@@ -92,6 +93,12 @@ func WithGaugeFields(fields map[string]string) func(*Option) {
 	}
 }
 
+func WithHistogramBuckets(buckets []float64) func(*Option) {
+	return func(o *Option) {
+		o.HistogramBuckets = buckets
+	}
+}
+
 func NewController[T Object](client client.Client, eventRecorder record.EventRecorder, opts ...option.Function[Option]) *Controller[T] {
 	options := option.Resolve(opts...)
 	obj := reflect.New(reflect.TypeOf(*new(T)).Elem()).Interface().(runtime.Object)
@@ -107,7 +114,7 @@ func NewController[T Object](client client.Client, eventRecorder record.EventRec
 		kubeClient:                  client,
 		eventRecorder:               eventRecorder,
 		emitDeprecatedMetrics:       options.EmitDeprecatedMetrics,
-		ConditionDuration: conditionDurationMetric(strings.ToLower(gvk.Kind), lo.Map(
+		ConditionDuration: conditionDurationMetric(strings.ToLower(gvk.Kind), options.HistogramBuckets, lo.Map(
 			append(options.MetricLabels, lo.Keys(options.MetricFields)...),
 			func(k string, _ int) string { return toPrometheusLabel(k) })...),
 		ConditionCount: conditionCountMetric(strings.ToLower(gvk.Kind), lo.Map(
@@ -128,7 +135,7 @@ func NewController[T Object](client client.Client, eventRecorder record.EventRec
 				append(lo.Keys(options.MetricFields), lo.Keys(options.GaugeMetricFields)...),
 				append(options.MetricLabels, options.GaugeMetricLabels...)...,
 			), func(k string, _ int) string { return toPrometheusLabel(k) })...),
-		TerminationDuration: terminationDurationMetric(strings.ToLower(gvk.Kind), lo.Map(
+		TerminationDuration: terminationDurationMetric(strings.ToLower(gvk.Kind), options.HistogramBuckets, lo.Map(
 			append(options.MetricLabels, lo.Keys(options.MetricFields)...),
 			func(k string, _ int) string { return toPrometheusLabel(k) })...),
 	}

status/controller_test.go

@@ -763,6 +763,45 @@ var _ = Describe("Controller", func() {
 		Entry("when using WithFields", status.WithFields(map[string]string{"operator.pkg/key1": ".spec.field1", "operator.pkg/key2": ".spec.field2", "operator.pkg/key3": ".spec.field3"}), false),
 		Entry("when using WithGaugeFields", status.WithGaugeFields(map[string]string{"operator.pkg/key1": ".spec.field1", "operator.pkg/key2": ".spec.field2", "operator.pkg/key3": ".spec.field3"}), true),
 	)
+	It("should use custom histogram buckets when specified", func() {
+		customBuckets := []float64{0.1, 0.5, 1.0, 2.0, 5.0}
+		metrics.Registry = prometheus.NewRegistry()
+
+		controller = status.NewController[*test.CustomObject](kubeClient, recorder, status.WithHistogramBuckets(customBuckets))
+
+		testObject := test.Object(&test.CustomObject{})
+		testObject.StatusConditions() // initialize conditions
+
+		// Apply object and reconcile to set initial state
+		ExpectApplied(ctx, kubeClient, testObject)
+		ExpectReconciled(ctx, controller, testObject)
+
+		// Wait a bit to ensure some time passes for duration measurement
+		time.Sleep(100 * time.Millisecond)
+
+		// Transition a condition to trigger histogram observation
+		testObject.StatusConditions().SetTrue(test.ConditionTypeFoo)
+		ExpectApplied(ctx, kubeClient, testObject)
+		ExpectReconciled(ctx, controller, testObject)
+
+		// Verify that the histogram metric exists and has data
+		metric := GetMetric("operator_customobject_status_condition_transition_seconds", conditionLabels(test.ConditionTypeFoo, metav1.ConditionUnknown))
+		Expect(metric).ToNot(BeNil())
+
+		histogram := metric.GetHistogram()
+		Expect(histogram).ToNot(BeNil())
+		Expect(histogram.GetSampleCount()).To(BeNumerically(">", 0))
+
+		// Verify custom buckets are being used by checking bucket count matches our custom buckets
+		// The histogram should have len(customBuckets) + 1 buckets (including +Inf)
+		buckets := histogram.GetBucket()
+		Expect(len(buckets)).To(Equal(len(customBuckets)))
+
+		// Verify the bucket upper bounds match our custom buckets
+		for i, bucket := range buckets {
+			Expect(bucket.GetUpperBound()).To(Equal(customBuckets[i]))
+		}
+	})
 })
 
 var _ = Describe("Generic Controller", func() {

status/metrics.go

@@ -21,10 +21,11 @@ const (
 )
 
 // Cardinality is limited to # objects * # conditions * # objectives
-var ConditionDuration = conditionDurationMetric("", pmetrics.LabelGroup, pmetrics.LabelKind)
+var ConditionDuration = conditionDurationMetric("", nil, pmetrics.LabelGroup, pmetrics.LabelKind)
 
-func conditionDurationMetric(objectName string, additionalLabels ...string) pmetrics.ObservationMetric {
+func conditionDurationMetric(objectName string, buckets []float64, additionalLabels ...string) pmetrics.ObservationMetric {
 	subsystem := lo.Ternary(len(objectName) == 0, MetricSubsystem, fmt.Sprintf("%s_%s", objectName, MetricSubsystem))
+	buckets = lo.Ternary(len(buckets) == 0, prometheus.DefBuckets, buckets)
 
 	return pmetrics.NewPrometheusHistogram(
 		metrics.Registry,
@@ -33,6 +34,7 @@ func conditionDurationMetric(objectName string, additionalLabels ...string) pmet
 			Subsystem: subsystem,
 			Name:      "transition_seconds",
 			Help:      "The amount of time a condition was in a given state before transitioning. e.g. Alarm := P99(Updated=False) > 5 minutes",
+			Buckets:   buckets,
 		},
 		append([]string{
 			pmetrics.LabelType,
@@ -134,10 +136,11 @@ func terminationCurrentTimeSecondsMetric(objectName string, additionalLabels ...
 	)
 }
 
-var TerminationDuration = terminationDurationMetric("", pmetrics.LabelGroup, pmetrics.LabelKind)
+var TerminationDuration = terminationDurationMetric("", nil, pmetrics.LabelGroup, pmetrics.LabelKind)
 
-func terminationDurationMetric(objectName string, additionalLabels ...string) pmetrics.ObservationMetric {
+func terminationDurationMetric(objectName string, buckets []float64, additionalLabels ...string) pmetrics.ObservationMetric {
 	subsystem := lo.Ternary(len(objectName) == 0, TerminationSubsystem, fmt.Sprintf("%s_%s", objectName, TerminationSubsystem))
+	buckets = lo.Ternary(len(buckets) == 0, prometheus.DefBuckets, buckets)
 
 	return pmetrics.NewPrometheusHistogram(
 		metrics.Registry,
@@ -146,6 +149,7 @@ func terminationDurationMetric(objectName string, additionalLabels ...string) pm
 			Subsystem: subsystem,
 			Name:      "duration_seconds",
 			Help:      "The amount of time taken by an object to terminate completely.",
+			Buckets:   buckets,
 		},
 		additionalLabels,
 	)