Record metrics for readiness and duration

Using the conventions of the day.

Signed-off-by: Michael Bridgen <[email protected]>

Author: squaremo

controllers/imageupdateautomation_controller.go

@@ -33,10 +33,12 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	kuberecorder "k8s.io/client-go/tools/record"
+	"k8s.io/client-go/tools/reference"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -46,6 +48,7 @@ import (
 	imagev1_reflect "github.com/fluxcd/image-reflector-controller/api/v1alpha1"
 	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/runtime/events"
+	"github.com/fluxcd/pkg/runtime/metrics"
 	"github.com/fluxcd/pkg/runtime/predicates"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 	"github.com/fluxcd/source-controller/pkg/git"
@@ -68,9 +71,10 @@ const imagePolicyKey = ".spec.update.imagePolicy"
 // ImageUpdateAutomationReconciler reconciles a ImageUpdateAutomation object
 type ImageUpdateAutomationReconciler struct {
 	client.Client
-	Log           logr.Logger
-	Scheme        *runtime.Scheme
-	EventRecorder kuberecorder.EventRecorder
+	Log             logr.Logger
+	Scheme          *runtime.Scheme
+	EventRecorder   kuberecorder.EventRecorder
+	MetricsRecorder *metrics.Recorder
 }
 
 // +kubebuilder:rbac:groups=image.toolkit.fluxcd.io,resources=imageupdateautomations,verbs=get;list;watch;create;update;patch;delete
@@ -92,6 +96,19 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, nil
 	}
 
+	// Record readiness metric when exiting; if there's any points at
+	// which the readiness is updated _without also exiting_, they
+	// should also record the readiness.
+	defer r.recordReadinessMetric(&auto)
+	// Record reconciliation duration when exiting
+	if r.MetricsRecorder != nil {
+		objRef, err := reference.GetReference(r.Scheme, &auto)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		defer r.MetricsRecorder.RecordDuration(*objRef, now)
+	}
+
 	// whatever else happens, we've now "seen" the reconcile
 	// annotation if it's there
 	if token, ok := meta.ReconcileAnnotationValue(auto.GetAnnotations()); ok {
@@ -359,14 +376,37 @@ func commitAllAndPush(ctx context.Context, repo *gogit.Repository, access repoAc
 	})
 }
 
-// --- events
+// --- events, metrics
 
 func (r *ImageUpdateAutomationReconciler) event(auto imagev1.ImageUpdateAutomation, severity, msg string) {
 	if r.EventRecorder != nil {
 		r.EventRecorder.Event(&auto, "Normal", severity, msg)
 	}
 }
 
+func (r *ImageUpdateAutomationReconciler) recordReadinessMetric(auto *imagev1.ImageUpdateAutomation) {
+	if r.MetricsRecorder == nil {
+		return
+	}
+
+	objRef, err := reference.GetReference(r.Scheme, auto)
+	if err != nil {
+		r.Log.WithValues(
+			strings.ToLower(auto.Kind),
+			fmt.Sprintf("%s/%s", auto.GetNamespace(), auto.GetName()),
+		).Error(err, "unable to record readiness metric")
+		return
+	}
+	if rc := apimeta.FindStatusCondition(auto.Status.Conditions, meta.ReadyCondition); rc != nil {
+		r.MetricsRecorder.RecordCondition(*objRef, *rc, !auto.DeletionTimestamp.IsZero())
+	} else {
+		r.MetricsRecorder.RecordCondition(*objRef, metav1.Condition{
+			Type:   meta.ReadyCondition,
+			Status: metav1.ConditionUnknown,
+		}, !auto.DeletionTimestamp.IsZero())
+	}
+}
+
 // --- updates
 
 // updateAccordingToSetters updates files under the root by treating

main.go

@@ -25,9 +25,11 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	imagev1alpha1_reflect "github.com/fluxcd/image-reflector-controller/api/v1alpha1"
 	"github.com/fluxcd/pkg/runtime/logger"
+	"github.com/fluxcd/pkg/runtime/metrics"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 
 	imagev1alpha1_auto "github.com/fluxcd/image-automation-controller/api/v1alpha1"
@@ -72,6 +74,9 @@ func main() {
 
 	ctrl.SetLogger(logger.NewLogger(logLevel, logJSON))
 
+	metricsRecorder := metrics.NewRecorder()
+	ctrlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...)
+
 	watchNamespace := ""
 	if !watchAllNamespaces {
 		watchNamespace = os.Getenv("RUNTIME_NAMESPACE")
@@ -94,10 +99,11 @@ func main() {
 	setupChecks(mgr)
 
 	if err = (&controllers.ImageUpdateAutomationReconciler{
-		Client:        mgr.GetClient(),
-		Log:           ctrl.Log.WithName("controllers").WithName("ImageUpdateAutomation"),
-		Scheme:        mgr.GetScheme(),
-		EventRecorder: mgr.GetEventRecorderFor("image-automation-controller"),
+		Client:          mgr.GetClient(),
+		Log:             ctrl.Log.WithName("controllers").WithName("ImageUpdateAutomation"),
+		Scheme:          mgr.GetScheme(),
+		EventRecorder:   mgr.GetEventRecorderFor("image-automation-controller"),
+		MetricsRecorder: metricsRecorder,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ImageUpdateAutomation")
 		os.Exit(1)