Merge pull request #60 from fluxcd/metricals-probes

Metrics and notifications

Author: squaremo

controllers/imageupdateautomation_controller.go

@@ -33,10 +33,12 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	kuberecorder "k8s.io/client-go/tools/record"
+	"k8s.io/client-go/tools/reference"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -46,6 +48,7 @@ import (
 	imagev1_reflect "github.com/fluxcd/image-reflector-controller/api/v1alpha1"
 	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/runtime/events"
+	"github.com/fluxcd/pkg/runtime/metrics"
 	"github.com/fluxcd/pkg/runtime/predicates"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 	"github.com/fluxcd/source-controller/pkg/git"
@@ -68,9 +71,11 @@ const imagePolicyKey = ".spec.update.imagePolicy"
 // ImageUpdateAutomationReconciler reconciles a ImageUpdateAutomation object
 type ImageUpdateAutomationReconciler struct {
 	client.Client
-	Log           logr.Logger
-	Scheme        *runtime.Scheme
-	EventRecorder kuberecorder.EventRecorder
+	Log                   logr.Logger
+	Scheme                *runtime.Scheme
+	EventRecorder         kuberecorder.EventRecorder
+	ExternalEventRecorder *events.Recorder
+	MetricsRecorder       *metrics.Recorder
 }
 
 // +kubebuilder:rbac:groups=image.toolkit.fluxcd.io,resources=imageupdateautomations,verbs=get;list;watch;create;update;patch;delete
@@ -92,6 +97,19 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, nil
 	}
 
+	// Record readiness metric when exiting; if there's any points at
+	// which the readiness is updated _without also exiting_, they
+	// should also record the readiness.
+	defer r.recordReadinessMetric(&auto)
+	// Record reconciliation duration when exiting
+	if r.MetricsRecorder != nil {
+		objRef, err := reference.GetReference(r.Scheme, &auto)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		defer r.MetricsRecorder.RecordDuration(*objRef, now)
+	}
+
 	// whatever else happens, we've now "seen" the reconcile
 	// annotation if it's there
 	if token, ok := meta.ReconcileAnnotationValue(auto.GetAnnotations()); ok {
@@ -359,12 +377,53 @@ func commitAllAndPush(ctx context.Context, repo *gogit.Repository, access repoAc
 	})
 }
 
-// --- events
+// --- events, metrics
 
 func (r *ImageUpdateAutomationReconciler) event(auto imagev1.ImageUpdateAutomation, severity, msg string) {
 	if r.EventRecorder != nil {
 		r.EventRecorder.Event(&auto, "Normal", severity, msg)
 	}
+	if r.ExternalEventRecorder != nil {
+		objRef, err := reference.GetReference(r.Scheme, &auto)
+		if err != nil {
+			r.Log.WithValues(
+				"request",
+				fmt.Sprintf("%s/%s", auto.GetNamespace(), auto.GetName()),
+			).Error(err, "unable to send event")
+			return
+		}
+
+		if err := r.ExternalEventRecorder.Eventf(*objRef, nil, severity, severity, msg); err != nil {
+			r.Log.WithValues(
+				"request",
+				fmt.Sprintf("%s/%s", auto.GetNamespace(), auto.GetName()),
+			).Error(err, "unable to send event")
+			return
+		}
+	}
+}
+
+func (r *ImageUpdateAutomationReconciler) recordReadinessMetric(auto *imagev1.ImageUpdateAutomation) {
+	if r.MetricsRecorder == nil {
+		return
+	}
+
+	objRef, err := reference.GetReference(r.Scheme, auto)
+	if err != nil {
+		r.Log.WithValues(
+			strings.ToLower(auto.Kind),
+			fmt.Sprintf("%s/%s", auto.GetNamespace(), auto.GetName()),
+		).Error(err, "unable to record readiness metric")
+		return
+	}
+	if rc := apimeta.FindStatusCondition(auto.Status.Conditions, meta.ReadyCondition); rc != nil {
+		r.MetricsRecorder.RecordCondition(*objRef, *rc, !auto.DeletionTimestamp.IsZero())
+	} else {
+		r.MetricsRecorder.RecordCondition(*objRef, metav1.Condition{
+			Type:   meta.ReadyCondition,
+			Status: metav1.ConditionUnknown,
+		}, !auto.DeletionTimestamp.IsZero())
+	}
 }
 
 // --- updates

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/fluxcd/image-reflector-controller v0.0.0-20200810165546-c2265d9b49b9
 	github.com/fluxcd/pkg/apis/meta v0.4.0
 	github.com/fluxcd/pkg/gittestserver v0.0.2
-	github.com/fluxcd/pkg/runtime v0.3.0
+	github.com/fluxcd/pkg/runtime v0.3.1
 	github.com/fluxcd/source-controller v0.3.0
 	// If you bump this, change TOOLKIT_VERSION in the Makefile to match
 	github.com/fluxcd/source-controller/api v0.3.0

go.sum

@@ -290,6 +290,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fluxcd/image-reflector-controller v0.0.0-20200810165546-c2265d9b49b9 h1:puMXg9dmh/i3rpAuzVq5R5L84pZMA/KUNZbPxi0JdWw=
 github.com/fluxcd/image-reflector-controller v0.0.0-20200810165546-c2265d9b49b9/go.mod h1:v2cJ+q7zRlj1VhYjtE1i3cGAFkSTz21/tKidWIIGUtU=
+github.com/fluxcd/pkg v0.0.4 h1:fMA6GG3FTSBFDrlB026gQJhxj4xVuvcvwP3rhX/Yqrw=
 github.com/fluxcd/pkg/apis/meta v0.2.0 h1:bxoFQtZM6OLLj0+n3h6ga7IEWUtGEDJPc65OWiXSMvY=
 github.com/fluxcd/pkg/apis/meta v0.2.0/go.mod h1:50RLLSfqM4LlQrh/+5LiJVf7Hjdthee8WDdXBvpjBdA=
 github.com/fluxcd/pkg/apis/meta v0.3.0/go.mod h1:wOzQQx8CdtUQCGaLzqGu4QgnNxYkI6/wvdvlovxWhF0=
@@ -303,6 +304,8 @@ github.com/fluxcd/pkg/runtime v0.2.0 h1:aZmSLuyA9pF/KANf4wi7pZIICE19BKTYFSPRbl6W
 github.com/fluxcd/pkg/runtime v0.2.0/go.mod h1:P1/S8TOSuJgVPU0SRahWzbNxLWYoUwvBcPCNGc+dWWg=
 github.com/fluxcd/pkg/runtime v0.3.0 h1:WpeTmDT2meIe4NsU081I8zmUGgTYs3bIMRgs9F3Lj90=
 github.com/fluxcd/pkg/runtime v0.3.0/go.mod h1:gPe6JgfPB4EDh5gaVkuI0SPuATk3PmclbFa1kPcZrKE=
+github.com/fluxcd/pkg/runtime v0.3.1 h1:UI+FQd1OgipZ6N8YxXHtKYMAu1NRFWaR/Gp1M3T6RZA=
+github.com/fluxcd/pkg/runtime v0.3.1/go.mod h1:wg33L6k5FkGEYoZta7hbUlBIscM5VZd/PfCbaEi+wK4=
 github.com/fluxcd/pkg/ssh v0.0.5 h1:rnbFZ7voy2JBlUfMbfyqArX2FYaLNpDhccGFC3qW83A=
 github.com/fluxcd/pkg/ssh v0.0.5/go.mod h1:7jXPdXZpc0ttMNz2kD9QuMi3RNn/e0DOFbj0Tij/+Hs=
 github.com/fluxcd/pkg/testserver v0.0.2/go.mod h1:pgUZTh9aQ44FSTQo+5NFlh7YMbUfdz1B80DalW7k96Y=

main.go

@@ -24,17 +24,22 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	imagev1alpha1_reflect "github.com/fluxcd/image-reflector-controller/api/v1alpha1"
+	"github.com/fluxcd/pkg/runtime/events"
 	"github.com/fluxcd/pkg/runtime/logger"
+	"github.com/fluxcd/pkg/runtime/metrics"
+	"github.com/fluxcd/pkg/runtime/probes"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 
 	imagev1alpha1_auto "github.com/fluxcd/image-automation-controller/api/v1alpha1"
 	"github.com/fluxcd/image-automation-controller/controllers"
 	// +kubebuilder:scaffold:imports
 )
 
+const controllerName = "image-automation-controller"
+
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
@@ -52,6 +57,7 @@ func init() {
 func main() {
 	var (
 		metricsAddr          string
+		eventsAddr           string
 		healthAddr           string
 		enableLeaderElection bool
 		logLevel             string
@@ -60,6 +66,7 @@ func main() {
 	)
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&eventsAddr, "events-addr", "", "The address of the events receiver.")
 	flag.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. "+
@@ -72,6 +79,19 @@ func main() {
 
 	ctrl.SetLogger(logger.NewLogger(logLevel, logJSON))
 
+	var eventRecorder *events.Recorder
+	if eventsAddr != "" {
+		if er, err := events.NewRecorder(eventsAddr, controllerName); err != nil {
+			setupLog.Error(err, "unable to create event recorder")
+			os.Exit(1)
+		} else {
+			eventRecorder = er
+		}
+	}
+
+	metricsRecorder := metrics.NewRecorder()
+	ctrlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...)
+
 	watchNamespace := ""
 	if !watchAllNamespaces {
 		watchNamespace = os.Getenv("RUNTIME_NAMESPACE")
@@ -91,13 +111,15 @@ func main() {
 		os.Exit(1)
 	}
 
-	setupChecks(mgr)
+	probes.SetupChecks(mgr, setupLog)
 
 	if err = (&controllers.ImageUpdateAutomationReconciler{
-		Client:        mgr.GetClient(),
-		Log:           ctrl.Log.WithName("controllers").WithName("ImageUpdateAutomation"),
-		Scheme:        mgr.GetScheme(),
-		EventRecorder: mgr.GetEventRecorderFor("image-automation-controller"),
+		Client:                mgr.GetClient(),
+		Log:                   ctrl.Log.WithName("controllers").WithName("ImageUpdateAutomation"),
+		Scheme:                mgr.GetScheme(),
+		EventRecorder:         mgr.GetEventRecorderFor(controllerName),
+		ExternalEventRecorder: eventRecorder,
+		MetricsRecorder:       metricsRecorder,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ImageUpdateAutomation")
 		os.Exit(1)
@@ -110,15 +132,3 @@ func main() {
 		os.Exit(1)
 	}
 }
-
-func setupChecks(mgr ctrl.Manager) {
-	if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to create ready check")
-		os.Exit(1)
-	}
-
-	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to create health check")
-		os.Exit(1)
-	}
-}