Merge pull request #570 from fluxcd/stale-metrics

Delete stale metrics on object delete

Author: darkowlzz

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/fluxcd/pkg/git v0.12.4
 	github.com/fluxcd/pkg/git/gogit v0.12.1
 	github.com/fluxcd/pkg/gittestserver v0.8.5
-	github.com/fluxcd/pkg/runtime v0.41.0
+	github.com/fluxcd/pkg/runtime v0.42.0
 	github.com/fluxcd/pkg/ssh v0.8.1
 	github.com/fluxcd/source-controller/api v1.0.1
 	github.com/go-git/go-billy/v5 v5.4.1

go.sum

@@ -95,8 +95,8 @@ github.com/fluxcd/pkg/git/gogit v0.12.1 h1:06jzHOTntYN5xCSQvyFXtLXdqoP8crLh7VYgt
 github.com/fluxcd/pkg/git/gogit v0.12.1/go.mod h1:Z4Ysp8VifKTvWpjJMKncJsgb2iBqHuIeK80VGjlU41Y=
 github.com/fluxcd/pkg/gittestserver v0.8.5 h1:EGqDF4240xPRgW1FFrQAs0Du7fZb8OGXC5qKDIqyXD8=
 github.com/fluxcd/pkg/gittestserver v0.8.5/go.mod h1:SyGEh+OBzFpdlTWWqv3XBkiLB42Iu+mijfIQ4hPlEZQ=
-github.com/fluxcd/pkg/runtime v0.41.0 h1:hjWUwVRCKDuGEUhovWrygt/6PRry4p278yKuJNgTfv8=
-github.com/fluxcd/pkg/runtime v0.41.0/go.mod h1:1GN+nxoQ7LmSsLJwjH8JW8pA27tBSO+KLH43HpywCDM=
+github.com/fluxcd/pkg/runtime v0.42.0 h1:a5DQ/f90YjoHBmiXZUpnp4bDSLORjInbmqP7K11L4uY=
+github.com/fluxcd/pkg/runtime v0.42.0/go.mod h1:p6A3xWVV8cKLLQW0N90GehKgGMMmbNYv+OSJ/0qB0vg=
 github.com/fluxcd/pkg/ssh v0.8.1 h1:v35y7Ks/+ABWce8RcnrC7psVIhf3EdCUNFJi5+tYOps=
 github.com/fluxcd/pkg/ssh v0.8.1/go.mod h1:M1ouDXuDG+QuhGB4JYEjCNCykNytLJGDhwKn9y4DEOE=
 github.com/fluxcd/pkg/version v0.2.2 h1:ZpVXECeLA5hIQMft11iLp6gN3cKcz6UNuVTQPw/bRdI=

internal/controller/imageupdateautomation_controller.go

@@ -110,6 +110,13 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	defer func() {
+		// Always record suspend, readiness and duration metrics.
+		r.Metrics.RecordSuspend(ctx, &auto, auto.Spec.Suspend)
+		r.Metrics.RecordReadiness(ctx, &auto)
+		r.Metrics.RecordDuration(ctx, &auto, start)
+	}()
+
 	// If the object is under deletion, record the readiness, and remove our finalizer.
 	if !auto.ObjectMeta.DeletionTimestamp.IsZero() {
 		controllerutil.RemoveFinalizer(&auto, imagev1.ImageUpdateAutomationFinalizer)
@@ -131,22 +138,13 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr
 		}
 	}
 
-	// record suspension metrics
-	r.RecordSuspend(ctx, &auto, auto.Spec.Suspend)
-
 	if auto.Spec.Suspend {
 		log.Info("ImageUpdateAutomation is suspended, skipping automation run")
 		return ctrl.Result{}, nil
 	}
 
 	templateValues.AutomationObject = req.NamespacedName
 
-	defer func() {
-		// Always record readiness and duration metrics
-		r.Metrics.RecordReadiness(ctx, &auto)
-		r.Metrics.RecordDuration(ctx, &auto, start)
-	}()
-
 	// whatever else happens, we've now "seen" the reconcile
 	// annotation if it's there
 	if token, ok := meta.ReconcileAnnotationValue(auto.GetAnnotations()); ok {

main.go

@@ -40,6 +40,7 @@ import (
 	feathelper "github.com/fluxcd/pkg/runtime/features"
 	"github.com/fluxcd/pkg/runtime/leaderelection"
 	"github.com/fluxcd/pkg/runtime/logger"
+	"github.com/fluxcd/pkg/runtime/metrics"
 	"github.com/fluxcd/pkg/runtime/pprof"
 	"github.com/fluxcd/pkg/runtime/probes"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1"
@@ -184,7 +185,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	metricsH := helper.MustMakeMetrics(mgr)
+	metricsH := helper.NewMetrics(mgr, metrics.MustMakeRecorder(), imagev1.ImageUpdateAutomationFinalizer)
 
 	ctx := ctrl.SetupSignalHandler()