Fix up not all metrics were being cleared down

Author: David Collom

cmd/app/app.go

@@ -40,7 +40,13 @@ func NewCommand(ctx context.Context) *cobra.Command {
 		RunE: func(_ *cobra.Command, _ []string) error {
 			opts.complete()
 
-			log := logrus.New().WithField("component", "controller")
+			logLevel, err := logrus.ParseLevel(opts.LogLevel)
+			if err != nil {
+				return fmt.Errorf("failed to parse --log-level %q: %s",
+					opts.LogLevel, err)
+			}
+
+			log := newLogger(logLevel).WithField("component", "controller")
 
 			defaultTestAllInfoMsg := fmt.Sprintf(`only containers with the annotation "%s/${my-container}=true" will be parsed`, api.EnableAnnotationKey)
 			if opts.DefaultTestAll {
@@ -57,13 +63,13 @@ func NewCommand(ctx context.Context) *cobra.Command {
 			mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
 				Logger:         logrusr.New(log.WithField("controller", "manager").Logger),
 				LeaderElection: false,
-				// TODO: See if we can get newer/better solution
 				Metrics: server.Options{
 					BindAddress:   opts.MetricsServingAddress,
 					SecureServing: false,
 				},
 				GracefulShutdownTimeout: &opts.GracefulShutdownTimeout,
 				Cache:                   cache.Options{SyncPeriod: &opts.CacheSyncPeriod},
+				PprofBindAddress:        opts.PprofBindAddress,
 			})
 			if err != nil {
 				return err
@@ -99,10 +105,6 @@ func NewCommand(ctx context.Context) *cobra.Command {
 				metricsServer.RoundTripper,
 			)
 
-			// nodeController := controller.NewNodeReconciler(log, mgr.GetClient())
-			// if err := nodeController.SetupWithManager(mgr); err != nil {
-			// }
-
 			client, err := client.New(ctx, log, opts.Client)
 			if err != nil {
 				return fmt.Errorf("failed to setup image registry clients: %s", err)

cmd/app/options.go

@@ -68,6 +68,7 @@ type Options struct {
 	CacheTimeout          time.Duration
 	LogLevel              string
 
+	PprofBindAddress        string
 	GracefulShutdownTimeout time.Duration
 	CacheSyncPeriod         time.Duration
 
@@ -108,6 +109,10 @@ func (o *Options) addAppFlags(fs *pflag.FlagSet) {
 		"metrics-serving-address", "m", "0.0.0.0:8080",
 		"Address to serve metrics on at the /metrics path.")
 
+	fs.StringVarP(&o.PprofBindAddress,
+		"pprof-serving-address", "", "",
+		"Address to serve pprof on for profiling.")
+
 	fs.BoolVarP(&o.DefaultTestAll,
 		"test-all-containers", "a", false,
 		"If enabled, all containers will be tested, unless they have the "+

pkg/client/client.go

@@ -44,6 +44,7 @@ type Options struct {
 }
 
 func New(ctx context.Context, log *logrus.Entry, opts Options) (*Client, error) {
+	log = log.WithField("component", "client")
 	// Setup Transporters for all remaining clients (if one is set)
 	if opts.Transport != nil {
 		opts.Quay.Transporter = opts.Transport
@@ -106,7 +107,7 @@ func New(ctx context.Context, log *logrus.Entry, opts Options) (*Client, error)
 			quay.New(opts.Quay, log),
 		),
 		fallbackClient: fallbackClient,
-		log:            log.WithField("client", "registry"),
+		log:            log,
 	}
 
 	for _, client := range append(c.clients, fallbackClient) {

pkg/controller/pod_controller.go

@@ -7,8 +7,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 
-	schema "k8s.io/apimachinery/pkg/runtime/schema"
-
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -68,12 +66,6 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 
 	// Fetch the Pod instance
 	pod := &corev1.Pod{}
-	// Because we're using builder.OnlyMetadata and caching only metadata
-	pod.SetGroupVersionKind(schema.GroupVersionKind{
-		Group:   "",
-		Version: "v1",
-		Kind:    "Pod",
-	})
 	err := r.Get(ctx, req.NamespacedName, pod)
 	if apierrors.IsNotFound(err) {
 		// Pod deleted, remove from metrics

pkg/metrics/metrics.go

@@ -106,31 +106,43 @@ func (m *Metrics) AddImage(namespace, pod, container, containerType, imageURL st
 func (m *Metrics) RemoveImage(namespace, pod, container, containerType string) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
+	total := 0
 
-	m.containerImageVersion.DeletePartialMatch(
+	total += m.containerImageVersion.DeletePartialMatch(
 		m.buildPartialLabels(namespace, pod),
 	)
-	m.containerImageDuration.DeletePartialMatch(
+	total += m.containerImageDuration.DeletePartialMatch(
 		m.buildPartialLabels(namespace, pod),
 	)
-	m.containerImageChecked.DeletePartialMatch(
+
+	total += m.containerImageChecked.DeletePartialMatch(
+		m.buildPartialLabels(namespace, pod),
+	)
+	total += m.containerImageErrors.DeletePartialMatch(
 		m.buildPartialLabels(namespace, pod),
 	)
+	m.log.Infof("Removed %d metrics for image %s/%s/%s", total, namespace, pod, container)
 }
 
 func (m *Metrics) RemovePod(namespace, pod string) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
-	m.containerImageVersion.DeletePartialMatch(
+	total := 0
+	total += m.containerImageVersion.DeletePartialMatch(
 		m.buildPartialLabels(namespace, pod),
 	)
-	m.containerImageDuration.DeletePartialMatch(
+	total += m.containerImageDuration.DeletePartialMatch(
 		m.buildPartialLabels(namespace, pod),
 	)
-	m.containerImageChecked.DeletePartialMatch(
+	total += m.containerImageChecked.DeletePartialMatch(
 		m.buildPartialLabels(namespace, pod),
 	)
+	total += m.containerImageErrors.DeletePartialMatch(
+		m.buildPartialLabels(namespace, pod),
+	)
+
+	m.log.Infof("Removed %d metrics for pod %s/%s", total, namespace, pod)
 }
 
 func (m *Metrics) RegisterImageDuration(namespace, pod, container, image string, startTime time.Time) {