[OSPRH-21020] Change rabbitmq to use podmonitors

RabbitMQ can be deployed with multiple replicas, while each replica
will export different metrics. Having a ScrapeConfig with a service
as a target means we're not scraping all metrics. PodMonitor
will find all rabbitMQ pods based on a label, so all pods will
get scraped everytime.

Author: vyzigold

controllers/metricstorage_controller.go

@@ -224,6 +224,14 @@ func (r *MetricStorageReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return r.reconcileDelete(ctx, instance, helper)
 	}
 
+	// Handle service update
+	ctrlResult, err := r.reconcileUpdate(ctx, instance, helper)
+	if err != nil {
+		return ctrlResult, err
+	} else if (ctrlResult != ctrl.Result{}) {
+		return ctrlResult, nil
+	}
+
 	// Handle non-deleted clusters
 	return r.reconcileNormal(ctx, instance, helper)
 }
@@ -247,8 +255,6 @@ func (r *MetricStorageReconciler) reconcileDelete(
 	return ctrl.Result{}, nil
 }
 
-// TODO: call the function appropriately
-//
 //nolint:all
 func (r *MetricStorageReconciler) reconcileUpdate(
 	ctx context.Context,
@@ -262,6 +268,10 @@ func (r *MetricStorageReconciler) reconcileUpdate(
 	if err != nil {
 		return ctrl.Result{}, err
 	}
+	err = r.deleteRabbitMQScrapeConfig(ctx, instance)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
 
 	Log.Info(fmt.Sprintf("Reconciled Service '%s' update successfully", instance.Name))
 
@@ -295,6 +305,33 @@ func (r *MetricStorageReconciler) deleteOldServiceMonitors(
 	return nil
 }
 
+// Delete RabbitMQ ScrapeConfig
+// A ScrapeConfig for RabbitMQ was last used at the beginning of FR4
+func (r *MetricStorageReconciler) deleteRabbitMQScrapeConfig(
+	ctx context.Context,
+	instance *telemetryv1.MetricStorage,
+) error {
+	namespacedName := types.NamespacedName{
+		Name:      fmt.Sprintf("%s-rabbitmq", telemetry.ServiceName),
+		Namespace: instance.Namespace,
+	}
+	scrapeConfig := &monv1alpha1.ScrapeConfig{}
+	err := r.Get(ctx, namespacedName, scrapeConfig)
+	if err != nil {
+		if k8s_errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	if object.CheckOwnerRefExist(instance.UID, scrapeConfig.OwnerReferences) {
+		err = r.Delete(ctx, scrapeConfig)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (r *MetricStorageReconciler) reconcileNormal(
 	ctx context.Context,
 	instance *telemetryv1.MetricStorage,
@@ -684,6 +721,31 @@ func (r *MetricStorageReconciler) prometheusEndpointSecret(
 	return nil
 }
 
+func (r *MetricStorageReconciler) createPodMonitor(
+	ctx context.Context,
+	instance *telemetryv1.MetricStorage,
+	log logr.Logger,
+	desiredPodMonitor *monv1.PodMonitor,
+) error {
+	podMonitor := &monv1.PodMonitor{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      desiredPodMonitor.Name,
+			Namespace: instance.Namespace,
+		},
+	}
+	op, err := controllerutil.CreateOrPatch(ctx, r.Client, podMonitor, func() error {
+		desiredPodMonitor.Spec.DeepCopyInto(&podMonitor.Spec)
+		podMonitor.Labels = desiredPodMonitor.Labels
+		err := controllerutil.SetControllerReference(instance, podMonitor, r.Scheme)
+		return err
+	})
+
+	if err == nil && op != controllerutil.OperationResultNone {
+		log.Info(fmt.Sprintf("PodMonitor %s successfully changed - operation: %s", podMonitor.GetName(), string(op)))
+	}
+	return err
+}
+
 func (r *MetricStorageReconciler) createServiceScrapeConfig(
 	ctx context.Context,
 	instance *telemetryv1.MetricStorage,
@@ -760,10 +822,9 @@ func (r *MetricStorageReconciler) createScrapeConfigs(
 		return ctrl.Result{}, err
 	}
 
-	// ScrapeConfigs for RabbitMQ monitoring
+	// PodMonitors for RabbitMQ monitoring
 	// NOTE: We're watching Rabbits and reconciling with each of their change
-	//       that should keep the targets inside the ScrapeConfig always
-	//       up to date.
+	//       that should keep the PodMonitors always up to date.
 	rabbitList := &rabbitmqv1.RabbitmqClusterList{}
 	listOpts := []client.ListOption{
 		client.InNamespace(instance.GetNamespace()),
@@ -772,22 +833,17 @@ func (r *MetricStorageReconciler) createScrapeConfigs(
 	if err != nil && !k8s_errors.IsNotFound(err) {
 		return ctrl.Result{}, err
 	}
-	rabbitTargets := []string{}
 	for _, rabbit := range rabbitList.Items {
-		rabbitServerName := fmt.Sprintf("%s.%s.svc", rabbit.Name, rabbit.Namespace)
-		rabbitTargets = append(rabbitTargets, net.JoinHostPort(rabbitServerName, strconv.Itoa(metricstorage.RabbitMQPrometheusPort)))
-	}
-	rabbitCfgName := fmt.Sprintf("%s-rabbitmq", telemetry.ServiceName)
-	desiredScrapeConfig = metricstorage.ScrapeConfig(
-		instance,
-		serviceLabels,
-		rabbitTargets,
-		instance.Spec.PrometheusTLS.Enabled(),
-	)
-	err = r.createServiceScrapeConfig(ctx, instance, Log, "RabbitMQ",
-		rabbitCfgName, desiredScrapeConfig)
-	if err != nil {
-		return ctrl.Result{}, err
+		desiredPodMonitor := metricstorage.RabbitMQPodMonitor(
+			instance,
+			serviceLabels,
+			rabbit.Name,
+			instance.Spec.PrometheusTLS.Enabled(),
+		)
+		err = r.createPodMonitor(ctx, instance, Log, desiredPodMonitor)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
 	}
 
 	// mysqld exporter