Prevent re-including processes that are still in Terminating.

Author: brownleej

controllers/remove_pods.go

@@ -52,6 +52,7 @@ func (u RemovePods) Reconcile(r *FoundationDBClusterReconciler, context ctx.Cont
 	}
 
 	r.Recorder.Event(cluster, "Normal", "RemovingProcesses", fmt.Sprintf("Removing pods: %v", processGroupsToRemove))
+	removedProcessGroups := make(map[string]bool)
 	allRemoved := true
 	for _, id := range processGroupsToRemove {
 
@@ -64,15 +65,17 @@ func (u RemovePods) Reconcile(r *FoundationDBClusterReconciler, context ctx.Cont
 		if err != nil {
 			return false, err
 		}
-		if !removed {
+		if removed {
+			removedProcessGroups[id] = true
+		} else {
 			allRemoved = false
 			continue
 		}
+	}
 
-		err = includeInstance(r, context, cluster, id)
-		if err != nil {
-			return false, err
-		}
+	err := includeInstance(r, context, cluster, removedProcessGroups)
+	if err != nil {
+		return false, err
 	}
 
 	return allRemoved, nil
@@ -134,7 +137,7 @@ func confirmPodRemoval(r *FoundationDBClusterReconciler, context ctx.Context, cl
 		return false, err
 	}
 	if len(instances) == 1 {
-		log.Info("Waiting for instance get torn down", "namespace", cluster.Namespace, "cluster", cluster.Name, "instanceID", instanceID, "pod", instances[0].Metadata.Name)
+		log.Info("Waiting for instance to get torn down", "namespace", cluster.Namespace, "cluster", cluster.Name, "instanceID", instanceID, "pod", instances[0].Metadata.Name)
 		return false, nil
 	} else if len(instances) > 0 {
 		return false, fmt.Errorf("Multiple pods found for cluster %s, instance ID %s", cluster.Name, instanceID)
@@ -146,7 +149,7 @@ func confirmPodRemoval(r *FoundationDBClusterReconciler, context ctx.Context, cl
 		return false, err
 	}
 	if len(pods.Items) == 1 {
-		log.Info("Waiting for pod get torn down", "namespace", cluster.Namespace, "cluster", cluster.Name, "instanceID", instanceID, "pod", pods.Items[0].Name)
+		log.Info("Waiting for pod to get torn down", "namespace", cluster.Namespace, "cluster", cluster.Name, "instanceID", instanceID, "pod", pods.Items[0].Name)
 		return false, nil
 	} else if len(pods.Items) > 0 {
 		return false, fmt.Errorf("Multiple pods found for cluster %s, instance ID %s", cluster.Name, instanceID)
@@ -158,7 +161,7 @@ func confirmPodRemoval(r *FoundationDBClusterReconciler, context ctx.Context, cl
 		return false, err
 	}
 	if len(pvcs.Items) == 1 {
-		log.Info("Waiting for volume claim get torn down", "namespace", cluster.Namespace, "cluster", cluster.Name, "instanceID", instanceID, "pvc", pvcs.Items[0].Name)
+		log.Info("Waiting for volume claim to get torn down", "namespace", cluster.Namespace, "cluster", cluster.Name, "instanceID", instanceID, "pvc", pvcs.Items[0].Name)
 		return false, nil
 	} else if len(pvcs.Items) > 0 {
 		return false, fmt.Errorf("Multiple PVCs found for cluster %s, instance ID %s", cluster.Name, instanceID)
@@ -170,7 +173,7 @@ func confirmPodRemoval(r *FoundationDBClusterReconciler, context ctx.Context, cl
 		return false, err
 	}
 	if len(services.Items) == 1 {
-		log.Info("Waiting for service get torn down", "namespace", cluster.Namespace, "cluster", cluster.Name, "instanceID", instanceID, "service", services.Items[0].Name)
+		log.Info("Waiting for service to get torn down", "namespace", cluster.Namespace, "cluster", cluster.Name, "instanceID", instanceID, "service", services.Items[0].Name)
 		return false, nil
 	} else if len(services.Items) > 0 {
 		return false, fmt.Errorf("Multiple services found for cluster %s, instance ID %s", cluster.Name, instanceID)
@@ -179,7 +182,7 @@ func confirmPodRemoval(r *FoundationDBClusterReconciler, context ctx.Context, cl
 	return true, nil
 }
 
-func includeInstance(r *FoundationDBClusterReconciler, context ctx.Context, cluster *fdbtypes.FoundationDBCluster, instanceID string) error {
+func includeInstance(r *FoundationDBClusterReconciler, context ctx.Context, cluster *fdbtypes.FoundationDBCluster, removedProcessGroups map[string]bool) error {
 	adminClient, err := r.AdminClientProvider(cluster, r)
 	if err != nil {
 		return err
@@ -192,7 +195,7 @@ func includeInstance(r *FoundationDBClusterReconciler, context ctx.Context, clus
 
 	processGroups := make([]*fdbtypes.ProcessGroupStatus, 0, len(cluster.Status.ProcessGroups))
 	for _, processGroup := range cluster.Status.ProcessGroups {
-		if processGroup.Remove {
+		if processGroup.Remove && removedProcessGroups[processGroup.ProcessGroupID] {
 			addresses = append(addresses, processGroup.Addresses...)
 			hasStatusUpdate = true
 		} else {