refactor

Author: Arvind Thirumurugan

tools/cordon-cluster/main/main.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"os"
 
 	k8errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,9 +27,16 @@ var (
 )
 
 const (
-	testEvictionNameFormat = "test-eviction-%s-%s"
+	testEvictionNameFormat      = "test-eviction-%s-%s"
+	resourceIdentifierKeyFormat = "%s/%s/%s/%s/%s"
 )
 
+type DrainCluster struct {
+	hubClient                            client.Client
+	ClusterName                          string
+	ClusterResourcePlacementResourcesMap map[string][]placementv1beta1.ResourceIdentifier
+}
+
 func main() {
 	hubClusterContext := flag.String("hubClusterContext", "", "the kubectl context for the hub cluster")
 	clusterName := flag.String("clusterName", "", "name of the cluster to cordon")
@@ -53,65 +61,43 @@ func main() {
 		log.Fatalf("failed to create hub cluster client: %v", err)
 	}
 
-	var mc clusterv1beta1.MemberCluster
-	if err = hubClient.Get(ctx, types.NamespacedName{Name: *clusterName}, &mc); err != nil {
-		log.Fatalf("failed to get member cluster %s: %v", *clusterName, err)
+	drainCluster := DrainCluster{
+		hubClient:                            hubClient,
+		ClusterName:                          *clusterName,
+		ClusterResourcePlacementResourcesMap: make(map[string][]placementv1beta1.ResourceIdentifier),
+	}
+
+	if err = drainCluster.cordonCluster(); err != nil {
+		log.Fatalf("failed to cordon member cluster %s: %v", drainCluster.ClusterName, err)
 	}
 
 	var crpList placementv1beta1.ClusterResourcePlacementList
-	if err = hubClient.List(ctx, &crpList); err != nil {
+	if err = drainCluster.hubClient.List(ctx, &crpList); err != nil {
 		log.Fatalf("failed to list cluster resource placements: %v", err)
 	}
 
 	// find all unique CRP names for which eviction needs to occur.
-	crpNameMap := make(map[string]bool)
 	for i := range crpList.Items {
-		for j := range crpList.Items[i].Status.PlacementStatuses {
-			if crpList.Items[i].Status.PlacementStatuses[j].ClusterName == *clusterName {
-				crpNameMap[crpList.Items[i].Name] = true
+		crpStatus := crpList.Items[i].Status
+		for j := range crpStatus.PlacementStatuses {
+			placementStatus := crpStatus.PlacementStatuses[j]
+			if placementStatus.ClusterName == *clusterName {
+				drainCluster.ClusterResourcePlacementResourcesMap[crpList.Items[i].Name] = resourcePropagatedByCRP(crpStatus.SelectedResources, placementStatus.FailedPlacements)
 				break
 			}
 		}
 	}
 
-	if len(crpNameMap) == 0 {
-		log.Fatalf("failed to find any CRP which has propagated resource to cluster %s", *clusterName)
-	}
-
-	// add taint to member cluster to ensure resources aren't scheduled on it.
-	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		var mc clusterv1beta1.MemberCluster
-		if err := hubClient.Get(ctx, types.NamespacedName{Name: *clusterName}, &mc); err != nil {
-			return err
-		}
-		cordonTaint := clusterv1beta1.Taint{
-			Key:    "cordon-key",
-			Value:  "cordon-value",
-			Effect: "NoSchedule",
-		}
-
-		// search to see cordonTaint already exists on the cluster.
-		for i := range mc.Spec.Taints {
-			if mc.Spec.Taints[i].Key == cordonTaint.Key {
-				return nil
-			}
-		}
-
-		// add taint to member cluster to cordon.
-		mc.Spec.Taints = append(mc.Spec.Taints, cordonTaint)
-
-		return hubClient.Update(ctx, &mc)
-	})
-
-	if err != nil {
-		log.Fatalf("failed to update member cluster with taint: %v", err)
+	if len(drainCluster.ClusterResourcePlacementResourcesMap) == 0 {
+		log.Printf("There are no resources propagated to %s from fleet using ClusterResourcePlacement resources", *clusterName)
+		os.Exit(0)
 	}
 
 	// create eviction objects for all <crpName, targetCluster>.
-	for crpName := range crpNameMap {
+	for crpName := range drainCluster.ClusterResourcePlacementResourcesMap {
 		evictionName := fmt.Sprintf(testEvictionNameFormat, crpName, *clusterName)
 
-		if err = removeExistingEviction(ctx, hubClient, evictionName); err != nil {
+		if err = removeExistingEviction(ctx, drainCluster.hubClient, evictionName); err != nil {
 			log.Fatalf("failed to remove existing eviction for CRP %s", crpName)
 		}
 
@@ -127,7 +113,7 @@ func main() {
 					ClusterName:   *clusterName,
 				},
 			}
-			return hubClient.Create(ctx, &eviction)
+			return drainCluster.hubClient.Create(ctx, &eviction)
 		})
 
 		if err != nil {
@@ -137,11 +123,11 @@ func main() {
 
 	// TODO: move isEvictionInTerminalState to a function in the pkg/utils package.
 	// wait until all evictions reach a terminal state.
-	for crpName := range crpNameMap {
+	for crpName := range drainCluster.ClusterResourcePlacementResourcesMap {
 		err = wait.ExponentialBackoffWithContext(ctx, retry.DefaultBackoff, func(ctx context.Context) (bool, error) {
 			evictionName := fmt.Sprintf(testEvictionNameFormat, crpName, *clusterName)
 			eviction := placementv1beta1.ClusterResourcePlacementEviction{}
-			if err := hubClient.Get(ctx, types.NamespacedName{Name: evictionName}, &eviction); err != nil {
+			if err := drainCluster.hubClient.Get(ctx, types.NamespacedName{Name: evictionName}, &eviction); err != nil {
 				return false, err
 			}
 			validCondition := eviction.GetCondition(string(placementv1beta1.PlacementEvictionConditionTypeValid))
@@ -160,7 +146,82 @@ func main() {
 		}
 	}
 
-	log.Printf("Issued evictions to cordon member cluster %s, please verify to ensure evictions were successfully executed", *clusterName)
+	isDrainSuccessful := true
+	// check if all evictions have been executed.
+	for crpName := range drainCluster.ClusterResourcePlacementResourcesMap {
+		evictionName := fmt.Sprintf(testEvictionNameFormat, crpName, *clusterName)
+		eviction := placementv1beta1.ClusterResourcePlacementEviction{}
+		if err := drainCluster.hubClient.Get(ctx, types.NamespacedName{Name: evictionName}, &eviction); err != nil {
+			log.Fatalf("failed to get eviction %s: %v", evictionName, err)
+		}
+		executedCondition := eviction.GetCondition(string(placementv1beta1.PlacementEvictionConditionTypeExecuted))
+		if executedCondition == nil || executedCondition.Status == metav1.ConditionFalse {
+			isDrainSuccessful = false
+			log.Fatalf("eviction %s was not executed successfully for CRP %s", evictionName, crpName)
+		}
+		log.Printf("eviction %s was executed successfully for CRP %s", evictionName, crpName)
+		// log each resource evicted by CRP.
+		for i := range drainCluster.ClusterResourcePlacementResourcesMap[crpName] {
+			resourceIdentifier := drainCluster.ClusterResourcePlacementResourcesMap[crpName][i]
+			log.Printf("evicted resource %s propagated by CRP %s", fmt.Sprintf(resourceIdentifierKeyFormat, resourceIdentifier.Group, resourceIdentifier.Version, resourceIdentifier.Kind, resourceIdentifier.Name, resourceIdentifier.Namespace), crpName)
+		}
+	}
+
+	if isDrainSuccessful {
+		log.Printf("drain was successful for cluster %s", *clusterName)
+	} else {
+		log.Printf("drain was not successful for cluster %s, some eviction were not successfully executed", *clusterName)
+	}
+}
+
+func (d *DrainCluster) cordonCluster() error {
+	// add taint to member cluster to ensure resources aren't scheduled on it.
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		var mc clusterv1beta1.MemberCluster
+		if err := d.hubClient.Get(ctx, types.NamespacedName{Name: d.ClusterName}, &mc); err != nil {
+			return err
+		}
+		cordonTaint := clusterv1beta1.Taint{
+			Key:    "cordon-key",
+			Value:  "cordon-value",
+			Effect: "NoSchedule",
+		}
+
+		// search to see cordonTaint already exists on the cluster.
+		for i := range mc.Spec.Taints {
+			if mc.Spec.Taints[i].Key == cordonTaint.Key {
+				return nil
+			}
+		}
+
+		// add taint to member cluster to cordon.
+		mc.Spec.Taints = append(mc.Spec.Taints, cordonTaint)
+
+		return d.hubClient.Update(ctx, &mc)
+	})
+}
+
+func resourcePropagatedByCRP(selectedResources []placementv1beta1.ResourceIdentifier, failedPlacements []placementv1beta1.FailedResourcePlacement) []placementv1beta1.ResourceIdentifier {
+	selectedResourcesMap := make(map[string]placementv1beta1.ResourceIdentifier, len(selectedResources))
+	for i := range selectedResources {
+		selectedResourcesMap[generateResourceIdentifierKey(selectedResources[i])] = selectedResources[i]
+	}
+	for i := range failedPlacements {
+		if failedPlacements[i].Condition.Type == placementv1beta1.WorkConditionTypeApplied {
+			delete(selectedResourcesMap, generateResourceIdentifierKey(failedPlacements[i].ResourceIdentifier))
+		}
+	}
+	resourcesPropagated := make([]placementv1beta1.ResourceIdentifier, len(selectedResourcesMap))
+	i := 0
+	for key := range selectedResourcesMap {
+		resourcesPropagated[i] = selectedResourcesMap[key]
+		i++
+	}
+	return resourcesPropagated
+}
+
+func generateResourceIdentifierKey(resourceIdentifier placementv1beta1.ResourceIdentifier) string {
+	return fmt.Sprintf(resourceIdentifierKeyFormat, resourceIdentifier.Group, resourceIdentifier.Version, resourceIdentifier.Kind, resourceIdentifier.Name, resourceIdentifier.Namespace)
 }
 
 func removeExistingEviction(ctx context.Context, client client.Client, evictionName string) error {