inspect drain evictions, mc

Signed-off-by: Arvind Thirumurugan <[email protected]>

Author: Arvind Thirumurugan

test/e2e/actuals_test.go

@@ -29,12 +29,14 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1"
 	placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
 	"go.goms.io/fleet/pkg/controllers/clusterresourceplacement"
 	"go.goms.io/fleet/pkg/controllers/workapplier"
 	scheduler "go.goms.io/fleet/pkg/scheduler/framework"
 	"go.goms.io/fleet/pkg/utils/condition"
 	"go.goms.io/fleet/test/e2e/framework"
+	toolsutils "go.goms.io/fleet/tools/utils"
 )
 
 func validateWorkNamespaceOnCluster(cluster *framework.Cluster, name types.NamespacedName) error {
@@ -1266,3 +1268,35 @@ func updateRunStrategyRemovedActual(strategyName string) func() error {
 		return nil
 	}
 }
+
+func memberClusterCordonTaintAddedActual(mcName string) func() error {
+	return func() error {
+		var mc clusterv1beta1.MemberCluster
+		if err := hubClient.Get(ctx, types.NamespacedName{Name: mcName}, &mc); err != nil {
+			return fmt.Errorf("failed to get member cluster %s: %w", mcName, err)
+		}
+
+		for _, taint := range mc.Spec.Taints {
+			if taint == toolsutils.CordonTaint {
+				return nil
+			}
+		}
+		return fmt.Errorf("cordon taint not found on member cluster %s", mcName)
+	}
+}
+
+func memberClusterCordonTaintRemovedActual(mcName string) func() error {
+	return func() error {
+		var mc clusterv1beta1.MemberCluster
+		if err := hubClient.Get(ctx, types.NamespacedName{Name: mcName}, &mc); err != nil {
+			return fmt.Errorf("failed to get member cluster %s: %w", mcName, err)
+		}
+
+		for _, taint := range mc.Spec.Taints {
+			if taint == toolsutils.CordonTaint {
+				return fmt.Errorf("cordon taint found on member cluster %s", mcName)
+			}
+		}
+		return nil
+	}
+}

test/e2e/drain_tool_test.go

@@ -26,11 +26,14 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
+	"go.goms.io/fleet/pkg/utils/condition"
 	"go.goms.io/fleet/test/e2e/framework"
+	testutilseviction "go.goms.io/fleet/test/utils/eviction"
 )
 
 var _ = Describe("Drain cluster successfully", Ordered, Serial, func() {
 	crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())
+	var drainEvictions []placementv1beta1.ClusterResourcePlacementEviction
 	var drainClusters, noDrainClusters []*framework.Cluster
 	var noDrainClusterNames []string
 
@@ -61,6 +64,10 @@ var _ = Describe("Drain cluster successfully", Ordered, Serial, func() {
 	})
 
 	AfterAll(func() {
+		// remove drain evictions.
+		for _, eviction := range drainEvictions {
+			ensureCRPEvictionDeleted(eviction.Name)
+		}
 		// remove taints from member cluster 1 again to guarantee clean up of cordon taint on test failure.
 		removeTaintsFromMemberClusters([]string{memberCluster1EastProdName})
 		ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters)
@@ -75,6 +82,24 @@ var _ = Describe("Drain cluster successfully", Ordered, Serial, func() {
 
 	It("drain cluster using binary, should succeed", func() { runDrainClusterBinary(hubClusterName, memberCluster1EastProdName) })
 
+	It("should update member cluster with cordon taint", func() {
+		taintAddedActual := memberClusterCordonTaintAddedActual(memberCluster1EastProdName)
+		Eventually(taintAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cordon taint to member cluster")
+	})
+
+	It("should update drain cluster resource placement evictions status as expected", func() {
+		var fetchError error
+		drainEvictions, fetchError = fetchDrainEvictions()
+		Eventually(fetchError).Should(Succeed(), "Failed to fetch drain evictions")
+		for _, eviction := range drainEvictions {
+			crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual(
+				ctx, hubClient, eviction.Name,
+				&testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage},
+				&testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: condition.EvictionAllowedNoPDBMessage})
+			Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected")
+		}
+	})
+
 	It("should ensure no resources exist on drained clusters", func() {
 		for _, cluster := range drainClusters {
 			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
@@ -96,6 +121,11 @@ var _ = Describe("Drain cluster successfully", Ordered, Serial, func() {
 
 	It("uncordon cluster using binary", func() { runUncordonClusterBinary(hubClusterName, memberCluster1EastProdName) })
 
+	It("should remove cordon taint from member cluster", func() {
+		taintRemovedActual := memberClusterCordonTaintRemovedActual(memberCluster1EastProdName)
+		Eventually(taintRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove cordon taint from member cluster")
+	})
+
 	It("should update cluster resource placement status as expected", func() {
 		crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
 		Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
@@ -104,8 +134,9 @@ var _ = Describe("Drain cluster successfully", Ordered, Serial, func() {
 	It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
 })
 
-var _ = Describe("Drain cluster blocked - ClusterResourcePlacementDisruptionBudget blocks evictions on all clusters", Ordered, Serial, func() {
+var _ = FDescribe("Drain cluster blocked - ClusterResourcePlacementDisruptionBudget blocks evictions on all clusters", Ordered, Serial, func() {
 	crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())
+	var drainEvictions []placementv1beta1.ClusterResourcePlacementEviction
 
 	BeforeAll(func() {
 		By("creating work resources")
@@ -130,6 +161,10 @@ var _ = Describe("Drain cluster blocked - ClusterResourcePlacementDisruptionBudg
 	})
 
 	AfterAll(func() {
+		// remove drain evictions.
+		for _, eviction := range drainEvictions {
+			ensureCRPEvictionDeleted(eviction.Name)
+		}
 		// remove taints from member cluster 1 again to guarantee clean up of cordon taint on test failure.
 		removeTaintsFromMemberClusters([]string{memberCluster1EastProdName})
 		ensureCRPDisruptionBudgetDeleted(crpName)
@@ -160,6 +195,24 @@ var _ = Describe("Drain cluster blocked - ClusterResourcePlacementDisruptionBudg
 
 	It("drain cluster using binary, should fail due to CRPDB", func() { runDrainClusterBinary(hubClusterName, memberCluster1EastProdName) })
 
+	It("should update member cluster with cordon taint", func() {
+		taintAddedActual := memberClusterCordonTaintAddedActual(memberCluster1EastProdName)
+		Eventually(taintAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cordon taint to member cluster")
+	})
+
+	It("should update drain cluster resource placement evictions status as expected", func() {
+		var fetchError error
+		drainEvictions, fetchError = fetchDrainEvictions()
+		Eventually(fetchError).Should(Succeed(), "Failed to fetch drain evictions")
+		for _, eviction := range drainEvictions {
+			crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual(
+				ctx, hubClient, eviction.Name,
+				&testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage},
+				&testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 3, 3)})
+			Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected")
+		}
+	})
+
 	It("should ensure cluster resource placement status remains unchanged", func() {
 		crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
 		Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
@@ -169,6 +222,11 @@ var _ = Describe("Drain cluster blocked - ClusterResourcePlacementDisruptionBudg
 
 	It("uncordon cluster using binary", func() { runUncordonClusterBinary(hubClusterName, memberCluster1EastProdName) })
 
+	It("should remove cordon taint from member cluster", func() {
+		taintRemovedActual := memberClusterCordonTaintRemovedActual(memberCluster1EastProdName)
+		Eventually(taintRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove cordon taint from member cluster")
+	})
+
 	It("should update cluster resource placement status as expected", func() {
 		crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
 		Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
@@ -177,8 +235,9 @@ var _ = Describe("Drain cluster blocked - ClusterResourcePlacementDisruptionBudg
 	It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
 })
 
-var _ = Describe("Drain is allowed on one cluster, blocked on others - ClusterResourcePlacementDisruptionBudget blocks evictions on some clusters", Ordered, Serial, func() {
+var _ = FDescribe("Drain is allowed on one cluster, blocked on others - ClusterResourcePlacementDisruptionBudget blocks evictions on some clusters", Ordered, Serial, func() {
 	crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())
+	var drainEvictions []placementv1beta1.ClusterResourcePlacementEviction
 	var drainClusters, noDrainClusters []*framework.Cluster
 	var noDrainClusterNames []string
 
@@ -209,6 +268,10 @@ var _ = Describe("Drain is allowed on one cluster, blocked on others - ClusterRe
 	})
 
 	AfterAll(func() {
+		// remove remaining drain evictions.
+		for _, eviction := range drainEvictions {
+			ensureCRPEvictionDeleted(eviction.Name)
+		}
 		// remove taints from member clusters 1,2 again to guarantee clean up of cordon taint on test failure.
 		removeTaintsFromMemberClusters([]string{memberCluster1EastProdName, memberCluster2EastCanaryName})
 		ensureCRPDisruptionBudgetDeleted(crpName)
@@ -239,8 +302,50 @@ var _ = Describe("Drain is allowed on one cluster, blocked on others - ClusterRe
 
 	It("drain cluster using binary, should succeed", func() { runDrainClusterBinary(hubClusterName, memberCluster1EastProdName) })
 
+	It("should update member cluster with cordon taint", func() {
+		taintAddedActual := memberClusterCordonTaintAddedActual(memberCluster1EastProdName)
+		Eventually(taintAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cordon taint to member cluster")
+	})
+
+	It("should update drain cluster resource placement evictions status as expected", func() {
+		var fetchError error
+		drainEvictions, fetchError = fetchDrainEvictions()
+		Eventually(fetchError).Should(Succeed(), "Failed to fetch drain evictions")
+		for _, eviction := range drainEvictions {
+			crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual(
+				ctx, hubClient, eviction.Name,
+				&testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage},
+				&testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: fmt.Sprintf(condition.EvictionAllowedPDBSpecifiedMessageFmt, 3, 3)})
+			Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected")
+		}
+	})
+
+	It("remove drain evictions for member cluster 1", func() {
+		for _, eviction := range drainEvictions {
+			ensureCRPEvictionDeleted(eviction.Name)
+		}
+	})
+
 	It("drain cluster using binary, should fail due to CRPDB", func() { runDrainClusterBinary(hubClusterName, memberCluster2EastCanaryName) })
 
+	It("should update member cluster with cordon taint", func() {
+		taintAddedActual := memberClusterCordonTaintAddedActual(memberCluster2EastCanaryName)
+		Eventually(taintAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cordon taint to member cluster")
+	})
+
+	It("should update drain cluster resource placement evictions status as expected", func() {
+		var fetchError error
+		drainEvictions, fetchError = fetchDrainEvictions()
+		Eventually(fetchError).Should(Succeed(), "Failed to fetch drain evictions")
+		for _, eviction := range drainEvictions {
+			crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual(
+				ctx, hubClient, eviction.Name,
+				&testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage},
+				&testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 2, 2)})
+			Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected")
+		}
+	})
+
 	It("should ensure no resources exist on drained clusters", func() {
 		for _, cluster := range drainClusters {
 			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
@@ -262,8 +367,18 @@ var _ = Describe("Drain is allowed on one cluster, blocked on others - ClusterRe
 
 	It("uncordon cluster using binary", func() { runUncordonClusterBinary(hubClusterName, memberCluster1EastProdName) })
 
+	It("should remove cordon taint from member cluster", func() {
+		taintRemovedActual := memberClusterCordonTaintRemovedActual(memberCluster1EastProdName)
+		Eventually(taintRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove cordon taint from member cluster")
+	})
+
 	It("uncordon cluster using binary", func() { runUncordonClusterBinary(hubClusterName, memberCluster2EastCanaryName) })
 
+	It("should remove cordon taint from member cluster", func() {
+		taintRemovedActual := memberClusterCordonTaintRemovedActual(memberCluster2EastCanaryName)
+		Eventually(taintRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove cordon taint from member cluster")
+	})
+
 	It("should place resources on all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
 })
 
@@ -284,3 +399,11 @@ func runUncordonClusterBinary(hubClusterName, memberClusterName string) {
 	_, err := cmd.CombinedOutput()
 	Expect(err).ToNot(HaveOccurred(), "Uncordon command failed with error: %v", err)
 }
+
+func fetchDrainEvictions() ([]placementv1beta1.ClusterResourcePlacementEviction, error) {
+	var evictionList placementv1beta1.ClusterResourcePlacementEvictionList
+	if err := hubClient.List(ctx, &evictionList); err != nil {
+		return nil, err
+	}
+	return evictionList.Items, nil
+}