From c754100dedb02d8d15b67edcec472e55162fb30a Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 18 Dec 2024 18:05:04 -0800 Subject: [PATCH 1/4] add E2Es --- cmd/hubagent/workload/setup.go | 3 +- .../controller.go | 4 +- .../controller_test.go | 3 +- .../suite_test.go | 3 +- test/e2e/actuals_test.go | 9 + test/e2e/enveloped_object_placement_test.go | 4 +- test/e2e/join_and_leave_test.go | 2 +- test/e2e/placement_cro_test.go | 14 +- test/e2e/placement_eviction_test.go | 1271 ++++++++++++++++- test/e2e/placement_pickall_test.go | 22 +- test/e2e/placement_pickfixed_test.go | 6 +- test/e2e/placement_pickn_test.go | 22 +- test/e2e/placement_ro_test.go | 16 +- .../e2e/placement_selecting_resources_test.go | 24 +- test/e2e/rollout_test.go | 18 +- test/e2e/scheduler_watchers_test.go | 46 +- test/e2e/setup_test.go | 6 +- test/e2e/taint_toleration_test.go | 65 +- test/e2e/utils_test.go | 48 +- test/e2e/webhook_test.go | 4 +- 20 files changed, 1450 insertions(+), 140 deletions(-) diff --git a/cmd/hubagent/workload/setup.go b/cmd/hubagent/workload/setup.go index 2d2f8b422..be18407cc 100644 --- a/cmd/hubagent/workload/setup.go +++ b/cmd/hubagent/workload/setup.go @@ -229,7 +229,8 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager, } klog.Info("Setting up cluster resource placement eviction controller") if err := (&clusterresourceplacementeviction.Reconciler{ - Client: mgr.GetClient(), + Client: mgr.GetClient(), + UncachedReader: mgr.GetAPIReader(), }).SetupWithManager(mgr); err != nil { klog.ErrorS(err, "Unable to set up cluster resource placement eviction controller") return err diff --git a/pkg/controllers/clusterresourceplacementeviction/controller.go b/pkg/controllers/clusterresourceplacementeviction/controller.go index 69d39133e..370a5fb2f 100644 --- a/pkg/controllers/clusterresourceplacementeviction/controller.go +++ b/pkg/controllers/clusterresourceplacementeviction/controller.go @@ -30,6 +30,8 @@ import ( // Reconciler reconciles a ClusterResourcePlacementEviction object. type Reconciler struct { client.Client + // UncachedReader is only used to read disruption budget objects directly from the API server to ensure we can enforce the disruption budget for eviction. + UncachedReader client.Reader } // Reconcile triggers a single eviction reconcile round. @@ -185,7 +187,7 @@ func (r *Reconciler) executeEviction(ctx context.Context, validationResult *evic } var db placementv1beta1.ClusterResourcePlacementDisruptionBudget - if err := r.Client.Get(ctx, types.NamespacedName{Name: crp.Name}, &db); err != nil { + if err := r.UncachedReader.Get(ctx, types.NamespacedName{Name: crp.Name}, &db); err != nil { if k8serrors.IsNotFound(err) { if err = r.deleteClusterResourceBinding(ctx, evictionTargetBinding); err != nil { return err diff --git a/pkg/controllers/clusterresourceplacementeviction/controller_test.go b/pkg/controllers/clusterresourceplacementeviction/controller_test.go index 60149851e..55e89f433 100644 --- a/pkg/controllers/clusterresourceplacementeviction/controller_test.go +++ b/pkg/controllers/clusterresourceplacementeviction/controller_test.go @@ -624,7 +624,8 @@ func TestExecuteEviction(t *testing.T) { WithObjects(objects...). Build() r := Reconciler{ - Client: fakeClient, + Client: fakeClient, + UncachedReader: fakeClient, } gotErr := r.executeEviction(ctx, tc.validationResult, tc.eviction) gotExecutedCondition := tc.eviction.GetCondition(string(placementv1beta1.PlacementEvictionConditionTypeExecuted)) diff --git a/pkg/controllers/clusterresourceplacementeviction/suite_test.go b/pkg/controllers/clusterresourceplacementeviction/suite_test.go index 695d0a711..c12d53c3e 100644 --- a/pkg/controllers/clusterresourceplacementeviction/suite_test.go +++ b/pkg/controllers/clusterresourceplacementeviction/suite_test.go @@ -81,7 +81,8 @@ var _ = BeforeSuite(func() { Expect(err).Should(Succeed()) err = (&Reconciler{ - Client: k8sClient, + Client: k8sClient, + UncachedReader: mgr.GetAPIReader(), }).SetupWithManager(mgr) Expect(err).Should(Succeed()) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index cf5ea216e..5767dfc80 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -921,6 +921,15 @@ func crpEvictionRemovedActual(crpEvictionName string) func() error { } } +func crpDisruptionBudgetRemovedActual(crpDisruptionBudgetName string) func() error { + return func() error { + if err := hubClient.Get(ctx, types.NamespacedName{Name: crpDisruptionBudgetName}, &placementv1alpha1.ClusterResourcePlacementDisruptionBudget{}); !errors.IsNotFound(err) { + return fmt.Errorf("CRP disruption budget still exists or an unexpected error occurred: %w", err) + } + return nil + } +} + func validateCRPSnapshotRevisions(crpName string, wantPolicySnapshotRevision, wantResourceSnapshotRevision int) error { matchingLabels := client.MatchingLabels{placementv1beta1.CRPTrackingLabel: crpName} diff --git a/test/e2e/enveloped_object_placement_test.go b/test/e2e/enveloped_object_placement_test.go index a5408ba63..eb5310642 100644 --- a/test/e2e/enveloped_object_placement_test.go +++ b/test/e2e/enveloped_object_placement_test.go @@ -165,7 +165,7 @@ var _ = Describe("placing wrapped resources using a CRP", func() { AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -324,7 +324,7 @@ var _ = Describe("placing wrapped resources using a CRP", func() { AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) }) diff --git a/test/e2e/join_and_leave_test.go b/test/e2e/join_and_leave_test.go index da714b071..2830ec914 100644 --- a/test/e2e/join_and_leave_test.go +++ b/test/e2e/join_and_leave_test.go @@ -122,7 +122,7 @@ var _ = Describe("Test member cluster join and leave flow", Ordered, Serial, fun AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) diff --git a/test/e2e/placement_cro_test.go b/test/e2e/placement_cro_test.go index ab67e5dac..9fba4ce63 100644 --- a/test/e2e/placement_cro_test.go +++ b/test/e2e/placement_cro_test.go @@ -77,7 +77,7 @@ var _ = Describe("creating clusterResourceOverride (selecting all clusters) to o AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting clusterResourceOverride %s", croName)) cleanupClusterResourceOverride(croName) @@ -205,7 +205,7 @@ var _ = Describe("creating clusterResourceOverride with multiple jsonPatchOverri AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting clusterResourceOverride %s", croName)) cleanupClusterResourceOverride(croName) @@ -341,7 +341,7 @@ var _ = Describe("creating clusterResourceOverride with different rules for each AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting clusterResourceOverride %s", croName)) cleanupClusterResourceOverride(croName) @@ -417,7 +417,7 @@ var _ = Describe("creating clusterResourceOverride with different rules for each AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting clusterResourceOverride %s", croName)) cleanupClusterResourceOverride(croName) @@ -494,7 +494,7 @@ var _ = Describe("creating clusterResourceOverride with incorrect path", Ordered AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting clusterResourceOverride %s", croName)) cleanupClusterResourceOverride(croName) @@ -564,7 +564,7 @@ var _ = Describe("creating clusterResourceOverride with and resource becomes inv AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting clusterResourceOverride %s", croName)) cleanupClusterResourceOverride(croName) @@ -653,7 +653,7 @@ var _ = Describe("creating clusterResourceOverride with delete rules for one clu AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting clusterResourceOverride %s", croName)) cleanupClusterResourceOverride(croName) diff --git a/test/e2e/placement_eviction_test.go b/test/e2e/placement_eviction_test.go index 313a98cbd..8d7b2ede6 100644 --- a/test/e2e/placement_eviction_test.go +++ b/test/e2e/placement_eviction_test.go @@ -11,17 +11,905 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" "go.goms.io/fleet/pkg/utils/condition" - "go.goms.io/fleet/test/e2e/framework" testutilseviction "go.goms.io/fleet/test/utils/eviction" ) +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickFixed CRP, invalid eviction denied - No PDB specified", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickFixedPlacementType, + ClusterNames: allMemberClusterNames, + }, + ResourceSelectors: workResourceSelector(), + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: false, Msg: condition.EvictionInvalidPickFixedCRPMessage}, + nil) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint cluster before eviction - No PDB specified", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) - taintClusterNames := []string{memberCluster1EastProdName} + var taintClusterNames, noTaintClusterNames []string + + BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName} + noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + + By("creating work resources") + createWorkResources() + + // Create the CRP. + createCRP(crpName) + }) + + AfterAll(func() { + // Remove taint from member cluster 1. + removeTaintsFromMemberClusters(taintClusterNames) + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("add taint to member cluster 1", func() { + addTaintsToMemberClusters(taintClusterNames, buildTaints(taintClusterNames)) + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: taintClusterNames[0], + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: condition.EvictionAllowedNoPDBMessage}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure no resources exist on evicted member cluster with taint", func() { + taintClusters := buildClustersFromNames(taintClusterNames) + for _, cluster := range taintClusters { + resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) + Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") + } + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noTaintClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the selected clusters with no taint", func() { + noTaintClusters := buildClustersFromNames(noTaintClusterNames) + for _, cluster := range noTaintClusters { + resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) + Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") + } + }) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint specified, evicted cluster is picked again by scheduler - No PDB specified", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + createCRP(crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: condition.EvictionAllowedNoPDBMessage}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure evicted cluster is picked again by scheduler & update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Integer, eviction denied due to misconfigured PDB", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + createCRP(crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: int32(len(allMemberClusterNames)), + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: condition.EvictionBlockedMisconfiguredPDBSpecifiedMessage}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + createCRP(crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "100%", + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: condition.EvictionBlockedMisconfiguredPDBSpecifiedMessage}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + createCRP(crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "100%", + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: condition.EvictionBlockedMisconfiguredPDBSpecifiedMessage}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + createCRP(crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: int32(len(allMemberClusterNames)), + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources in all but one cluster, eviction allowed", Ordered, Serial, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + var taintClusterNames, noTaintClusterNames []string + + BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName} + noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + + By("creating work resources") + createWorkResources() + + // Create the CRP. + createCRP(crpName) + }) + + AfterAll(func() { + removeTaintsFromMemberClusters(taintClusterNames) + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: int32(len(allMemberClusterNames)) - 1, + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("add taint to member cluster 1", func() { + addTaintsToMemberClusters(taintClusterNames, buildTaints(taintClusterNames)) + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: fmt.Sprintf(condition.EvictionAllowedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure no resources exist on evicted member cluster with taint", func() { + taintClusters := buildClustersFromNames(taintClusterNames) + for _, cluster := range taintClusters { + resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) + Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") + } + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noTaintClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the selected clusters with no taint", func() { + noTaintClusters := buildClustersFromNames(noTaintClusterNames) + for _, cluster := range noTaintClusters { + resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) + Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") + } + }) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(int32(len(allMemberClusterNames))), + }, + ResourceSelectors: workResourceSelector(), + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 0, + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + var taintClusterNames, noTaintClusterNames []string + + BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName} + noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + + By("creating work resources") + createWorkResources() + + // Create the CRP. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(int32(len(allMemberClusterNames))), + }, + ResourceSelectors: workResourceSelector(), + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) + }) + + AfterAll(func() { + removeTaintsFromMemberClusters(taintClusterNames) + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("add taint to member cluster 1", func() { + addTaintsToMemberClusters(taintClusterNames, buildTaints(taintClusterNames)) + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: fmt.Sprintf(condition.EvictionAllowedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure no resources exist on evicted member cluster with taint", func() { + taintClusters := buildClustersFromNames(taintClusterNames) + for _, cluster := range taintClusters { + resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) + Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") + } + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noTaintClusterNames, taintClusterNames, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the selected clusters with no taint", func() { + noTaintClusters := buildClustersFromNames(noTaintClusterNames) + for _, cluster := range noTaintClusters { + resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) + Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") + } + }) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + var taintClusterNames, noTaintClusterNames []string + + BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName} + noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + + By("creating work resources") + createWorkResources() + + // Create the CRP. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(int32(len(allMemberClusterNames))), + }, + ResourceSelectors: workResourceSelector(), + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) + }) + + AfterAll(func() { + removeTaintsFromMemberClusters(taintClusterNames) + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("add taint to member cluster 1", func() { + addTaintsToMemberClusters(taintClusterNames, buildTaints(taintClusterNames)) + }) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: fmt.Sprintf("%d%%", 100/len(allMemberClusterNames)), + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: fmt.Sprintf(condition.EvictionAllowedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure no resources exist on evicted member cluster with taint", func() { + taintClusters := buildClustersFromNames(taintClusterNames) + for _, cluster := range taintClusters { + resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) + Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") + } + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noTaintClusterNames, taintClusterNames, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the selected clusters with no taint", func() { + noTaintClusters := buildClustersFromNames(noTaintClusterNames) + for _, cluster := range noTaintClusters { + resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) + Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") + } + }) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) BeforeAll(func() { By("creating work resources") @@ -36,6 +924,196 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus Finalizers: []string{customDeletionBlockerFinalizer}, }, Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(int32(len(allMemberClusterNames))), + }, + ResourceSelectors: workResourceSelector(), + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "0%", + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(int32(len(allMemberClusterNames))), + }, + ResourceSelectors: workResourceSelector(), + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: int32(len(allMemberClusterNames)), + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + var taintClusterNames, noTaintClusterNames []string + + BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName} + noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + + By("creating work resources") + createWorkResources() + + // Create the CRP. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(int32(len(allMemberClusterNames))), + }, ResourceSelectors: workResourceSelector(), }, } @@ -43,10 +1121,10 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus }) AfterAll(func() { - // Remove taint from member cluster 1. removeTaintsFromMemberClusters(taintClusterNames) - ensureCRPEvictionDeletion(crpEvictionName) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update cluster resource placement status as expected", func() { @@ -56,6 +1134,24 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: int32(len(allMemberClusterNames)) - 1, + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + It("add taint to member cluster 1", func() { addTaintsToMemberClusters(taintClusterNames, buildTaints(taintClusterNames)) }) @@ -67,7 +1163,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus }, Spec: placementv1beta1.PlacementEvictionSpec{ PlacementName: crpName, - ClusterName: taintClusterNames[0], + ClusterName: memberCluster1EastProdName, }, } Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) @@ -77,37 +1173,41 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( ctx, hubClient, crpEvictionName, &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, - &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: condition.EvictionAllowedNoPDBMessage}) + &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: fmt.Sprintf(condition.EvictionAllowedPDBSpecifiedMessageFmt, 3, 3)}) Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") }) It("should ensure no resources exist on evicted member cluster with taint", func() { - unSelectedClusters := []*framework.Cluster{memberCluster1EastProd} - for _, cluster := range unSelectedClusters { + taintClusters := buildClustersFromNames(taintClusterNames) + for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") } }) It("should update cluster resource placement status as expected", func() { - crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), []string{memberCluster2EastCanaryName, memberCluster3WestProdName}, nil, "0") + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noTaintClusterNames, taintClusterNames, "0") Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") }) It("should place resources on the selected clusters with no taint", func() { - targetClusters := []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd} - for _, cluster := range targetClusters { + noTaintClusters := buildClustersFromNames(noTaintClusterNames) + for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") } }) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint specified, evicted cluster is picked again by scheduler - No PDB specified", Ordered, Serial, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + var taintClusterNames, noTaintClusterNames []string BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName} + noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + By("creating work resources") createWorkResources() @@ -120,6 +1220,10 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint s Finalizers: []string{customDeletionBlockerFinalizer}, }, Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(int32(len(allMemberClusterNames))), + }, ResourceSelectors: workResourceSelector(), }, } @@ -127,8 +1231,10 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint s }) AfterAll(func() { - ensureCRPEvictionDeletion(crpEvictionName) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + removeTaintsFromMemberClusters(taintClusterNames) + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update cluster resource placement status as expected", func() { @@ -138,6 +1244,28 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint s It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + It("add taint to member cluster 1", func() { + addTaintsToMemberClusters(taintClusterNames, buildTaints(taintClusterNames)) + }) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: fmt.Sprintf("%d%%", (len(allMemberClusterNames)-1)*100/len(allMemberClusterNames)), + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + It("create cluster resource placement eviction targeting member cluster 1", func() { crpe := &placementv1beta1.ClusterResourcePlacementEviction{ ObjectMeta: metav1.ObjectMeta{ @@ -155,14 +1283,123 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint s crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( ctx, hubClient, crpEvictionName, &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, - &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: condition.EvictionAllowedNoPDBMessage}) + &testutilseviction.IsExecutedEviction{IsExecuted: true, Msg: fmt.Sprintf(condition.EvictionAllowedPDBSpecifiedMessageFmt, 3, 3)}) Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") }) - It("should ensure evicted cluster is picked again by scheduler & update cluster resource placement status as expected", func() { + It("should ensure no resources exist on evicted member cluster with taint", func() { + taintClusters := buildClustersFromNames(taintClusterNames) + for _, cluster := range taintClusters { + resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) + Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") + } + }) + + It("should update cluster resource placement status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noTaintClusterNames, taintClusterNames, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should place resources on the selected clusters with no taint", func() { + noTaintClusters := buildClustersFromNames(noTaintClusterNames) + for _, cluster := range noTaintClusters { + resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) + Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") + } + }) +}) + +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) + + BeforeAll(func() { + By("creating work resources") + createWorkResources() + + // Create the CRP. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.ClusterResourcePlacementSpec{ + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickNPlacementType, + NumberOfClusters: ptr.To(int32(len(allMemberClusterNames))), + }, + ResourceSelectors: workResourceSelector(), + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) + }) + + AfterAll(func() { + ensureCRPEvictionDeleted(crpEvictionName) + ensureCRPDisruptionBudgetDeleted(crpName) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update cluster resource placement status as expected", func() { crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") }) It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + + It("create cluster resource placement disruption budget to block eviction", func() { + crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementDisruptionBudgetSpec{ + MinAvailable: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "100%", + }, + }, + } + Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName) + + ensureCRPDisruptionBudgetExists := ensureCRPDisruptionBudgetExists(crpName) + Eventually(ensureCRPDisruptionBudgetExists, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to ensure CRP Disruption Budget exists") + }) + + It("create cluster resource placement eviction targeting member cluster 1", func() { + crpe := &placementv1beta1.ClusterResourcePlacementEviction{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpEvictionName, + }, + Spec: placementv1beta1.PlacementEvictionSpec{ + PlacementName: crpName, + ClusterName: memberCluster1EastProdName, + }, + } + Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name) + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should update cluster resource placement eviction status as expected", func() { + crpEvictionStatusUpdatedActual := testutilseviction.StatusUpdatedActual( + ctx, hubClient, crpEvictionName, + &testutilseviction.IsValidEviction{IsValid: true, Msg: condition.EvictionValidMessage}, + &testutilseviction.IsExecutedEviction{IsExecuted: false, Msg: fmt.Sprintf(condition.EvictionBlockedPDBSpecifiedMessageFmt, 3, 3)}) + Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") + }) + + It("should ensure cluster resource placement status is unchanged", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + }) + + It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) diff --git a/test/e2e/placement_pickall_test.go b/test/e2e/placement_pickall_test.go index d569332c5..9f099be5d 100644 --- a/test/e2e/placement_pickall_test.go +++ b/test/e2e/placement_pickall_test.go @@ -56,7 +56,7 @@ var _ = Describe("placing resources using a CRP with no placement policy specifi It("should place the resources on all member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -101,7 +101,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func It("should place the resources on all member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -171,7 +171,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster1EastProd}) }) }) @@ -287,7 +287,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd}) }) }) @@ -354,7 +354,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func It("should not place resources on any cluster", checkIfRemovedWorkResourcesFromAllMemberClusters) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, nil) + ensureCRPAndRelatedResourcesDeleted(crpName, nil) }) }) @@ -428,7 +428,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd}) }) }) @@ -581,7 +581,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd}) }) }) @@ -656,7 +656,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func It("should not place resources on any cluster", checkIfRemovedWorkResourcesFromAllMemberClusters) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, nil) + ensureCRPAndRelatedResourcesDeleted(crpName, nil) }) }) @@ -732,7 +732,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster2EastCanary}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster2EastCanary}) }) }) @@ -912,7 +912,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster1EastProd}) }) }) @@ -985,7 +985,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func It("should not place resources on any cluster", checkIfRemovedWorkResourcesFromAllMemberClusters) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, nil) + ensureCRPAndRelatedResourcesDeleted(crpName, nil) }) }) }) diff --git a/test/e2e/placement_pickfixed_test.go b/test/e2e/placement_pickfixed_test.go index 5ee5c8058..1b1f38f7d 100644 --- a/test/e2e/placement_pickfixed_test.go +++ b/test/e2e/placement_pickfixed_test.go @@ -64,7 +64,7 @@ var _ = Describe("placing resources using a CRP of PickFixed placement type", fu }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster1EastProd}) }) }) @@ -131,7 +131,7 @@ var _ = Describe("placing resources using a CRP of PickFixed placement type", fu }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster2EastCanary}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster2EastCanary}) }) }) @@ -177,7 +177,7 @@ var _ = Describe("placing resources using a CRP of PickFixed placement type", fu }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{}) }) }) }) diff --git a/test/e2e/placement_pickn_test.go b/test/e2e/placement_pickn_test.go index b03e62d5a..6e5abaafc 100644 --- a/test/e2e/placement_pickn_test.go +++ b/test/e2e/placement_pickn_test.go @@ -67,7 +67,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd}) }) }) @@ -139,7 +139,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd}) }) }) @@ -207,7 +207,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd, memberCluster2EastCanary}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd, memberCluster2EastCanary}) }) }) @@ -285,7 +285,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary}) }) }) @@ -405,7 +405,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd, memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster1EastProd, memberCluster3WestProd}) }) }) @@ -479,7 +479,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary}) }) }) @@ -549,7 +549,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, nil) + ensureCRPAndRelatedResourcesDeleted(crpName, nil) }) }) @@ -624,7 +624,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary}) }) }) @@ -708,7 +708,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd, memberCluster2EastCanary}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd, memberCluster2EastCanary}) }) }) @@ -802,7 +802,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster2EastCanary, memberCluster1EastProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster2EastCanary, memberCluster1EastProd}) }) }) @@ -915,7 +915,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd}) }) }) }) diff --git a/test/e2e/placement_ro_test.go b/test/e2e/placement_ro_test.go index 5ec4aa785..7ad9493ed 100644 --- a/test/e2e/placement_ro_test.go +++ b/test/e2e/placement_ro_test.go @@ -65,7 +65,7 @@ var _ = Describe("creating resourceOverride (selecting all clusters) to override AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting resourceOverride %s", roName)) cleanupResourceOverride(roName, roNamespace) @@ -186,7 +186,7 @@ var _ = Describe("creating resourceOverride with multiple jsonPatchOverrides to AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting resourceOverride %s", roName)) cleanupResourceOverride(roName, roNamespace) @@ -318,7 +318,7 @@ var _ = Describe("creating resourceOverride with different rules for each cluste AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting resourceOverride %s", roName)) cleanupResourceOverride(roName, roNamespace) @@ -413,7 +413,7 @@ var _ = Describe("creating resourceOverride and clusterResourceOverride, resourc AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting resourceOverride %s", roName)) cleanupResourceOverride(roName, roNamespace) @@ -494,7 +494,7 @@ var _ = Describe("creating resourceOverride with incorrect path", Ordered, func( AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting resourceOverride %s", roName)) cleanupResourceOverride(roName, roNamespace) @@ -557,7 +557,7 @@ var _ = Describe("creating resourceOverride and resource becomes invalid after o AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting resourceOverride %s", roName)) cleanupResourceOverride(roName, roNamespace) @@ -635,7 +635,7 @@ var _ = Describe("creating resourceOverride with a templated rules with cluster AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting resourceOverride %s", roName)) cleanupResourceOverride(roName, roNamespace) @@ -731,7 +731,7 @@ var _ = Describe("creating resourceOverride with delete configMap", Ordered, fun AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) By(fmt.Sprintf("deleting resourceOverride %s", roName)) cleanupResourceOverride(roName, roNamespace) diff --git a/test/e2e/placement_selecting_resources_test.go b/test/e2e/placement_selecting_resources_test.go index 4a9f880c6..6c03022b9 100644 --- a/test/e2e/placement_selecting_resources_test.go +++ b/test/e2e/placement_selecting_resources_test.go @@ -58,7 +58,7 @@ var _ = Describe("creating CRP and selecting resources by name", Ordered, func() AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -122,7 +122,7 @@ var _ = Describe("creating CRP and selecting resources by label", Ordered, func( AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -191,7 +191,7 @@ var _ = Describe("validating CRP when cluster-scoped resources become selected a AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -277,7 +277,7 @@ var _ = Describe("validating CRP when cluster-scoped resources become unselected AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -351,7 +351,7 @@ var _ = Describe("validating CRP when cluster-scoped and namespace-scoped resour AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -444,7 +444,7 @@ var _ = Describe("validating CRP when adding resources in a matching namespace", AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -531,7 +531,7 @@ var _ = Describe("validating CRP when deleting resources in a matching namespace AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -714,7 +714,7 @@ var _ = Describe("validating CRP when failed to apply resources", Ordered, func( Expect(allMemberClusters[0].KubeClient.Delete(ctx, &existingNS, &client.DeleteOptions{PropagationPolicy: &propagage})).Should(Succeed(), "Failed to delete namespace %s", existingNS.Name) By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -855,7 +855,7 @@ var _ = Describe("validating CRP when placing cluster scope resource (other than AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -961,7 +961,7 @@ var _ = Describe("validating CRP revision history allowing single revision when AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -1054,7 +1054,7 @@ var _ = Describe("validating CRP revision history allowing multiple revisions wh AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should update CRP status as expected", func() { @@ -1146,7 +1146,7 @@ var _ = Describe("validating CRP when selected resources cross the 1MB limit", O AfterAll(func() { By(fmt.Sprintf("garbage all things related to placement %s", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("check if created cluster resource snapshots are as expected", func() { diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index 7537c9dd9..8fb3253ef 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -145,7 +145,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -224,7 +224,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -307,7 +307,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -390,7 +390,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -469,7 +469,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -625,7 +625,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -701,7 +701,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) @@ -808,7 +808,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster1EastProd}) }) }) @@ -936,7 +936,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { AfterAll(func() { // Remove the custom deletion blocker finalizer from the CRP. - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) }) diff --git a/test/e2e/scheduler_watchers_test.go b/test/e2e/scheduler_watchers_test.go index ed95ec4be..c0d468177 100644 --- a/test/e2e/scheduler_watchers_test.go +++ b/test/e2e/scheduler_watchers_test.go @@ -109,7 +109,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -197,7 +197,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -303,7 +303,7 @@ var _ = Describe("responding to specific member cluster changes", func() { return nil }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete the node") - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd}) }) }) @@ -368,7 +368,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -457,7 +457,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -525,7 +525,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -589,7 +589,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -695,7 +695,7 @@ var _ = Describe("responding to specific member cluster changes", func() { return nil }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete the node") - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd}) }) }) @@ -756,7 +756,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -820,7 +820,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -885,7 +885,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -949,7 +949,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -1015,7 +1015,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -1104,7 +1104,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -1183,7 +1183,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -1266,7 +1266,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName2ForWatcherTests) }) @@ -1387,7 +1387,7 @@ var _ = Describe("responding to specific member cluster changes", func() { return nil }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete the node") - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd}) }) }) @@ -1450,7 +1450,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -1515,7 +1515,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -1602,7 +1602,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -1681,7 +1681,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) }) }) @@ -1769,7 +1769,7 @@ var _ = Describe("responding to specific member cluster changes", func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName1ForWatcherTests) ensureMemberClusterAndRelatedResourcesDeletion(fakeClusterName2ForWatcherTests) }) @@ -1866,7 +1866,7 @@ var _ = Describe("responding to specific member cluster changes", func() { return nil }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete the node") - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster3WestProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster3WestProd}) }) }) }) diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index a7b3e178d..546240053 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -95,6 +95,7 @@ var ( allMemberClusters []*framework.Cluster allMemberClusterNames = []string{} + allMemberClustersMap = map[string]*framework.Cluster{} ) var ( @@ -308,8 +309,9 @@ func beforeSuiteForAllProcesses() { once.Do(func() { // Set these arrays only once; this is necessary as for the first spawned Ginkgo process, // the `beforeSuiteForAllProcesses` function is called twice. - for _, cluster := range allMemberClusters { - allMemberClusterNames = append(allMemberClusterNames, cluster.ClusterName) + for i := range allMemberClusters { + allMemberClusterNames = append(allMemberClusterNames, allMemberClusters[i].ClusterName) + allMemberClustersMap[allMemberClusters[i].ClusterName] = allMemberClusters[i] } }) } diff --git a/test/e2e/taint_toleration_test.go b/test/e2e/taint_toleration_test.go index 35c3e69f4..7a4351089 100644 --- a/test/e2e/taint_toleration_test.go +++ b/test/e2e/taint_toleration_test.go @@ -14,7 +14,6 @@ import ( "k8s.io/utils/ptr" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/test/e2e/framework" ) var _ = Describe("placing resource using a cluster resource placement with pickFixed placement policy specified, taint clusters, pick all specified clusters", Serial, Ordered, func() { @@ -55,16 +54,18 @@ var _ = Describe("placing resource using a cluster resource placement with pickF AfterAll(func() { // Remove taint from all member clusters. removeTaintsFromMemberClusters(allMemberClusterNames) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) var _ = Describe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, update cluster resource placement with tolerations", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) - taintClusterNames := []string{memberCluster1EastProdName, memberCluster2EastCanaryName} - selectedClusterNames := []string{memberCluster3WestProdName} + var taintClusterNames, noTaintClusterNames []string BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName, memberCluster2EastCanaryName} + noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + // Create the resources. createWorkResources() // Add taint to member clusters 1, 2. @@ -92,21 +93,24 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) It("should update cluster resource placement status as expected", func() { - crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), selectedClusterNames, nil, "0") + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noTaintClusterNames, nil, "0") Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") }) It("should ensure no resources exist on member clusters with taint", func() { - unSelectedClusters := []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary} - for _, cluster := range unSelectedClusters { + taintClusters := buildClustersFromNames(taintClusterNames) + for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") } }) It("should place resources on the selected cluster without taint", func() { - resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(memberCluster3WestProd) - Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on selected cluster") + noTaintClusters := buildClustersFromNames(noTaintClusterNames) + for _, cluster := range noTaintClusters { + resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) + Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on selected cluster") + } }) It("should update cluster resource placement spec with tolerations for tainted cluster", func() { @@ -124,16 +128,18 @@ var _ = Describe("placing resources using a cluster resource placement with no p AfterAll(func() { // Remove taint from member cluster 1,2. removeTaintsFromMemberClusters(taintClusterNames) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) var _ = Describe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, remove taints from cluster, all cluster should be picked", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) - taintClusterNames := []string{memberCluster1EastProdName, memberCluster2EastCanaryName} - selectedClusterNames := []string{memberCluster3WestProdName} + var taintClusterNames, noTaintClusterNames []string BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName, memberCluster2EastCanaryName} + noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + // Create the resources. createWorkResources() // Add taint to member clusters 1, 2. @@ -161,21 +167,24 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) It("should update cluster resource placement status as expected", func() { - crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), selectedClusterNames, nil, "0") + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noTaintClusterNames, nil, "0") Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") }) It("should ensure no resources exist on member clusters with taint", func() { - unSelectedClusters := []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary} - for _, cluster := range unSelectedClusters { + taintClusters := buildClustersFromNames(taintClusterNames) + for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") } }) It("should place resources on the selected cluster without taint", func() { - resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(memberCluster3WestProd) - Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on selected cluster") + noTaintClusters := buildClustersFromNames(noTaintClusterNames) + for _, cluster := range noTaintClusters { + resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) + Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on selected cluster") + } }) It("should remove taints from member clusters", func() { @@ -191,16 +200,19 @@ var _ = Describe("placing resources using a cluster resource placement with no p It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) AfterAll(func() { - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) var _ = Describe("picking N clusters with affinities and topology spread constraints, taint clusters, create cluster resource placement with toleration for one cluster", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) - taintClusterNames := []string{memberCluster1EastProdName, memberCluster2EastCanaryName} - tolerateClusterNames := []string{memberCluster1EastProdName} + var taintClusterNames, tolerateClusterNames, unSelectedClusterNames []string BeforeAll(func() { + taintClusterNames = []string{memberCluster1EastProdName, memberCluster2EastCanaryName} + tolerateClusterNames = []string{memberCluster1EastProdName} + unSelectedClusterNames = []string{memberCluster2EastCanaryName} + // Create the resources. createWorkResources() // Add taint to member cluster 1, 2. @@ -261,20 +273,21 @@ var _ = Describe("picking N clusters with affinities and topology spread constra }) It("should update cluster resource placement status as expected", func() { - statusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), []string{memberCluster1EastProdName}, []string{memberCluster2EastCanaryName}, "0") + // we choose two clusters using a label. + statusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), tolerateClusterNames, unSelectedClusterNames, "0") Eventually(statusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") }) It("should place resources on the selected clusters with tolerated taint", func() { - targetClusters := []*framework.Cluster{memberCluster1EastProd} - for _, cluster := range targetClusters { + tolerateClusters := buildClustersFromNames(tolerateClusterNames) + for _, cluster := range tolerateClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") } }) It("should ensure no resources exist on member clusters with untolerated taint", func() { - unSelectedClusters := []*framework.Cluster{memberCluster2EastCanary} + unSelectedClusters := buildClustersFromNames(unSelectedClusterNames) for _, cluster := range unSelectedClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -284,7 +297,7 @@ var _ = Describe("picking N clusters with affinities and topology spread constra AfterAll(func() { // Remove taint from member cluster 1, 2. removeTaintsFromMemberClusters(taintClusterNames) - ensureCRPAndRelatedResourcesDeletion(crpName, []*framework.Cluster{memberCluster1EastProd}) + ensureCRPAndRelatedResourcesDeleted(crpName, buildClustersFromNames(tolerateClusterNames)) }) }) @@ -336,6 +349,6 @@ var _ = Describe("picking all clusters using pickAll placement policy, add taint AfterAll(func() { // Remove taint from member cluster 1. removeTaintsFromMemberClusters(taintClusterNames) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index e552457f9..00416e9a3 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -889,7 +889,7 @@ func createClusterResourceOverrides(number int) { } } -func ensureCRPAndRelatedResourcesDeletion(crpName string, memberClusters []*framework.Cluster) { +func ensureCRPAndRelatedResourcesDeleted(crpName string, memberClusters []*framework.Cluster) { // Delete the CRP. crp := &placementv1beta1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ @@ -917,7 +917,7 @@ func ensureCRPAndRelatedResourcesDeletion(crpName string, memberClusters []*fram cleanupWorkResources() } -func ensureCRPEvictionDeletion(crpEvictionName string) { +func ensureCRPEvictionDeleted(crpEvictionName string) { crpe := &placementv1beta1.ClusterResourcePlacementEviction{ ObjectMeta: metav1.ObjectMeta{ Name: crpEvictionName, @@ -928,6 +928,28 @@ func ensureCRPEvictionDeletion(crpEvictionName string) { Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "CRP eviction still exists") } +func ensureCRPDisruptionBudgetExists(crpDisruptionBudgetName string) func() bool { + return func() bool { + crpdb := &placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpDisruptionBudgetName, + }, + } + return !k8serrors.IsNotFound(hubClient.Get(ctx, types.NamespacedName{Name: crpdb.Name}, crpdb)) + } +} + +func ensureCRPDisruptionBudgetDeleted(crpDisruptionBudgetName string) { + crpdb := &placementv1beta1.ClusterResourcePlacementDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpDisruptionBudgetName, + }, + } + Expect(hubClient.Delete(ctx, crpdb)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{}), "Failed to delete CRP disruption budget") + removedActual := crpDisruptionBudgetRemovedActual(crpDisruptionBudgetName) + Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "CRP disruption budget still exists") +} + // verifyWorkPropagationAndMarkAsAvailable verifies that works derived from a specific CPR have been created // for a specific cluster, and marks these works in the specific member cluster's // reserved namespace as applied and available. @@ -1101,6 +1123,28 @@ func updateCRPWithTolerations(tolerations []placementv1beta1.Toleration) { }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement with tolerations %s", crpName) } +func buildClusterNamesWithoutTaints(taintClusterNames []string) []string { + taintClusterNameMap := make(map[string]bool) + for i := range taintClusterNames { + taintClusterNameMap[taintClusterNames[i]] = true + } + var noTaintClustersNames []string + for i := range allMemberClusterNames { + if !taintClusterNameMap[allMemberClusterNames[i]] { + noTaintClustersNames = append(noTaintClustersNames, allMemberClusterNames[i]) + } + } + return noTaintClustersNames +} + +func buildClustersFromNames(clusterNames []string) []*framework.Cluster { + clusters := make([]*framework.Cluster, len(clusterNames)) + for i := range clusterNames { + clusters[i] = allMemberClustersMap[clusterNames[i]] + } + return clusters +} + func cleanupClusterResourceOverride(name string) { cro := &placementv1alpha1.ClusterResourceOverride{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/webhook_test.go b/test/e2e/webhook_test.go index b801a2689..5580fe535 100644 --- a/test/e2e/webhook_test.go +++ b/test/e2e/webhook_test.go @@ -199,7 +199,7 @@ var _ = Describe("webhook tests for CRP UPDATE operations", Ordered, func() { AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should deny update on CRP with invalid label selector", func() { @@ -319,7 +319,7 @@ var _ = Describe("webhook tests for CRP tolerations", Ordered, func() { AfterAll(func() { By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) It("should deny update on CRP with invalid toleration", func() { From ea2cbc7035b14603edb5ea457844218635cf4b8a Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 9 Jan 2025 15:37:25 -0800 Subject: [PATCH 2/4] rebase fix --- test/e2e/actuals_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 5767dfc80..0f25fdd0b 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -923,7 +923,7 @@ func crpEvictionRemovedActual(crpEvictionName string) func() error { func crpDisruptionBudgetRemovedActual(crpDisruptionBudgetName string) func() error { return func() error { - if err := hubClient.Get(ctx, types.NamespacedName{Name: crpDisruptionBudgetName}, &placementv1alpha1.ClusterResourcePlacementDisruptionBudget{}); !errors.IsNotFound(err) { + if err := hubClient.Get(ctx, types.NamespacedName{Name: crpDisruptionBudgetName}, &placementv1beta1.ClusterResourcePlacementDisruptionBudget{}); !errors.IsNotFound(err) { return fmt.Errorf("CRP disruption budget still exists or an unexpected error occurred: %w", err) } return nil From bf82b6f07304203f32dd6c553e9cf45bb2516d0e Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 15 Jan 2025 16:31:53 +0530 Subject: [PATCH 3/4] address comments --- test/e2e/placement_eviction_test.go | 81 +++++++++++++++++------------ test/e2e/setup_test.go | 2 - test/e2e/taint_toleration_test.go | 36 ++++++++----- test/e2e/utils_test.go | 22 -------- 4 files changed, 69 insertions(+), 72 deletions(-) diff --git a/test/e2e/placement_eviction_test.go b/test/e2e/placement_eviction_test.go index 8d7b2ede6..795ad4c62 100644 --- a/test/e2e/placement_eviction_test.go +++ b/test/e2e/placement_eviction_test.go @@ -16,10 +16,11 @@ import ( placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" "go.goms.io/fleet/pkg/utils/condition" + "go.goms.io/fleet/test/e2e/framework" testutilseviction "go.goms.io/fleet/test/utils/eviction" ) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickFixed CRP, invalid eviction denied - No PDB specified", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickFixed CRP, invalid eviction denied - No PDB specified", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -87,14 +88,18 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickFixed It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint cluster before eviction - No PDB specified", Ordered, Serial, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding, taint cluster before eviction - No PDB specified", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string + var taintClusters, noTaintClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName} - noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + taintClusters = []*framework.Cluster{memberCluster1EastProd} + + noTaintClusterNames = []string{memberCluster2EastCanaryName, memberCluster3WestProdName} + noTaintClusters = []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd} By("creating work resources") createWorkResources() @@ -143,7 +148,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus }) It("should ensure no resources exist on evicted member cluster with taint", func() { - taintClusters := buildClustersFromNames(taintClusterNames) for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -156,7 +160,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus }) It("should place resources on the selected clusters with no taint", func() { - noTaintClusters := buildClustersFromNames(noTaintClusterNames) for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") @@ -164,7 +167,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus }) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint specified, evicted cluster is picked again by scheduler - No PDB specified", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding, no taint specified, evicted cluster is picked again by scheduler - No PDB specified", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -217,7 +220,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint s It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Integer, eviction denied due to misconfigured PDB", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Integer, eviction denied due to misconfigured PDB", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -289,7 +292,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -361,7 +364,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -433,7 +436,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -505,14 +508,18 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources in all but one cluster, eviction allowed", Ordered, Serial, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources in all but one cluster, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string + var taintClusters, noTaintClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName} - noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + taintClusters = []*framework.Cluster{memberCluster1EastProd} + + noTaintClusterNames = []string{memberCluster2EastCanaryName, memberCluster3WestProdName} + noTaintClusters = []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd} By("creating work resources") createWorkResources() @@ -579,7 +586,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C }) It("should ensure no resources exist on evicted member cluster with taint", func() { - taintClusters := buildClustersFromNames(taintClusterNames) for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -592,7 +598,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C }) It("should place resources on the selected clusters with no taint", func() { - noTaintClusters := buildClustersFromNames(noTaintClusterNames) for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") @@ -600,7 +605,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C }) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -687,14 +692,18 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string + var taintClusters, noTaintClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName} - noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + taintClusters = []*framework.Cluster{memberCluster1EastProd} + + noTaintClusterNames = []string{memberCluster2EastCanaryName, memberCluster3WestProdName} + noTaintClusters = []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd} By("creating work resources") createWorkResources() @@ -776,7 +785,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should ensure no resources exist on evicted member cluster with taint", func() { - taintClusters := buildClustersFromNames(taintClusterNames) for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -789,7 +797,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should place resources on the selected clusters with no taint", func() { - noTaintClusters := buildClustersFromNames(noTaintClusterNames) for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") @@ -797,14 +804,18 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string + var taintClusters, noTaintClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName} - noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + taintClusters = []*framework.Cluster{memberCluster1EastProd} + + noTaintClusterNames = []string{memberCluster2EastCanaryName, memberCluster3WestProdName} + noTaintClusters = []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd} By("creating work resources") createWorkResources() @@ -886,7 +897,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should ensure no resources exist on evicted member cluster with taint", func() { - taintClusters := buildClustersFromNames(taintClusterNames) for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -899,7 +909,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should place resources on the selected clusters with no taint", func() { - noTaintClusters := buildClustersFromNames(noTaintClusterNames) for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") @@ -907,7 +916,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -1002,7 +1011,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -1089,14 +1098,18 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string + var taintClusters, noTaintClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName} - noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + taintClusters = []*framework.Cluster{memberCluster1EastProd} + + noTaintClusterNames = []string{memberCluster2EastCanaryName, memberCluster3WestProdName} + noTaintClusters = []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd} By("creating work resources") createWorkResources() @@ -1178,7 +1191,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should ensure no resources exist on evicted member cluster with taint", func() { - taintClusters := buildClustersFromNames(taintClusterNames) for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -1191,7 +1203,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should place resources on the selected clusters with no taint", func() { - noTaintClusters := buildClustersFromNames(noTaintClusterNames) for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") @@ -1199,14 +1210,18 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string + var taintClusters, noTaintClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName} - noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + taintClusters = []*framework.Cluster{memberCluster1EastProd} + + noTaintClusterNames = []string{memberCluster2EastCanaryName, memberCluster3WestProdName} + noTaintClusters = []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd} By("creating work resources") createWorkResources() @@ -1288,7 +1303,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should ensure no resources exist on evicted member cluster with taint", func() { - taintClusters := buildClustersFromNames(taintClusterNames) for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -1301,7 +1315,6 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should place resources on the selected clusters with no taint", func() { - noTaintClusters := buildClustersFromNames(noTaintClusterNames) for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") @@ -1309,7 +1322,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) }) -var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 546240053..1c1d268fd 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -95,7 +95,6 @@ var ( allMemberClusters []*framework.Cluster allMemberClusterNames = []string{} - allMemberClustersMap = map[string]*framework.Cluster{} ) var ( @@ -311,7 +310,6 @@ func beforeSuiteForAllProcesses() { // the `beforeSuiteForAllProcesses` function is called twice. for i := range allMemberClusters { allMemberClusterNames = append(allMemberClusterNames, allMemberClusters[i].ClusterName) - allMemberClustersMap[allMemberClusters[i].ClusterName] = allMemberClusters[i] } }) } diff --git a/test/e2e/taint_toleration_test.go b/test/e2e/taint_toleration_test.go index 7a4351089..c6def389e 100644 --- a/test/e2e/taint_toleration_test.go +++ b/test/e2e/taint_toleration_test.go @@ -14,9 +14,10 @@ import ( "k8s.io/utils/ptr" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/test/e2e/framework" ) -var _ = Describe("placing resource using a cluster resource placement with pickFixed placement policy specified, taint clusters, pick all specified clusters", Serial, Ordered, func() { +var _ = FDescribe("placing resource using a cluster resource placement with pickFixed placement policy specified, taint clusters, pick all specified clusters", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) BeforeAll(func() { @@ -58,13 +59,17 @@ var _ = Describe("placing resource using a cluster resource placement with pickF }) }) -var _ = Describe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, update cluster resource placement with tolerations", Serial, Ordered, func() { +var _ = FDescribe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, update cluster resource placement with tolerations", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string + var taintClusters, noTaintClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName, memberCluster2EastCanaryName} - noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + taintClusters = []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary} + + noTaintClusterNames = []string{memberCluster3WestProdName} + noTaintClusters = []*framework.Cluster{memberCluster3WestProd} // Create the resources. createWorkResources() @@ -98,7 +103,6 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) It("should ensure no resources exist on member clusters with taint", func() { - taintClusters := buildClustersFromNames(taintClusterNames) for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -106,7 +110,6 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) It("should place resources on the selected cluster without taint", func() { - noTaintClusters := buildClustersFromNames(noTaintClusterNames) for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on selected cluster") @@ -132,13 +135,17 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) }) -var _ = Describe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, remove taints from cluster, all cluster should be picked", Serial, Ordered, func() { +var _ = FDescribe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, remove taints from cluster, all cluster should be picked", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string + var taintClusters, noTaintClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName, memberCluster2EastCanaryName} - noTaintClusterNames = buildClusterNamesWithoutTaints(taintClusterNames) + taintClusters = []*framework.Cluster{memberCluster1EastProd, memberCluster2EastCanary} + + noTaintClusterNames = []string{memberCluster3WestProdName} + noTaintClusters = []*framework.Cluster{memberCluster3WestProd} // Create the resources. createWorkResources() @@ -172,7 +179,6 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) It("should ensure no resources exist on member clusters with taint", func() { - taintClusters := buildClustersFromNames(taintClusterNames) for _, cluster := range taintClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -180,7 +186,6 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) It("should place resources on the selected cluster without taint", func() { - noTaintClusters := buildClustersFromNames(noTaintClusterNames) for _, cluster := range noTaintClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on selected cluster") @@ -204,14 +209,19 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) }) -var _ = Describe("picking N clusters with affinities and topology spread constraints, taint clusters, create cluster resource placement with toleration for one cluster", Serial, Ordered, func() { +var _ = FDescribe("picking N clusters with affinities and topology spread constraints, taint clusters, create cluster resource placement with toleration for one cluster", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) var taintClusterNames, tolerateClusterNames, unSelectedClusterNames []string + var tolerateClusters, unSelectedClusters []*framework.Cluster BeforeAll(func() { taintClusterNames = []string{memberCluster1EastProdName, memberCluster2EastCanaryName} + tolerateClusterNames = []string{memberCluster1EastProdName} + tolerateClusters = []*framework.Cluster{memberCluster1EastProd} + unSelectedClusterNames = []string{memberCluster2EastCanaryName} + unSelectedClusters = []*framework.Cluster{memberCluster2EastCanary} // Create the resources. createWorkResources() @@ -279,7 +289,6 @@ var _ = Describe("picking N clusters with affinities and topology spread constra }) It("should place resources on the selected clusters with tolerated taint", func() { - tolerateClusters := buildClustersFromNames(tolerateClusterNames) for _, cluster := range tolerateClusters { resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster) Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters") @@ -287,7 +296,6 @@ var _ = Describe("picking N clusters with affinities and topology spread constra }) It("should ensure no resources exist on member clusters with untolerated taint", func() { - unSelectedClusters := buildClustersFromNames(unSelectedClusterNames) for _, cluster := range unSelectedClusters { resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") @@ -297,11 +305,11 @@ var _ = Describe("picking N clusters with affinities and topology spread constra AfterAll(func() { // Remove taint from member cluster 1, 2. removeTaintsFromMemberClusters(taintClusterNames) - ensureCRPAndRelatedResourcesDeleted(crpName, buildClustersFromNames(tolerateClusterNames)) + ensureCRPAndRelatedResourcesDeleted(crpName, tolerateClusters) }) }) -var _ = Describe("picking all clusters using pickAll placement policy, add taint to a cluster that's already selected", Serial, Ordered, func() { +var _ = FDescribe("picking all clusters using pickAll placement policy, add taint to a cluster that's already selected", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) taintClusterNames := []string{memberCluster1EastProdName} diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 00416e9a3..b44da52e1 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -1123,28 +1123,6 @@ func updateCRPWithTolerations(tolerations []placementv1beta1.Toleration) { }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement with tolerations %s", crpName) } -func buildClusterNamesWithoutTaints(taintClusterNames []string) []string { - taintClusterNameMap := make(map[string]bool) - for i := range taintClusterNames { - taintClusterNameMap[taintClusterNames[i]] = true - } - var noTaintClustersNames []string - for i := range allMemberClusterNames { - if !taintClusterNameMap[allMemberClusterNames[i]] { - noTaintClustersNames = append(noTaintClustersNames, allMemberClusterNames[i]) - } - } - return noTaintClustersNames -} - -func buildClustersFromNames(clusterNames []string) []*framework.Cluster { - clusters := make([]*framework.Cluster, len(clusterNames)) - for i := range clusterNames { - clusters[i] = allMemberClustersMap[clusterNames[i]] - } - return clusters -} - func cleanupClusterResourceOverride(name string) { cro := &placementv1alpha1.ClusterResourceOverride{ ObjectMeta: metav1.ObjectMeta{ From f3d77ce7745a62257001774c1febf5bf7def85c6 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 15 Jan 2025 16:34:11 +0530 Subject: [PATCH 4/4] remove FDescribe --- test/e2e/placement_eviction_test.go | 32 ++++++++++++++--------------- test/e2e/taint_toleration_test.go | 10 ++++----- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/test/e2e/placement_eviction_test.go b/test/e2e/placement_eviction_test.go index 795ad4c62..725699557 100644 --- a/test/e2e/placement_eviction_test.go +++ b/test/e2e/placement_eviction_test.go @@ -20,7 +20,7 @@ import ( testutilseviction "go.goms.io/fleet/test/utils/eviction" ) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickFixed CRP, invalid eviction denied - No PDB specified", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickFixed CRP, invalid eviction denied - No PDB specified", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -88,7 +88,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickFixe It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding, taint cluster before eviction - No PDB specified", Ordered, Serial, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint cluster before eviction - No PDB specified", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string @@ -167,7 +167,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding, taint clu }) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding, no taint specified, evicted cluster is picked again by scheduler - No PDB specified", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint specified, evicted cluster is picked again by scheduler - No PDB specified", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -220,7 +220,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding, no taint It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Integer, eviction denied due to misconfigured PDB", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Integer, eviction denied due to misconfigured PDB", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -292,7 +292,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MaxUnavailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -364,7 +364,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable set as Percentage, eviction denied due to misconfigured PDB", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -436,7 +436,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -508,7 +508,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources in all but one cluster, eviction allowed", Ordered, Serial, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources in all but one cluster, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string @@ -605,7 +605,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickAll }) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -692,7 +692,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CR It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string @@ -804,7 +804,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CR }) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string @@ -916,7 +916,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CR }) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MaxUnavailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -1011,7 +1011,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CR It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) @@ -1098,7 +1098,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CR It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as Integer to protect resources all clusters but one cluster, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string @@ -1210,7 +1210,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CR }) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect all clusters but one, eviction allowed", Ordered, Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string @@ -1322,7 +1322,7 @@ var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CR }) }) -var _ = FDescribe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { +var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP, PDB with MinAvailable specified as percentage to protect resources on all clusters, eviction denied", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess()) diff --git a/test/e2e/taint_toleration_test.go b/test/e2e/taint_toleration_test.go index c6def389e..a17e9d5af 100644 --- a/test/e2e/taint_toleration_test.go +++ b/test/e2e/taint_toleration_test.go @@ -17,7 +17,7 @@ import ( "go.goms.io/fleet/test/e2e/framework" ) -var _ = FDescribe("placing resource using a cluster resource placement with pickFixed placement policy specified, taint clusters, pick all specified clusters", Serial, Ordered, func() { +var _ = Describe("placing resource using a cluster resource placement with pickFixed placement policy specified, taint clusters, pick all specified clusters", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) BeforeAll(func() { @@ -59,7 +59,7 @@ var _ = FDescribe("placing resource using a cluster resource placement with pick }) }) -var _ = FDescribe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, update cluster resource placement with tolerations", Serial, Ordered, func() { +var _ = Describe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, update cluster resource placement with tolerations", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string var taintClusters, noTaintClusters []*framework.Cluster @@ -135,7 +135,7 @@ var _ = FDescribe("placing resources using a cluster resource placement with no }) }) -var _ = FDescribe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, remove taints from cluster, all cluster should be picked", Serial, Ordered, func() { +var _ = Describe("placing resources using a cluster resource placement with no placement policy specified, taint clusters, remove taints from cluster, all cluster should be picked", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) var taintClusterNames, noTaintClusterNames []string var taintClusters, noTaintClusters []*framework.Cluster @@ -209,7 +209,7 @@ var _ = FDescribe("placing resources using a cluster resource placement with no }) }) -var _ = FDescribe("picking N clusters with affinities and topology spread constraints, taint clusters, create cluster resource placement with toleration for one cluster", Serial, Ordered, func() { +var _ = Describe("picking N clusters with affinities and topology spread constraints, taint clusters, create cluster resource placement with toleration for one cluster", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) var taintClusterNames, tolerateClusterNames, unSelectedClusterNames []string var tolerateClusters, unSelectedClusters []*framework.Cluster @@ -309,7 +309,7 @@ var _ = FDescribe("picking N clusters with affinities and topology spread constr }) }) -var _ = FDescribe("picking all clusters using pickAll placement policy, add taint to a cluster that's already selected", Serial, Ordered, func() { +var _ = Describe("picking all clusters using pickAll placement policy, add taint to a cluster that's already selected", Serial, Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) taintClusterNames := []string{memberCluster1EastProdName}