add e2e for cluster-scope resource reschedule testing

Signed-off-by: changzhen <[email protected]>

Author: XiShanYongYe-Chang

test/e2e/framework/clusterresourcebinding.go

@@ -18,12 +18,17 @@ package framework
 
 import (
 	"context"
+	"strings"
 
 	"github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
 	karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
+	"github.com/karmada-io/karmada/pkg/util/helper"
+	"github.com/karmada-io/karmada/pkg/util/names"
 )
 
 // WaitClusterResourceBindingFitWith wait clusterResourceBinding fit with util timeout
@@ -36,3 +41,26 @@ func WaitClusterResourceBindingFitWith(client karmada.Interface, name string, fi
 		return fit(clusterResourceBinding)
 	}, PollTimeout, PollInterval).Should(gomega.Equal(true))
 }
+
+// ExtractTargetClustersFromCRB extract the target cluster names from clusterResourceBinding object.
+func ExtractTargetClustersFromCRB(c client.Client, resourceKind, resourceName string) []string {
+	bindingName := names.GenerateBindingName(resourceKind, resourceName)
+	binding := &workv1alpha2.ClusterResourceBinding{}
+	gomega.Eventually(func(g gomega.Gomega) (bool, error) {
+		err := c.Get(context.TODO(), client.ObjectKey{Name: bindingName}, binding)
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+
+		if !helper.IsBindingScheduled(&binding.Status) {
+			klog.Infof("The ClusterResourceBinding(%s) hasn't been scheduled.", binding.Name)
+			return false, nil
+		}
+		return true, nil
+	}, PollTimeout, PollInterval).Should(gomega.Equal(true))
+
+	targetClusterNames := make([]string, 0, len(binding.Spec.Clusters))
+	for _, cluster := range binding.Spec.Clusters {
+		targetClusterNames = append(targetClusterNames, cluster.Name)
+	}
+	klog.Infof("The ClusterResourceBinding(%s) schedule result is: %s", binding.Name, strings.Join(targetClusterNames, ","))
+	return targetClusterNames
+}

test/e2e/framework/deployment.go

@@ -19,7 +19,6 @@ package framework
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
@@ -29,11 +28,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
-	"github.com/karmada-io/karmada/pkg/util/helper"
-	"github.com/karmada-io/karmada/pkg/util/names"
 )
 
 // CreateDeployment create Deployment.
@@ -251,29 +245,6 @@ func UpdateDeploymentServiceAccountName(client kubernetes.Interface, deployment
 	})
 }
 
-// ExtractTargetClustersFrom extract the target cluster names from deployment's related resourceBinding Information.
-func ExtractTargetClustersFrom(c client.Client, deployment *appsv1.Deployment) []string {
-	bindingName := names.GenerateBindingName(deployment.Kind, deployment.Name)
-	binding := &workv1alpha2.ResourceBinding{}
-	gomega.Eventually(func(g gomega.Gomega) (bool, error) {
-		err := c.Get(context.TODO(), client.ObjectKey{Namespace: deployment.Namespace, Name: bindingName}, binding)
-		g.Expect(err).NotTo(gomega.HaveOccurred())
-
-		if !helper.IsBindingScheduled(&binding.Status) {
-			klog.Infof("The ResourceBinding(%s/%s) hasn't been scheduled.", binding.Namespace, binding.Name)
-			return false, nil
-		}
-		return true, nil
-	}, PollTimeout, PollInterval).Should(gomega.Equal(true))
-
-	targetClusterNames := make([]string, 0, len(binding.Spec.Clusters))
-	for _, cluster := range binding.Spec.Clusters {
-		targetClusterNames = append(targetClusterNames, cluster.Name)
-	}
-	klog.Infof("The ResourceBinding(%s/%s) schedule result is: %s", binding.Namespace, binding.Name, strings.Join(targetClusterNames, ","))
-	return targetClusterNames
-}
-
 // CheckDeploymentReadyStatus check the deployment status By checking the replicas
 func CheckDeploymentReadyStatus(deployment *appsv1.Deployment, wantedReplicas int32) bool {
 	if deployment.Status.ReadyReplicas == wantedReplicas &&

test/e2e/framework/resourcebinding.go

@@ -21,13 +21,18 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"strings"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
 	karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
+	"github.com/karmada-io/karmada/pkg/util/helper"
+	"github.com/karmada-io/karmada/pkg/util/names"
 )
 
 // WaitResourceBindingFitWith wait resourceBinding fit with util timeout
@@ -80,3 +85,26 @@ func WaitGracefulEvictionTasksDone(client karmada.Interface, namespace, name str
 		}, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred())
 	})
 }
+
+// ExtractTargetClustersFromRB extract the target cluster names from resourceBinding object.
+func ExtractTargetClustersFromRB(c client.Client, resourceKind, resourceNs, resourceName string) []string {
+	bindingName := names.GenerateBindingName(resourceKind, resourceName)
+	binding := &workv1alpha2.ResourceBinding{}
+	gomega.Eventually(func(g gomega.Gomega) (bool, error) {
+		err := c.Get(context.TODO(), client.ObjectKey{Namespace: resourceNs, Name: bindingName}, binding)
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+
+		if !helper.IsBindingScheduled(&binding.Status) {
+			klog.Infof("The ResourceBinding(%s/%s) hasn't been scheduled.", binding.Namespace, binding.Name)
+			return false, nil
+		}
+		return true, nil
+	}, PollTimeout, PollInterval).Should(gomega.Equal(true))
+
+	targetClusterNames := make([]string, 0, len(binding.Spec.Clusters))
+	for _, cluster := range binding.Spec.Clusters {
+		targetClusterNames = append(targetClusterNames, cluster.Name)
+	}
+	klog.Infof("The ResourceBinding(%s/%s) schedule result is: %s", binding.Namespace, binding.Name, strings.Join(targetClusterNames, ","))
+	return targetClusterNames
+}

test/e2e/suites/base/failover_test.go

@@ -99,7 +99,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 
 		ginkgo.It("deployment failover testing", func() {
 			var disabledClusters []string
-			targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+			targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 
 			ginkgo.By(fmt.Sprintf("add taint %v to the random one cluster", framework.NotReadyTaintTemplate), func() {
 				temp := numOfFailedClusters
@@ -116,7 +116,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 
 			ginkgo.By("check whether deployment of failed cluster is rescheduled to other available cluster", func() {
 				gomega.Eventually(func() int {
-					targetClusterNames = framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+					targetClusterNames = framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 					for _, targetClusterName := range targetClusterNames {
 						// the target cluster should be overwritten to another available cluster
 						if !testhelper.IsExclude(targetClusterName, disabledClusters) {
@@ -203,7 +203,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 
 		ginkgo.It("taint Cluster with NoExecute taint", func() {
 			var disabledClusters []string
-			targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+			targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 			ginkgo.By(fmt.Sprintf("add taint %v to the random one cluster", taint), func() {
 				temp := numOfFailedClusters
 				for _, targetClusterName := range targetClusterNames {
@@ -219,7 +219,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 
 			ginkgo.By("check whether deployment of taint cluster is rescheduled to other available cluster", func() {
 				gomega.Eventually(func() int {
-					targetClusterNames = framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+					targetClusterNames = framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 					for _, targetClusterName := range targetClusterNames {
 						// the target cluster should be overwritten to another available cluster
 						if !testhelper.IsExclude(targetClusterName, disabledClusters) {
@@ -312,7 +312,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 		})
 
 		ginkgo.It("application failover with purgeMode graciously when the application come back to healthy on the new cluster", func() {
-			disabledClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+			disabledClusters := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 			ginkgo.By("create an error op", func() {
 				overridePolicy = testhelper.NewOverridePolicyByOverrideRules(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
 					{
@@ -357,7 +357,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 
 			ginkgo.By("check whether the failed deployment is rescheduled to other available cluster", func() {
 				gomega.Eventually(func() int {
-					targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+					targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 					for _, targetClusterName := range targetClusterNames {
 						// the target cluster should be overwritten to another available cluster
 						if !testhelper.IsExclude(targetClusterName, disabledClusters) {
@@ -388,7 +388,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 				framework.PatchPropagationPolicy(karmadaClient, policy.Namespace, policy.Name, patch, types.JSONPatchType)
 			})
 
-			disabledClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+			disabledClusters := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 			var beginTime time.Time
 			ginkgo.By("create an error op", func() {
 				overridePolicy = testhelper.NewOverridePolicyByOverrideRules(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
@@ -503,7 +503,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 		})
 
 		ginkgo.It("application failover with purgeMode never", func() {
-			disabledClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+			disabledClusters := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 			ginkgo.By("create an error op", func() {
 				overridePolicy = testhelper.NewOverridePolicyByOverrideRules(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
 					{
@@ -544,7 +544,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
 
 			ginkgo.By("check whether the failed deployment is rescheduled to other available cluster", func() {
 				gomega.Eventually(func() int {
-					targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+					targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 					for _, targetClusterName := range targetClusterNames {
 						// the target cluster should be overwritten to another available cluster
 						if !testhelper.IsExclude(targetClusterName, disabledClusters) {

test/e2e/suites/base/fieldselector_test.go

@@ -145,7 +145,7 @@ var _ = ginkgo.Describe("propagation with fieldSelector testing", func() {
 
 		ginkgo.It("propagation with fieldSelector testing", func() {
 			ginkgo.By("check whether deployment is scheduled to clusters which meeting the fieldSelector requirements", func() {
-				targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
+				targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
 				gomega.Expect(len(targetClusterNames)).Should(gomega.Equal(1))
 				gomega.Expect(targetClusterNames[0]).Should(gomega.Equal(desiredScheduleResult))
 			})