Skip to content

Commit ff813d9

Browse files
karmada-botkarmada-bot
authored
Merge pull request #6425 from XiShanYongYe-Chang/e2e-for-crb-reschedule
Add e2e for cluster-scope resource reschedule testing
2 parents 721b472 + ad438dc commit ff813d9

File tree

9 files changed

+269
-268
lines changed

9 files changed

+269
-268
lines changed

‎test/e2e/framework/clusterresourcebinding.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,17 @@ package framework
1818

1919
import (
2020
"context"
21+
"strings"
2122

2223
"github.com/onsi/gomega"
2324
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25+
"k8s.io/klog/v2"
26+
"sigs.k8s.io/controller-runtime/pkg/client"
2427

2528
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
2629
karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
30+
"github.com/karmada-io/karmada/pkg/util/helper"
31+
"github.com/karmada-io/karmada/pkg/util/names"
2732
)
2833

2934
// WaitClusterResourceBindingFitWith wait clusterResourceBinding fit with util timeout
@@ -36,3 +41,26 @@ func WaitClusterResourceBindingFitWith(client karmada.Interface, name string, fi
3641
return fit(clusterResourceBinding)
3742
}, PollTimeout, PollInterval).Should(gomega.Equal(true))
3843
}
44+
45+
// ExtractTargetClustersFromCRB extract the target cluster names from clusterResourceBinding object.
46+
func ExtractTargetClustersFromCRB(c client.Client, resourceKind, resourceName string) []string {
47+
bindingName := names.GenerateBindingName(resourceKind, resourceName)
48+
binding := &workv1alpha2.ClusterResourceBinding{}
49+
gomega.Eventually(func(g gomega.Gomega) (bool, error) {
50+
err := c.Get(context.TODO(), client.ObjectKey{Name: bindingName}, binding)
51+
g.Expect(err).NotTo(gomega.HaveOccurred())
52+
53+
if !helper.IsBindingScheduled(&binding.Status) {
54+
klog.Infof("The ClusterResourceBinding(%s) hasn't been scheduled.", binding.Name)
55+
return false, nil
56+
}
57+
return true, nil
58+
}, PollTimeout, PollInterval).Should(gomega.Equal(true))
59+
60+
targetClusterNames := make([]string, 0, len(binding.Spec.Clusters))
61+
for _, cluster := range binding.Spec.Clusters {
62+
targetClusterNames = append(targetClusterNames, cluster.Name)
63+
}
64+
klog.Infof("The ClusterResourceBinding(%s) schedule result is: %s", binding.Name, strings.Join(targetClusterNames, ","))
65+
return targetClusterNames
66+
}

‎test/e2e/framework/deployment.go‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package framework
1919
import (
2020
"context"
2121
"fmt"
22-
"strings"
2322

2423
"github.com/onsi/ginkgo/v2"
2524
"github.com/onsi/gomega"
@@ -29,11 +28,6 @@ import (
2928
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3029
"k8s.io/client-go/kubernetes"
3130
"k8s.io/klog/v2"
32-
"sigs.k8s.io/controller-runtime/pkg/client"
33-
34-
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
35-
"github.com/karmada-io/karmada/pkg/util/helper"
36-
"github.com/karmada-io/karmada/pkg/util/names"
3731
)
3832

3933
// CreateDeployment create Deployment.
@@ -251,29 +245,6 @@ func UpdateDeploymentServiceAccountName(client kubernetes.Interface, deployment
251245
})
252246
}
253247

254-
// ExtractTargetClustersFrom extract the target cluster names from deployment's related resourceBinding Information.
255-
func ExtractTargetClustersFrom(c client.Client, deployment *appsv1.Deployment) []string {
256-
bindingName := names.GenerateBindingName(deployment.Kind, deployment.Name)
257-
binding := &workv1alpha2.ResourceBinding{}
258-
gomega.Eventually(func(g gomega.Gomega) (bool, error) {
259-
err := c.Get(context.TODO(), client.ObjectKey{Namespace: deployment.Namespace, Name: bindingName}, binding)
260-
g.Expect(err).NotTo(gomega.HaveOccurred())
261-
262-
if !helper.IsBindingScheduled(&binding.Status) {
263-
klog.Infof("The ResourceBinding(%s/%s) hasn't been scheduled.", binding.Namespace, binding.Name)
264-
return false, nil
265-
}
266-
return true, nil
267-
}, PollTimeout, PollInterval).Should(gomega.Equal(true))
268-
269-
targetClusterNames := make([]string, 0, len(binding.Spec.Clusters))
270-
for _, cluster := range binding.Spec.Clusters {
271-
targetClusterNames = append(targetClusterNames, cluster.Name)
272-
}
273-
klog.Infof("The ResourceBinding(%s/%s) schedule result is: %s", binding.Namespace, binding.Name, strings.Join(targetClusterNames, ","))
274-
return targetClusterNames
275-
}
276-
277248
// CheckDeploymentReadyStatus check the deployment status By checking the replicas
278249
func CheckDeploymentReadyStatus(deployment *appsv1.Deployment, wantedReplicas int32) bool {
279250
if deployment.Status.ReadyReplicas == wantedReplicas &&

‎test/e2e/framework/resourcebinding.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,18 @@ import (
2121
"fmt"
2222
"reflect"
2323
"sort"
24+
"strings"
2425

2526
"github.com/onsi/ginkgo/v2"
2627
"github.com/onsi/gomega"
2728
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
"k8s.io/klog/v2"
30+
"sigs.k8s.io/controller-runtime/pkg/client"
2831

2932
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
3033
karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
34+
"github.com/karmada-io/karmada/pkg/util/helper"
35+
"github.com/karmada-io/karmada/pkg/util/names"
3136
)
3237

3338
// WaitResourceBindingFitWith wait resourceBinding fit with util timeout
@@ -80,3 +85,26 @@ func WaitGracefulEvictionTasksDone(client karmada.Interface, namespace, name str
8085
}, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred())
8186
})
8287
}
88+
89+
// ExtractTargetClustersFromRB extract the target cluster names from resourceBinding object.
90+
func ExtractTargetClustersFromRB(c client.Client, resourceKind, resourceNs, resourceName string) []string {
91+
bindingName := names.GenerateBindingName(resourceKind, resourceName)
92+
binding := &workv1alpha2.ResourceBinding{}
93+
gomega.Eventually(func(g gomega.Gomega) (bool, error) {
94+
err := c.Get(context.TODO(), client.ObjectKey{Namespace: resourceNs, Name: bindingName}, binding)
95+
g.Expect(err).NotTo(gomega.HaveOccurred())
96+
97+
if !helper.IsBindingScheduled(&binding.Status) {
98+
klog.Infof("The ResourceBinding(%s/%s) hasn't been scheduled.", binding.Namespace, binding.Name)
99+
return false, nil
100+
}
101+
return true, nil
102+
}, PollTimeout, PollInterval).Should(gomega.Equal(true))
103+
104+
targetClusterNames := make([]string, 0, len(binding.Spec.Clusters))
105+
for _, cluster := range binding.Spec.Clusters {
106+
targetClusterNames = append(targetClusterNames, cluster.Name)
107+
}
108+
klog.Infof("The ResourceBinding(%s/%s) schedule result is: %s", binding.Namespace, binding.Name, strings.Join(targetClusterNames, ","))
109+
return targetClusterNames
110+
}

‎test/e2e/suites/base/failover_test.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
9999

100100
ginkgo.It("deployment failover testing", func() {
101101
var disabledClusters []string
102-
targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
102+
targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
103103

104104
ginkgo.By(fmt.Sprintf("add taint %v to the random one cluster", framework.NotReadyTaintTemplate), func() {
105105
temp := numOfFailedClusters
@@ -116,7 +116,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
116116

117117
ginkgo.By("check whether deployment of failed cluster is rescheduled to other available cluster", func() {
118118
gomega.Eventually(func() int {
119-
targetClusterNames = framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
119+
targetClusterNames = framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
120120
for _, targetClusterName := range targetClusterNames {
121121
// the target cluster should be overwritten to another available cluster
122122
if !testhelper.IsExclude(targetClusterName, disabledClusters) {
@@ -203,7 +203,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
203203

204204
ginkgo.It("taint Cluster with NoExecute taint", func() {
205205
var disabledClusters []string
206-
targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
206+
targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
207207
ginkgo.By(fmt.Sprintf("add taint %v to the random one cluster", taint), func() {
208208
temp := numOfFailedClusters
209209
for _, targetClusterName := range targetClusterNames {
@@ -219,7 +219,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
219219

220220
ginkgo.By("check whether deployment of taint cluster is rescheduled to other available cluster", func() {
221221
gomega.Eventually(func() int {
222-
targetClusterNames = framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
222+
targetClusterNames = framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
223223
for _, targetClusterName := range targetClusterNames {
224224
// the target cluster should be overwritten to another available cluster
225225
if !testhelper.IsExclude(targetClusterName, disabledClusters) {
@@ -312,7 +312,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
312312
})
313313

314314
ginkgo.It("application failover with purgeMode graciously when the application come back to healthy on the new cluster", func() {
315-
disabledClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
315+
disabledClusters := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
316316
ginkgo.By("create an error op", func() {
317317
overridePolicy = testhelper.NewOverridePolicyByOverrideRules(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
318318
{
@@ -357,7 +357,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
357357

358358
ginkgo.By("check whether the failed deployment is rescheduled to other available cluster", func() {
359359
gomega.Eventually(func() int {
360-
targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
360+
targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
361361
for _, targetClusterName := range targetClusterNames {
362362
// the target cluster should be overwritten to another available cluster
363363
if !testhelper.IsExclude(targetClusterName, disabledClusters) {
@@ -388,7 +388,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
388388
framework.PatchPropagationPolicy(karmadaClient, policy.Namespace, policy.Name, patch, types.JSONPatchType)
389389
})
390390

391-
disabledClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
391+
disabledClusters := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
392392
var beginTime time.Time
393393
ginkgo.By("create an error op", func() {
394394
overridePolicy = testhelper.NewOverridePolicyByOverrideRules(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
@@ -503,7 +503,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
503503
})
504504

505505
ginkgo.It("application failover with purgeMode never", func() {
506-
disabledClusters := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
506+
disabledClusters := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
507507
ginkgo.By("create an error op", func() {
508508
overridePolicy = testhelper.NewOverridePolicyByOverrideRules(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
509509
{
@@ -544,7 +544,7 @@ var _ = framework.SerialDescribe("failover testing", func() {
544544

545545
ginkgo.By("check whether the failed deployment is rescheduled to other available cluster", func() {
546546
gomega.Eventually(func() int {
547-
targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
547+
targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
548548
for _, targetClusterName := range targetClusterNames {
549549
// the target cluster should be overwritten to another available cluster
550550
if !testhelper.IsExclude(targetClusterName, disabledClusters) {

‎test/e2e/suites/base/fieldselector_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ var _ = ginkgo.Describe("propagation with fieldSelector testing", func() {
145145

146146
ginkgo.It("propagation with fieldSelector testing", func() {
147147
ginkgo.By("check whether deployment is scheduled to clusters which meeting the fieldSelector requirements", func() {
148-
targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
148+
targetClusterNames := framework.ExtractTargetClustersFromRB(controlPlaneClient, deployment.Kind, deployment.Namespace, deployment.Name)
149149
gomega.Expect(len(targetClusterNames)).Should(gomega.Equal(1))
150150
gomega.Expect(targetClusterNames[0]).Should(gomega.Equal(desiredScheduleResult))
151151
})

0 commit comments

Comments
 (0)