Skip to content

Commit b61f337

Browse files
author
Arvind Thirumurugan
committed
test: Drain tool E2Es, UTs
Signed-off-by: Arvind Thirumurugan <[email protected]>
1 parent 4e5ea3c commit b61f337

File tree

12 files changed

+1093
-107
lines changed

12 files changed

+1093
-107
lines changed

‎test/e2e/drain_tool_test.go‎

Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
/*
2+
Copyright 2025 The KubeFleet Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"fmt"
21+
"os/exec"
22+
23+
. "github.com/onsi/ginkgo/v2"
24+
. "github.com/onsi/gomega"
25+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
"k8s.io/apimachinery/pkg/util/intstr"
27+
28+
placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
29+
"go.goms.io/fleet/test/e2e/framework"
30+
)
31+
32+
var _ = Describe("Drain cluster successfully", Ordered, Serial, func() {
33+
crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())
34+
var drainClusters, noDrainClusters []*framework.Cluster
35+
var noDrainClusterNames []string
36+
37+
BeforeAll(func() {
38+
drainClusters = []*framework.Cluster{memberCluster1EastProd}
39+
noDrainClusters = []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd}
40+
noDrainClusterNames = []string{memberCluster2EastCanaryName, memberCluster3WestProdName}
41+
42+
By("creating work resources")
43+
createWorkResources()
44+
45+
// Create the CRP.
46+
crp := &placementv1beta1.ClusterResourcePlacement{
47+
ObjectMeta: metav1.ObjectMeta{
48+
Name: crpName,
49+
// Add a custom finalizer; this would allow us to better observe
50+
// the behavior of the controllers.
51+
Finalizers: []string{customDeletionBlockerFinalizer},
52+
},
53+
Spec: placementv1beta1.ClusterResourcePlacementSpec{
54+
Policy: &placementv1beta1.PlacementPolicy{
55+
PlacementType: placementv1beta1.PickAllPlacementType,
56+
},
57+
ResourceSelectors: workResourceSelector(),
58+
},
59+
}
60+
Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName)
61+
})
62+
63+
AfterAll(func() {
64+
// remove taints from member cluster 1 again to guarantee clean up of cordon taint on test failure.
65+
removeTaintsFromMemberClusters([]string{memberCluster1EastProdName})
66+
ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters)
67+
})
68+
69+
It("should update cluster resource placement status as expected", func() {
70+
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
71+
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
72+
})
73+
74+
It("should place resources on all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
75+
76+
It("drain cluster using binary, should succeed", func() { runDrainClusterBinary(hubClusterName, memberCluster1EastProdName) })
77+
78+
It("should ensure no resources exist on drained clusters", func() {
79+
for _, cluster := range drainClusters {
80+
resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
81+
Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
82+
}
83+
})
84+
85+
It("should update cluster resource placement status as expected", func() {
86+
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noDrainClusterNames, nil, "0")
87+
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
88+
})
89+
90+
It("should still place resources on the selected clusters which were not drained", func() {
91+
for _, cluster := range noDrainClusters {
92+
resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster)
93+
Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters")
94+
}
95+
})
96+
97+
It("uncordon cluster using binary", func() { runUncordonClusterBinary(hubClusterName, memberCluster1EastProdName) })
98+
99+
It("should update cluster resource placement status as expected", func() {
100+
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
101+
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
102+
})
103+
104+
It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
105+
})
106+
107+
var _ = Describe("Drain cluster blocked - ClusterResourcePlacementDisruptionBudget blocks evictions on all clusters", Ordered, Serial, func() {
108+
crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())
109+
110+
BeforeAll(func() {
111+
By("creating work resources")
112+
createWorkResources()
113+
114+
// Create the CRP.
115+
crp := &placementv1beta1.ClusterResourcePlacement{
116+
ObjectMeta: metav1.ObjectMeta{
117+
Name: crpName,
118+
// Add a custom finalizer; this would allow us to better observe
119+
// the behavior of the controllers.
120+
Finalizers: []string{customDeletionBlockerFinalizer},
121+
},
122+
Spec: placementv1beta1.ClusterResourcePlacementSpec{
123+
Policy: &placementv1beta1.PlacementPolicy{
124+
PlacementType: placementv1beta1.PickAllPlacementType,
125+
},
126+
ResourceSelectors: workResourceSelector(),
127+
},
128+
}
129+
Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName)
130+
})
131+
132+
AfterAll(func() {
133+
// remove taints from member cluster 1 again to guarantee clean up of cordon taint on test failure.
134+
removeTaintsFromMemberClusters([]string{memberCluster1EastProdName})
135+
ensureCRPDisruptionBudgetDeleted(crpName)
136+
ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters)
137+
})
138+
139+
It("should update cluster resource placement status as expected", func() {
140+
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
141+
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
142+
})
143+
144+
It("should place resources on all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
145+
146+
It("create cluster resource placement disruption budget to block draining", func() {
147+
crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{
148+
ObjectMeta: metav1.ObjectMeta{
149+
Name: crpName,
150+
},
151+
Spec: placementv1beta1.PlacementDisruptionBudgetSpec{
152+
MinAvailable: &intstr.IntOrString{
153+
Type: intstr.Int,
154+
IntVal: int32(len(allMemberClusterNames)),
155+
},
156+
},
157+
}
158+
Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName)
159+
})
160+
161+
It("drain cluster using binary, should fail due to CRPDB", func() { runDrainClusterBinary(hubClusterName, memberCluster1EastProdName) })
162+
163+
It("should ensure cluster resource placement status remains unchanged", func() {
164+
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
165+
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
166+
})
167+
168+
It("should still place resources on all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
169+
170+
It("uncordon cluster using binary", func() { runUncordonClusterBinary(hubClusterName, memberCluster1EastProdName) })
171+
172+
It("should update cluster resource placement status as expected", func() {
173+
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
174+
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
175+
})
176+
177+
It("should place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
178+
})
179+
180+
var _ = Describe("Drain is allowed on one cluster, blocked on others - ClusterResourcePlacementDisruptionBudget blocks evictions on some clusters", Ordered, Serial, func() {
181+
crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())
182+
var drainClusters, noDrainClusters []*framework.Cluster
183+
var noDrainClusterNames []string
184+
185+
BeforeAll(func() {
186+
drainClusters = []*framework.Cluster{memberCluster1EastProd}
187+
noDrainClusters = []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd}
188+
noDrainClusterNames = []string{memberCluster2EastCanaryName, memberCluster3WestProdName}
189+
190+
By("creating work resources")
191+
createWorkResources()
192+
193+
// Create the CRP.
194+
crp := &placementv1beta1.ClusterResourcePlacement{
195+
ObjectMeta: metav1.ObjectMeta{
196+
Name: crpName,
197+
// Add a custom finalizer; this would allow us to better observe
198+
// the behavior of the controllers.
199+
Finalizers: []string{customDeletionBlockerFinalizer},
200+
},
201+
Spec: placementv1beta1.ClusterResourcePlacementSpec{
202+
Policy: &placementv1beta1.PlacementPolicy{
203+
PlacementType: placementv1beta1.PickAllPlacementType,
204+
},
205+
ResourceSelectors: workResourceSelector(),
206+
},
207+
}
208+
Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName)
209+
})
210+
211+
AfterAll(func() {
212+
// remove taints from member clusters 1,2 again to guarantee clean up of cordon taint on test failure.
213+
removeTaintsFromMemberClusters([]string{memberCluster1EastProdName, memberCluster2EastCanaryName})
214+
ensureCRPDisruptionBudgetDeleted(crpName)
215+
ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters)
216+
})
217+
218+
It("should update cluster resource placement status as expected", func() {
219+
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0")
220+
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
221+
})
222+
223+
It("should place resources on all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
224+
225+
It("create cluster resource placement disruption budget to block draining on all but one cluster", func() {
226+
crpdb := placementv1beta1.ClusterResourcePlacementDisruptionBudget{
227+
ObjectMeta: metav1.ObjectMeta{
228+
Name: crpName,
229+
},
230+
Spec: placementv1beta1.PlacementDisruptionBudgetSpec{
231+
MinAvailable: &intstr.IntOrString{
232+
Type: intstr.Int,
233+
IntVal: int32(len(allMemberClusterNames)) - 1,
234+
},
235+
},
236+
}
237+
Expect(hubClient.Create(ctx, &crpdb)).To(Succeed(), "Failed to create CRP Disruption Budget %s", crpName)
238+
})
239+
240+
It("drain cluster using binary, should succeed", func() { runDrainClusterBinary(hubClusterName, memberCluster1EastProdName) })
241+
242+
It("drain cluster using binary, should fail due to CRPDB", func() { runDrainClusterBinary(hubClusterName, memberCluster2EastCanaryName) })
243+
244+
It("should ensure no resources exist on drained clusters", func() {
245+
for _, cluster := range drainClusters {
246+
resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
247+
Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
248+
}
249+
})
250+
251+
It("should update cluster resource placement status as expected", func() {
252+
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), noDrainClusterNames, nil, "0")
253+
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
254+
})
255+
256+
It("should still place resources on the selected clusters which were not drained", func() {
257+
for _, cluster := range noDrainClusters {
258+
resourcePlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(cluster)
259+
Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on the selected clusters")
260+
}
261+
})
262+
263+
It("uncordon cluster using binary", func() { runUncordonClusterBinary(hubClusterName, memberCluster1EastProdName) })
264+
265+
It("uncordon cluster using binary", func() { runUncordonClusterBinary(hubClusterName, memberCluster2EastCanaryName) })
266+
267+
It("should place resources on all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters)
268+
})
269+
270+
func runDrainClusterBinary(hubClusterName, memberClusterName string) {
271+
By(fmt.Sprintf("draining cluster %s", memberClusterName))
272+
cmd := exec.Command(drainBinaryPath,
273+
"--hubClusterContext", hubClusterName,
274+
"--clusterName", memberClusterName)
275+
_, err := cmd.CombinedOutput()
276+
Expect(err).ToNot(HaveOccurred(), "Drain command failed with error: %v", err)
277+
}
278+
279+
func runUncordonClusterBinary(hubClusterName, memberClusterName string) {
280+
By(fmt.Sprintf("uncordoning cluster %s", memberClusterName))
281+
cmd := exec.Command(uncordonBinaryPath,
282+
"--hubClusterContext", hubClusterName,
283+
"--clusterName", memberClusterName)
284+
_, err := cmd.CombinedOutput()
285+
Expect(err).ToNot(HaveOccurred(), "Uncordon command failed with error: %v", err)
286+
}

‎test/e2e/setup.sh‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,12 @@ REGIONS=("" "" "eastasia")
3939
AKS_NODE_REGIONS=("westus" "northeurope" "eastasia")
4040
# The SKUs that should be set on each node of the respective clusters; if the AKS property
4141
# provider is used. See the AKS documentation for specifics.
42-
#
42+
#
4343
# Note that this is for information only; kind nodes always use the same fixed setup
4444
# (total/allocatable capacity = host capacity).
4545
AKS_NODE_SKUS=("Standard_A4_v2" "Standard_B4ms" "Standard_D8s_v5" "Standard_E16_v5" "Standard_M16ms")
4646
AKS_SKU_COUNT=${#AKS_NODE_SKUS[@]}
47-
# The number of clusters that has pre-defined configuration for testing purposes.
47+
# The number of clusters that has pre-defined configuration for testing purposes.
4848
RESERVED_CLUSTER_COUNT=${MEMBER_CLUSTER_COUNT}
4949

5050
# Create the kind clusters
@@ -87,7 +87,7 @@ then
8787
k=$(( RANDOM % AKS_SKU_COUNT ))
8888
kubectl label node "${NODES[$j]}" beta.kubernetes.io/instance-type=${AKS_NODE_SKUS[$k]}
8989
done
90-
done
90+
done
9191
fi
9292

9393
# Build the Fleet agent images
@@ -207,3 +207,13 @@ do
207207
fi
208208
done
209209

210+
# Create tools directory if it doesn't exist
211+
mkdir -p ../../hack/tools/bin
212+
213+
# Build drain binary
214+
echo "Building drain binary..."
215+
go build -o ../../hack/tools/bin/kubectl-draincluster ../../tools/draincluster
216+
217+
# Build uncordon binary
218+
echo "Building uncordon binary..."
219+
go build -o ../../hack/tools/bin/kubectl-uncordoncluster ../../tools/uncordoncluster

‎test/e2e/setup_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"flag"
2222
"log"
2323
"os"
24+
"path/filepath"
2425
"sync"
2526
"testing"
2627
"time"
@@ -163,6 +164,11 @@ var (
163164
}
164165
)
165166

167+
var (
168+
drainBinaryPath = filepath.Join("../../", "hack", "tools", "bin", "kubectl-draincluster")
169+
uncordonBinaryPath = filepath.Join("../../", "hack", "tools", "bin", "kubectl-uncordoncluster")
170+
)
171+
166172
var (
167173
isAzurePropertyProviderEnabled = (os.Getenv(propertyProviderEnvVarName) == azurePropertyProviderEnvVarValue)
168174

@@ -366,4 +372,7 @@ var _ = SynchronizedAfterSuite(func() {}, func() {
366372
setAllMemberClustersToLeave()
367373
checkIfAllMemberClustersHaveLeft()
368374
cleanupInvalidClusters()
375+
// Cleanup tool binaries.
376+
Expect(os.Remove(drainBinaryPath)).Should(Succeed(), "Failed to remove drain binary")
377+
Expect(os.Remove(uncordonBinaryPath)).Should(Succeed(), "Failed to remove uncordon binary")
369378
})

‎tools/draincluster/README.md‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
1. Build the binary for the `draincluster` tool by running the following command in the root directory of the fleet repo:
44

55
```bash
6-
go build -o ./hack/tools/bin/kubectl-draincluster ./tools/draincluster/main.go
6+
go build -o ./hack/tools/bin/kubectl-draincluster ./tools/draincluster/
77
```
88

99
2. Copy the binary to a directory in your `PATH` so that it can be run as a kubectl plugin. For example, you can move it to
10-
`/usr/local/bin`:
10+
`/usr/local/bin`:
1111

1212
```bash
1313
sudo cp ./hack/tools/bin/kubectl-draincluster /usr/local/bin/
@@ -33,13 +33,13 @@ The following compatible plugins are available:
3333
/usr/local/bin/kubectl-draincluster
3434
```
3535

36-
please refer to the [kubectl plugin documentation](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/) for
36+
please refer to the [kubectl plugin documentation](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/) for
3737
more information.
3838

3939
# Drain Member Cluster connected to a fleet
4040

41-
After following the steps above to build the `draincluster` tool as a kubectl plugin, you can use it to remove all
42-
resources propagated to the member cluster from the hub cluster by any `Placement` resource. This is useful when you
41+
After following the steps above to build the `draincluster` tool as a kubectl plugin, you can use it to remove all
42+
resources propagated to the member cluster from the hub cluster by any `Placement` resource. This is useful when you
4343
want to temporarily move all workloads off a member cluster in preparation for an event like upgrade or reconfiguration.
4444

4545
The `draincluster` tool can be used to drain a member cluster by running the following command:
@@ -68,8 +68,8 @@ CURRENT NAME CLUSTER AUTHINFO
6868

6969
Here you can see that the context of the hub cluster is called `hub` under the `NAME` column.
7070

71-
The command adds a `Taint` to the `MemberCluster` resource of the member cluster to prevent any new resources from being
72-
propagated to the member cluster. Then it creates `Eviction` objects for all the `Placement` objects that have propagated
71+
The command adds a `Taint` to the `MemberCluster` resource of the member cluster to prevent any new resources from being
72+
propagated to the member cluster. Then it creates `Eviction` objects for all the `Placement` objects that have propagated
7373
resources to the member cluster.
7474

7575
>> **Note**: The `draincluster` tool is a best-effort mechanism at the moment, so once the command is run successfully

0 commit comments

Comments
 (0)