Skip to content

Commit 7abc7db

Browse files
wanyufewanyufe
committed
add testing for failure domain removing
1 parent 6606010 commit 7abc7db

9 files changed

+700
-77
lines changed

‎Makefile

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -248,10 +248,16 @@ cluster-api/tilt-settings.json: hack/tilt-settings.json cluster-api
248248
## --------------------------------------
249249
## Tests
250250
## --------------------------------------
251-
252251
export KUBEBUILDER_ASSETS=$(REPO_ROOT)/$(TOOLS_BIN_DIR)
252+
DEEPCOPY_GEN_TARGETS_TEST=$(shell find test/fakes -type d -name "fakes" -exec echo {}\/zz_generated.deepcopy.go \;)
253+
DEEPCOPY_GEN_INPUTS_TEST=$(shell find test/fakes/* -name "*_types.go" -name "*zz_generated*" -prune -o -type f -print)
254+
.PHONY: generate-deepcopy-test
255+
generate-deepcopy-test: $(DEEPCOPY_GEN_TARGETS_TEST) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
256+
test/fakes/zz_generated.deepcopy.go: $(CONTROLLER_GEN) $(DEEPCOPY_GEN_INPUTS_TEST)
257+
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
258+
253259
.PHONY: test
254-
test: generate-mocks lint $(GINKGO_V2) $(KUBECTL) $(API_SERVER) $(ETCD) ## Run tests. At the moment this is only unit tests.
260+
test: generate-deepcopy-test generate-mocks lint $(GINKGO_V2) $(KUBECTL) $(API_SERVER) $(ETCD) ## Run tests. At the moment this is only unit tests.
255261
@./hack/testing_ginkgo_recover_statements.sh --add # Add ginkgo.GinkgoRecover() statements to controllers.
256262
@# The following is a slightly funky way to make sure the ginkgo statements are removed regardless the test results.
257263
@$(GINKGO_V2) --label-filter="!integ" --cover -coverprofile cover.out --covermode=atomic -v ./api/... ./controllers/... ./pkg/...; EXIT_STATUS=$$?;\

‎controllers/cloudstackfailuredomain_controller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ func (r *CloudStackFailureDomainReconciliationRunner) AllMachinesCanBeCleared()
170170
return ctrl.Result{}, err
171171
}
172172
if specReplicas != statusReplicas {
173-
return r.RequeueWithMessage("spec.replicas <> status.replicas, ", "machineOwner", "owner", ref.Name)
173+
return r.RequeueWithMessage("spec.replicas <> status.replicas, ", "owner", ref.Name, "spec.replicas", specReplicas, "status.replicas", statusReplicas)
174174
}
175175

176176
statusReady, found, err := unstructured.NestedBool(owner.Object, "status", "ready")
@@ -212,7 +212,7 @@ func replicasLargerThanOne(owner *unstructured.Unstructured, ownerName, machineN
212212
}
213213

214214
if specReplicas < 2 {
215-
return specReplicas, 0, errors.Errorf("spec.replicas < 2 in %s, %s cannot be moved away from failure domain", ownerName, machineName)
215+
return specReplicas, statusReplicas, errors.Errorf("spec.replicas < 2 in %s, %s cannot be moved away from failure domain", ownerName, machineName)
216216
}
217217

218218
return specReplicas, statusReplicas, nil

‎controllers/cloudstackfailuredomain_controller_test.go

Lines changed: 192 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@ import (
2020
"github.com/golang/mock/gomock"
2121
. "github.com/onsi/ginkgo/v2"
2222
. "github.com/onsi/gomega"
23+
"k8s.io/apimachinery/pkg/api/errors"
2324
infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta2"
2425
"sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud"
2526
dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta2"
27+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
2628
"sigs.k8s.io/controller-runtime/pkg/client"
2729
)
2830

@@ -32,9 +34,6 @@ var _ = Describe("CloudStackFailureDomainReconciler", func() {
3234
dummies.SetDummyVars()
3335
SetupTestEnvironment() // Must happen before setting up managers/reconcilers.
3436
Ω(FailureDomainReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register CloudStack FailureDomainReconciler.
35-
})
36-
37-
It("Should set failure domain Status.Ready to true.", func() {
3837
// Modify failure domain name the same way the cluster controller would.
3938
dummies.CSFailureDomain1.Name = dummies.CSFailureDomain1.Name + "-" + dummies.CSCluster.Name
4039

@@ -49,14 +48,200 @@ var _ = Describe("CloudStackFailureDomainReconciler", func() {
4948
arg1.(*infrav1.CloudStackZoneSpec).Network.Type = cloud.NetworkTypeShared
5049
}).MinTimes(1)
5150

52-
tempfd := &infrav1.CloudStackFailureDomain{}
51+
})
52+
53+
It("Should set failure domain Status.Ready to true.", func() {
54+
assertFailureDomainCreated()
55+
})
56+
It("Should delete failure domain if no VM under this failure domain.", func() {
57+
assertFailureDomainCreated()
58+
Ω(k8sClient.Delete(ctx, dummies.CSFailureDomain1))
59+
60+
assertFailureDomainNotExisted()
61+
})
62+
63+
It("Should return error if spec.replicas < 2.", func() {
64+
assertFailureDomainCreated()
65+
var specReplicas int32 = 1
66+
var statusReplicas int32 = 1
67+
var statusReadyReplicas int32 = 1
68+
var statusReady = true
69+
setCSMachineOwnerCRD(dummies.CSMachineOwner, &specReplicas, &statusReplicas, &statusReadyReplicas, &statusReady)
70+
setCAPIMachineAndCSMachineCRDs(dummies.CSMachine1, dummies.CAPIMachine)
71+
setMachineOwnerReference(dummies.CSMachine1, dummies.CSMachineOwnerReference)
72+
labelMachineFailuredomain(dummies.CSMachine1, dummies.CSFailureDomain1)
73+
74+
Ω(k8sClient.Delete(ctx, dummies.CSFailureDomain1))
75+
76+
CAPIMachine := &clusterv1.Machine{}
77+
Eventually(func() bool {
78+
key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CAPIMachine.Name}
79+
if err := k8sClient.Get(ctx, key, CAPIMachine); err == nil {
80+
return CAPIMachine.DeletionTimestamp.IsZero()
81+
}
82+
return false
83+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
84+
})
85+
86+
It("Should return error if status.replicas < spec.replicas.", func() {
87+
assertFailureDomainCreated()
88+
var specReplicas int32 = 2
89+
var statusReplicas int32 = 1
90+
var statusReadyReplicas int32 = 1
91+
var statusReady = true
92+
setCSMachineOwnerCRD(dummies.CSMachineOwner, &specReplicas, &statusReplicas, &statusReadyReplicas, &statusReady)
93+
setCAPIMachineAndCSMachineCRDs(dummies.CSMachine1, dummies.CAPIMachine)
94+
setMachineOwnerReference(dummies.CSMachine1, dummies.CSMachineOwnerReference)
95+
labelMachineFailuredomain(dummies.CSMachine1, dummies.CSFailureDomain1)
96+
97+
Ω(k8sClient.Delete(ctx, dummies.CSFailureDomain1))
98+
99+
CAPIMachine := &clusterv1.Machine{}
100+
Eventually(func() bool {
101+
key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CAPIMachine.Name}
102+
if err := k8sClient.Get(ctx, key, CAPIMachine); err == nil {
103+
return CAPIMachine.DeletionTimestamp.IsZero()
104+
}
105+
return false
106+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
107+
})
108+
109+
It("Should return error if status.ready is false.", func() {
110+
assertFailureDomainCreated()
111+
var specReplicas int32 = 2
112+
var statusReplicas int32 = 2
113+
var statusReadyReplicas int32 = 2
114+
var statusReady = false
115+
setCSMachineOwnerCRD(dummies.CSMachineOwner, &specReplicas, &statusReplicas, &statusReadyReplicas, &statusReady)
116+
setCAPIMachineAndCSMachineCRDs(dummies.CSMachine1, dummies.CAPIMachine)
117+
setMachineOwnerReference(dummies.CSMachine1, dummies.CSMachineOwnerReference)
118+
labelMachineFailuredomain(dummies.CSMachine1, dummies.CSFailureDomain1)
119+
120+
Ω(k8sClient.Delete(ctx, dummies.CSFailureDomain1))
121+
122+
CAPIMachine := &clusterv1.Machine{}
123+
Eventually(func() bool {
124+
key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CAPIMachine.Name}
125+
if err := k8sClient.Get(ctx, key, CAPIMachine); err == nil {
126+
return CAPIMachine.DeletionTimestamp.IsZero()
127+
}
128+
return false
129+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
130+
})
131+
132+
It("Should return error if status.readyReplicas <> status.replicas.", func() {
133+
assertFailureDomainCreated()
134+
var specReplicas int32 = 2
135+
var statusReplicas int32 = 2
136+
var statusReadyReplicas int32 = 1
137+
var statusReady = true
138+
setCSMachineOwnerCRD(dummies.CSMachineOwner, &specReplicas, &statusReplicas, &statusReadyReplicas, &statusReady)
139+
setCAPIMachineAndCSMachineCRDs(dummies.CSMachine1, dummies.CAPIMachine)
140+
setMachineOwnerReference(dummies.CSMachine1, dummies.CSMachineOwnerReference)
141+
labelMachineFailuredomain(dummies.CSMachine1, dummies.CSFailureDomain1)
142+
143+
Ω(k8sClient.Delete(ctx, dummies.CSFailureDomain1))
144+
145+
CAPIMachine := &clusterv1.Machine{}
146+
Eventually(func() bool {
147+
key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CAPIMachine.Name}
148+
if err := k8sClient.Get(ctx, key, CAPIMachine); err == nil {
149+
return CAPIMachine.DeletionTimestamp.IsZero()
150+
}
151+
return false
152+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
153+
})
154+
155+
// simulate owner is kubeadmcontrolplane
156+
It("Should delete machine if spec.replicas > 1.", func() {
157+
assertFailureDomainCreated()
158+
var specReplicas int32 = 2
159+
var statusReplicas int32 = 2
160+
var statusReadyReplicas int32 = 2
161+
var statusReady = true
162+
setCSMachineOwnerCRD(dummies.CSMachineOwner, &specReplicas, &statusReplicas, &statusReadyReplicas, &statusReady)
163+
setCAPIMachineAndCSMachineCRDs(dummies.CSMachine1, dummies.CAPIMachine)
164+
setMachineOwnerReference(dummies.CSMachine1, dummies.CSMachineOwnerReference)
165+
labelMachineFailuredomain(dummies.CSMachine1, dummies.CSFailureDomain1)
166+
167+
Ω(k8sClient.Delete(ctx, dummies.CSFailureDomain1))
168+
169+
CAPIMachine := &clusterv1.Machine{}
53170
Eventually(func() bool {
54-
key := client.ObjectKeyFromObject(dummies.CSFailureDomain1)
55-
if err := k8sClient.Get(ctx, key, tempfd); err == nil {
56-
return tempfd.Status.Ready
171+
key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CAPIMachine.Name}
172+
if err := k8sClient.Get(ctx, key, CAPIMachine); err != nil {
173+
return errors.IsNotFound(err)
174+
}
175+
return false
176+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
177+
})
178+
179+
// simulate owner is etcdadmcluster
180+
It("Should delete machine if status.readyReplica does not exist.", func() {
181+
assertFailureDomainCreated()
182+
var specReplicas int32 = 2
183+
var statusReplicas int32 = 2
184+
var statusReady = true
185+
setCSMachineOwnerCRD(dummies.CSMachineOwner, &specReplicas, &statusReplicas, nil, &statusReady)
186+
setCAPIMachineAndCSMachineCRDs(dummies.CSMachine1, dummies.CAPIMachine)
187+
setMachineOwnerReference(dummies.CSMachine1, dummies.CSMachineOwnerReference)
188+
labelMachineFailuredomain(dummies.CSMachine1, dummies.CSFailureDomain1)
189+
190+
Ω(k8sClient.Delete(ctx, dummies.CSFailureDomain1))
191+
192+
CAPIMachine := &clusterv1.Machine{}
193+
Eventually(func() bool {
194+
key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CAPIMachine.Name}
195+
if err := k8sClient.Get(ctx, key, CAPIMachine); err != nil {
196+
return errors.IsNotFound(err)
197+
}
198+
return false
199+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
200+
})
201+
202+
// simulate owner is machineset
203+
It("Should delete machine if status.ready does not exist.", func() {
204+
assertFailureDomainCreated()
205+
var specReplicas int32 = 2
206+
var statusReplicas int32 = 2
207+
var statusReadyReplicas int32 = 2
208+
setCSMachineOwnerCRD(dummies.CSMachineOwner, &specReplicas, &statusReplicas, &statusReadyReplicas, nil)
209+
setCAPIMachineAndCSMachineCRDs(dummies.CSMachine1, dummies.CAPIMachine)
210+
setMachineOwnerReference(dummies.CSMachine1, dummies.CSMachineOwnerReference)
211+
labelMachineFailuredomain(dummies.CSMachine1, dummies.CSFailureDomain1)
212+
213+
Ω(k8sClient.Delete(ctx, dummies.CSFailureDomain1))
214+
215+
CAPIMachine := &clusterv1.Machine{}
216+
Eventually(func() bool {
217+
key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CAPIMachine.Name}
218+
if err := k8sClient.Get(ctx, key, CAPIMachine); err != nil {
219+
return errors.IsNotFound(err)
57220
}
58221
return false
59222
}, timeout).WithPolling(pollInterval).Should(BeTrue())
60223
})
61224
})
62225
})
226+
227+
func assertFailureDomainCreated() {
228+
tempfd := &infrav1.CloudStackFailureDomain{}
229+
Eventually(func() bool {
230+
key := client.ObjectKeyFromObject(dummies.CSFailureDomain1)
231+
if err := k8sClient.Get(ctx, key, tempfd); err == nil {
232+
return tempfd.Status.Ready
233+
}
234+
return false
235+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
236+
}
237+
238+
func assertFailureDomainNotExisted() {
239+
tempfd := &infrav1.CloudStackFailureDomain{}
240+
Eventually(func() bool {
241+
key := client.ObjectKeyFromObject(dummies.CSFailureDomain1)
242+
if err := k8sClient.Get(ctx, key, tempfd); err != nil {
243+
return true
244+
}
245+
return false
246+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
247+
}

‎controllers/controllers_suite_test.go

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@ import (
2121
"flag"
2222
"fmt"
2323
"go/build"
24+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
2425
"os"
2526
"os/exec"
2627
"path/filepath"
2728
"regexp"
29+
"sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes"
2830
"strings"
2931
"testing"
3032
"time"
@@ -142,6 +144,7 @@ var _ = BeforeSuite(func() {
142144

143145
Ω(infrav1.AddToScheme(scheme.Scheme)).Should(Succeed())
144146
Ω(clusterv1.AddToScheme(scheme.Scheme)).Should(Succeed())
147+
Ω(fakes.AddToScheme(scheme.Scheme)).Should(Succeed())
145148

146149
// Increase log verbosity.
147150
klog.InitFlags(nil)
@@ -189,6 +192,8 @@ func SetupTestEnvironment() {
189192
go func() {
190193
defer GinkgoRecover()
191194
cfg, err = testEnv.Start()
195+
extraCRDs := append([]*apiextensionsv1.CustomResourceDefinition{}, dummies.CSMachineOwnerCRD)
196+
err = envtest.CreateCRDs(cfg, extraCRDs)
192197
close(done)
193198
}()
194199
Eventually(done).WithTimeout(time.Minute).Should(BeClosed())
@@ -371,3 +376,83 @@ func setupMachineCRDs() {
371376
return ph.Patch(ctx, dummies.CSMachine1, patch.WithStatusObservedGeneration{})
372377
}, timeout).Should(Succeed())
373378
}
379+
380+
func setCSMachineOwnerCRD(owner *fakes.CloudStackMachineOwner, specReplicas, statusReplicas, statusReadyReplicas *int32, statusReady *bool) {
381+
owner.Spec.Replicas = specReplicas
382+
Ω(k8sClient.Create(ctx, owner)).Should(Succeed())
383+
key := client.ObjectKey{Namespace: owner.Namespace, Name: owner.Name}
384+
Eventually(func() error {
385+
return k8sClient.Get(ctx, key, owner)
386+
}, timeout).Should(BeNil())
387+
388+
Eventually(func() error {
389+
owner.Status.Ready = statusReady
390+
owner.Status.Replicas = statusReplicas
391+
owner.Status.ReadyReplicas = statusReadyReplicas
392+
return k8sClient.Status().Update(ctx, owner)
393+
}, timeout).Should(BeNil())
394+
395+
}
396+
397+
// setCAPIMachineAndCSMachineCRDs creates a CAPI and CloudStack machine with an appropriate ownership ref between them.
398+
func setCAPIMachineAndCSMachineCRDs(CSMachine *infrav1.CloudStackMachine, CAPIMachine *clusterv1.Machine) {
399+
// Create them.
400+
Ω(k8sClient.Create(ctx, CAPIMachine)).Should(Succeed())
401+
Ω(k8sClient.Create(ctx, CSMachine)).Should(Succeed())
402+
403+
// Fetch the CS Machine that was created.
404+
key := client.ObjectKey{Namespace: dummies.CSCluster.Namespace, Name: CSMachine.Name}
405+
Eventually(func() error {
406+
return k8sClient.Get(ctx, key, CSMachine)
407+
}, timeout).Should(BeNil())
408+
409+
// Fetch the CAPI Machine that was created.
410+
key = client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: CAPIMachine.Name}
411+
Eventually(func() error {
412+
return k8sClient.Get(ctx, key, CAPIMachine)
413+
}, timeout).Should(BeNil())
414+
415+
// Set ownerReference to CAPI machine in CS machine and patch back the CS machine.
416+
Eventually(func() error {
417+
ph, err := patch.NewHelper(dummies.CSMachine1, k8sClient)
418+
Ω(err).ShouldNot(HaveOccurred())
419+
dummies.CSMachine1.OwnerReferences = append(dummies.CSMachine1.OwnerReferences, metav1.OwnerReference{
420+
Kind: "Machine",
421+
APIVersion: clusterv1.GroupVersion.String(),
422+
Name: CAPIMachine.Name,
423+
UID: "uniqueness",
424+
})
425+
return ph.Patch(ctx, CSMachine, patch.WithStatusObservedGeneration{})
426+
}, timeout).Should(Succeed())
427+
}
428+
429+
func setMachineOwnerReference(CSMachine *infrav1.CloudStackMachine, ownerRef metav1.OwnerReference) {
430+
key := client.ObjectKey{Namespace: dummies.CSCluster.Namespace, Name: CSMachine.Name}
431+
Eventually(func() error {
432+
return k8sClient.Get(ctx, key, CSMachine)
433+
}, timeout).Should(BeNil())
434+
435+
// Set ownerReference to CAPI machine in CS machine and patch back the CS machine.
436+
Eventually(func() error {
437+
ph, err := patch.NewHelper(CSMachine, k8sClient)
438+
Ω(err).ShouldNot(HaveOccurred())
439+
CSMachine.OwnerReferences = append(CSMachine.OwnerReferences, ownerRef)
440+
return ph.Patch(ctx, CSMachine, patch.WithStatusObservedGeneration{})
441+
}, timeout).Should(Succeed())
442+
}
443+
444+
// labelMachineFailuredomain add cloudstackfailuredomain info in the labels.
445+
func labelMachineFailuredomain(CSMachine *infrav1.CloudStackMachine, CSFailureDomain1 *infrav1.CloudStackFailureDomain) {
446+
key := client.ObjectKey{Namespace: dummies.CSCluster.Namespace, Name: CSMachine.Name}
447+
Eventually(func() error {
448+
return k8sClient.Get(ctx, key, CSMachine)
449+
}, timeout).Should(BeNil())
450+
451+
// set cloudstack failuredomain in machine labels.
452+
Eventually(func() error {
453+
ph, err := patch.NewHelper(CSMachine, k8sClient)
454+
Ω(err).ShouldNot(HaveOccurred())
455+
CSMachine.Labels["cloudstackfailuredomain.infrastructure.cluster.x-k8s.io/name"] = CSFailureDomain1.Name
456+
return ph.Patch(ctx, CSMachine, patch.WithStatusObservedGeneration{})
457+
}, timeout).Should(Succeed())
458+
}

‎go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module sigs.k8s.io/cluster-api-provider-cloudstack
33
go 1.16
44

55
require (
6-
github.com/ReneKroon/ttlcache v1.7.0 // indirect
6+
github.com/ReneKroon/ttlcache v1.7.0
77
github.com/apache/cloudstack-go/v2 v2.13.0
88
github.com/go-logr/logr v1.2.3
99
github.com/golang/mock v1.6.0
@@ -16,6 +16,7 @@ require (
1616
github.com/spf13/pflag v1.0.5
1717
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
1818
k8s.io/api v0.23.0
19+
k8s.io/apiextensions-apiserver v0.23.0
1920
k8s.io/apimachinery v0.23.0
2021
k8s.io/client-go v0.23.0
2122
k8s.io/klog/v2 v2.30.0

0 commit comments

Comments
 (0)