Skip to content

Commit d0cb679

Browse files
wongniwongni
committed
Test machine health check after destroying machines
1 parent f23c127 commit d0cb679

File tree

11 files changed

+306
-0
lines changed

11 files changed

+306
-0
lines changed

‎test/e2e/common.go

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,20 @@ package e2e
1818

1919
import (
2020
"context"
21+
"encoding/base64"
2122
"fmt"
23+
"os"
2224
"path/filepath"
25+
"strings"
2326

27+
"github.com/apache/cloudstack-go/v2/cloudstack"
2428
"github.com/blang/semver"
2529
. "github.com/onsi/ginkgo"
2630
"github.com/onsi/gomega/types"
31+
"gopkg.in/ini.v1"
2732
corev1 "k8s.io/api/core/v1"
2833

34+
. "github.com/onsi/gomega"
2935
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3036
"sigs.k8s.io/cluster-api/test/framework"
3137
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -174,3 +180,80 @@ func DownloadFromAppInWorkloadCluster(ctx context.Context, workloadKubeconfigPat
174180
}
175181
return KubectlExec(ctx, "run", workloadKubeconfigPath, runArgs...)
176182
}
183+
184+
type cloudConfig struct {
185+
APIURL string `ini:"api-url"`
186+
APIKey string `ini:"api-key"`
187+
SecretKey string `ini:"secret-key"`
188+
VerifySSL bool `ini:"verify-ssl"`
189+
}
190+
191+
func DestroyOneMachineAndWaitForReplacement(clusterName string, machineType string, intervals []interface{}) {
192+
encodedSecret := os.Getenv("CLOUDSTACK_B64ENCODED_SECRET")
193+
secret, err := base64.StdEncoding.DecodeString(encodedSecret)
194+
if err != nil {
195+
Fail("Failed ")
196+
}
197+
cfg := &cloudConfig{VerifySSL: true}
198+
if rawCfg, err := ini.Load(secret); err != nil {
199+
Fail("Failed to load INI file")
200+
} else if g := rawCfg.Section("Global"); len(g.Keys()) == 0 {
201+
Fail("Global section not found")
202+
} else if err = rawCfg.Section("Global").StrictMapTo(cfg); err != nil {
203+
Fail("Error encountered while parsing Global section")
204+
}
205+
206+
By("Creating a CloudStack client")
207+
client := cloudstack.NewAsyncClient(cfg.APIURL, cfg.APIKey, cfg.SecretKey, cfg.VerifySSL)
208+
209+
matcher := clusterName + "-" + machineType
210+
211+
Byf("Listing machines with %q", matcher)
212+
listResp, err := client.VirtualMachine.ListVirtualMachines(client.VirtualMachine.NewListVirtualMachinesParams())
213+
if err != nil {
214+
Fail("Failed to list machines")
215+
}
216+
var vmToDestroy *cloudstack.VirtualMachine
217+
originalCount := 0
218+
for _, vm := range listResp.VirtualMachines {
219+
if strings.Contains(vm.Name, matcher) {
220+
originalCount++
221+
if vmToDestroy == nil {
222+
vmToDestroy = vm
223+
}
224+
}
225+
}
226+
Byf("Original number of machines: %d ", originalCount)
227+
228+
Byf("Destroying machine %s", vmToDestroy.Name)
229+
stopParams := client.VirtualMachine.NewStopVirtualMachineParams(vmToDestroy.Id)
230+
stopParams.SetForced(true)
231+
_, err = client.VirtualMachine.StopVirtualMachine(stopParams)
232+
if err != nil {
233+
Fail("Failed to stop machine")
234+
}
235+
destroyParams := client.VirtualMachine.NewDestroyVirtualMachineParams(vmToDestroy.Id)
236+
destroyParams.SetExpunge(true)
237+
_, err = client.VirtualMachine.DestroyVirtualMachine(destroyParams)
238+
if err != nil {
239+
Fail("Failed to destroy machine")
240+
}
241+
Byf("Destroyed machine %s", vmToDestroy.Name)
242+
243+
By("Waiting for a replacement machine")
244+
Eventually(func() (int, error) {
245+
By("Counting machines")
246+
listResp, err = client.VirtualMachine.ListVirtualMachines(client.VirtualMachine.NewListVirtualMachinesParams())
247+
if err != nil {
248+
return -1, err
249+
}
250+
count := 0
251+
for _, vm := range listResp.VirtualMachines {
252+
if strings.Contains(vm.Name, matcher) {
253+
count++
254+
}
255+
}
256+
Byf("Current number of machines: %d", count)
257+
return count, nil
258+
}, intervals...).Should(Equal(originalCount))
259+
}

‎test/e2e/config/cloudstack.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ providers:
8484
- sourcePath: "../data/infrastructure-cloudstack/v1beta1/cluster-template-insufficient-compute-resources.yaml"
8585
- sourcePath: "../data/infrastructure-cloudstack/v1beta1/cluster-template-invalid-worker-offering.yaml"
8686
- sourcePath: "../data/infrastructure-cloudstack/v1beta1/cluster-template-node-drain.yaml"
87+
- sourcePath: "../data/infrastructure-cloudstack/v1beta1/cluster-template-destroy-kcp-machine.yaml"
88+
- sourcePath: "../data/infrastructure-cloudstack/v1beta1/cluster-template-destroy-md-machine.yaml"
8789
- sourcePath: "../data/shared/v1beta1/metadata.yaml"
8890
versions:
8991
- name: v1.0.0

‎test/e2e/data/infrastructure-cloudstack/v1beta1/cluster-template-destroy-kcp-machine/kustomization.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
bases:
2+
- ../bases/cluster-with-kcp.yaml
3+
- ../bases/md.yaml
4+
# The following error occurs if crs.yaml is added in this test manifest file.
5+
# Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "default.clusterresourceset.addons.cluster.x-k8s.io": the server could not find the requested resource
6+
# - ../bases/crs.yaml
7+
- mhc.yaml

‎test/e2e/data/infrastructure-cloudstack/v1beta1/cluster-template-destroy-kcp-machine/mhc.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
# MachineHealthCheck object with
3+
# - a selector that targets all the machines with label cluster.x-k8s.io/control-plane=""
4+
# - unhealthyConditions triggering remediation after 10s the condition is set
5+
apiVersion: cluster.x-k8s.io/v1beta1
6+
kind: MachineHealthCheck
7+
metadata:
8+
name: "${CLUSTER_NAME}-mhc-0"
9+
spec:
10+
clusterName: "${CLUSTER_NAME}"
11+
maxUnhealthy: 100%
12+
selector:
13+
matchLabels:
14+
cluster.x-k8s.io/control-plane: ""
15+
unhealthyConditions:
16+
- type: Ready
17+
status: Unknown
18+
timeout: 10s
19+
- type: Ready
20+
status: "False"
21+
timeout: 10s

‎test/e2e/data/infrastructure-cloudstack/v1beta1/cluster-template-destroy-md-machine/kustomization.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
bases:
2+
- ../bases/cluster-with-kcp.yaml
3+
- ../bases/md.yaml
4+
- mhc.yaml
5+
6+
patchesStrategicMerge:
7+
- ./md.yaml

‎test/e2e/data/infrastructure-cloudstack/v1beta1/cluster-template-destroy-md-machine/md.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: MachineDeployment
3+
metadata:
4+
name: "${CLUSTER_NAME}-md-0"
5+
spec:
6+
template:
7+
metadata:
8+
labels:
9+
"e2e.mhc.label": ""

‎test/e2e/data/infrastructure-cloudstack/v1beta1/cluster-template-destroy-md-machine/mhc.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: MachineHealthCheck
4+
metadata:
5+
name: "${CLUSTER_NAME}-mhc-md-0"
6+
spec:
7+
clusterName: "${CLUSTER_NAME}"
8+
maxUnhealthy: 100%
9+
selector:
10+
matchLabels:
11+
e2e.mhc.label: ""
12+
unhealthyConditions:
13+
- type: Ready
14+
status: Unknown
15+
timeout: 10s
16+
- type: Ready
17+
status: "False"
18+
timeout: 10s

‎test/e2e/destroy_machine.go

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
25+
. "github.com/onsi/ginkgo"
26+
. "github.com/onsi/gomega"
27+
corev1 "k8s.io/api/core/v1"
28+
"k8s.io/utils/pointer"
29+
30+
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
31+
"sigs.k8s.io/cluster-api/util"
32+
)
33+
34+
// DestroyMachineSpec implements a test that verifies that an app deployed to the workload cluster works.
35+
func DestroyMachineSpec(ctx context.Context, inputGetter func() CommonSpecInput) {
36+
var (
37+
specName = "destroy-machine"
38+
input CommonSpecInput
39+
namespace *corev1.Namespace
40+
cancelWatches context.CancelFunc
41+
clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
42+
)
43+
44+
BeforeEach(func() {
45+
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
46+
input = inputGetter()
47+
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
48+
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
49+
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
50+
Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
51+
52+
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
53+
54+
// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
55+
namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
56+
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
57+
})
58+
59+
It("Should replace a KCP machine when it is destroyed", func() {
60+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
61+
ClusterProxy: input.BootstrapClusterProxy,
62+
CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
63+
ConfigCluster: clusterctl.ConfigClusterInput{
64+
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
65+
ClusterctlConfigPath: input.ClusterctlConfigPath,
66+
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
67+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
68+
Flavor: "destroy-kcp-machine",
69+
Namespace: namespace.Name,
70+
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
71+
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
72+
ControlPlaneMachineCount: pointer.Int64Ptr(3),
73+
WorkerMachineCount: pointer.Int64Ptr(1),
74+
},
75+
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
76+
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
77+
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
78+
}, clusterResources)
79+
80+
DestroyOneMachineAndWaitForReplacement(clusterResources.Cluster.Name, "control-plane", input.E2EConfig.GetIntervals(specName, "wait-machine-remediation"))
81+
82+
By("PASSED!")
83+
})
84+
85+
It("Should replace a machine deployment when it is destroyed", func() {
86+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
87+
ClusterProxy: input.BootstrapClusterProxy,
88+
CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
89+
ConfigCluster: clusterctl.ConfigClusterInput{
90+
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
91+
ClusterctlConfigPath: input.ClusterctlConfigPath,
92+
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
93+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
94+
Flavor: "destroy-md-machine",
95+
Namespace: namespace.Name,
96+
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
97+
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
98+
ControlPlaneMachineCount: pointer.Int64Ptr(1),
99+
WorkerMachineCount: pointer.Int64Ptr(3),
100+
},
101+
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
102+
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
103+
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
104+
}, clusterResources)
105+
106+
DestroyOneMachineAndWaitForReplacement(clusterResources.Cluster.Name, "md", input.E2EConfig.GetIntervals(specName, "wait-machine-remediation"))
107+
108+
By("PASSED!")
109+
})
110+
111+
AfterEach(func() {
112+
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
113+
dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
114+
})
115+
}

‎test/e2e/destroy_machine_test.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
/*
5+
Copyright 2020 The Kubernetes Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package e2e
21+
22+
import (
23+
"context"
24+
. "github.com/onsi/ginkgo"
25+
)
26+
27+
var _ = Describe("When testing destroying machine", func() {
28+
29+
DestroyMachineSpec(context.TODO(), func() CommonSpecInput {
30+
return CommonSpecInput{
31+
E2EConfig: e2eConfig,
32+
ClusterctlConfigPath: clusterctlConfigPath,
33+
BootstrapClusterProxy: bootstrapClusterProxy,
34+
ArtifactFolder: artifactFolder,
35+
SkipCleanup: skipCleanup,
36+
}
37+
})
38+
39+
})

‎test/e2e/go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ module github.com/aws/cluster-api-provider-cloudstack-staging/test/e2e
33
go 1.16
44

55
require (
6+
github.com/apache/cloudstack-go/v2 v2.12.0
67
github.com/blang/semver v3.5.1+incompatible
78
github.com/onsi/ginkgo v1.16.5
89
github.com/onsi/gomega v1.17.0
10+
gopkg.in/ini.v1 v1.63.2
911
k8s.io/api v0.23.0
1012
k8s.io/apimachinery v0.23.0
1113
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b

0 commit comments

Comments
 (0)