Skip to content

Commit 3533969

Browse files
wongniwongni
committed
Check if the created affinity gorups are deleted at the end
Check VM's host assignments Skip affinity test because the ACS used by Prow doesn't have multiple hosts
1 parent 01d7345 commit 3533969

File tree

4 files changed

+118
-80
lines changed

4 files changed

+118
-80
lines changed

‎test/e2e/affinity_group.go

Lines changed: 36 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ func AffinityGroupSpec(ctx context.Context, inputGetter func() CommonSpecInput)
3939
namespace *corev1.Namespace
4040
cancelWatches context.CancelFunc
4141
clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
42+
affinityIds []string
4243
)
4344

4445
BeforeEach(func() {
@@ -57,59 +58,48 @@ func AffinityGroupSpec(ctx context.Context, inputGetter func() CommonSpecInput)
5758
})
5859

5960
It("Should have host affinity group when affinity is pro", func() {
60-
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
61-
ClusterProxy: input.BootstrapClusterProxy,
62-
CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
63-
ConfigCluster: clusterctl.ConfigClusterInput{
64-
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
65-
ClusterctlConfigPath: input.ClusterctlConfigPath,
66-
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
67-
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
68-
Flavor: "affinity-group-" + "pro",
69-
Namespace: namespace.Name,
70-
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
71-
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
72-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
73-
WorkerMachineCount: pointer.Int64Ptr(1),
74-
},
75-
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
76-
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
77-
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
78-
}, clusterResources)
79-
80-
CheckAffinityGroup(clusterResources.Cluster.Name, "pro")
81-
82-
By("PASSED!")
61+
executeTest(ctx, input, namespace, specName, clusterResources, "pro")
8362
})
8463

8564
It("Should have host affinity group when affinity is anti", func() {
86-
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
87-
ClusterProxy: input.BootstrapClusterProxy,
88-
CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
89-
ConfigCluster: clusterctl.ConfigClusterInput{
90-
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
91-
ClusterctlConfigPath: input.ClusterctlConfigPath,
92-
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
93-
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
94-
Flavor: "affinity-group-" + "anti",
95-
Namespace: namespace.Name,
96-
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
97-
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
98-
ControlPlaneMachineCount: pointer.Int64Ptr(1),
99-
WorkerMachineCount: pointer.Int64Ptr(1),
100-
},
101-
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
102-
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
103-
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
104-
}, clusterResources)
105-
106-
CheckAffinityGroup(clusterResources.Cluster.Name, "anti")
107-
108-
By("PASSED!")
65+
executeTest(ctx, input, namespace, specName, clusterResources, "anti")
10966
})
11067

11168
AfterEach(func() {
11269
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
11370
dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
71+
72+
err := CheckAffinityGroupsDeleted(affinityIds)
73+
if err != nil {
74+
Fail(err.Error())
75+
}
11476
})
11577
}
78+
79+
func executeTest(ctx context.Context, input CommonSpecInput, namespace *corev1.Namespace, specName string, clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult, affinityType string) []string {
80+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
81+
ClusterProxy: input.BootstrapClusterProxy,
82+
CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
83+
ConfigCluster: clusterctl.ConfigClusterInput{
84+
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
85+
ClusterctlConfigPath: input.ClusterctlConfigPath,
86+
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
87+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
88+
Flavor: "affinity-group-" + affinityType,
89+
Namespace: namespace.Name,
90+
ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
91+
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
92+
ControlPlaneMachineCount: pointer.Int64Ptr(3),
93+
WorkerMachineCount: pointer.Int64Ptr(3),
94+
},
95+
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
96+
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
97+
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
98+
}, clusterResources)
99+
100+
affinityIds := CheckAffinityGroup(clusterResources.Cluster.Name, affinityType)
101+
102+
By("PASSED!")
103+
104+
return affinityIds
105+
}

‎test/e2e/affinity_group_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
)
2626

2727
var _ = Describe("When testing affinity group", func() {
28+
Skip("The ACS used by Prow doesn't have multiple hosts in the target zone")
2829

2930
AffinityGroupSpec(context.TODO(), func() CommonSpecInput {
3031
return CommonSpecInput{

‎test/e2e/common.go

Lines changed: 79 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package e2e
1919
import (
2020
"context"
2121
"encoding/base64"
22+
"errors"
2223
"fmt"
2324
"os"
2425
"path/filepath"
@@ -58,6 +59,11 @@ const (
5859
InvalidWorkerOfferingName = "CLOUDSTACK_INVALID_WORKER_MACHINE_OFFERING"
5960
)
6061

62+
const (
63+
ControlPlaneIndicator = "control-plane"
64+
MachineDeploymentIndicator = "md"
65+
)
66+
6167
type CommonSpecInput struct {
6268
E2EConfig *clusterctl.E2EConfig
6369
ClusterctlConfigPath string
@@ -191,23 +197,7 @@ type cloudConfig struct {
191197
}
192198

193199
func DestroyOneMachine(clusterName string, machineType string) {
194-
encodedSecret := os.Getenv("CLOUDSTACK_B64ENCODED_SECRET")
195-
secret, err := base64.StdEncoding.DecodeString(encodedSecret)
196-
if err != nil {
197-
Fail("Failed ")
198-
}
199-
cfg := &cloudConfig{VerifySSL: true}
200-
if rawCfg, err := ini.Load(secret); err != nil {
201-
Fail("Failed to load INI file")
202-
} else if g := rawCfg.Section("Global"); len(g.Keys()) == 0 {
203-
Fail("Global section not found")
204-
} else if err = rawCfg.Section("Global").StrictMapTo(cfg); err != nil {
205-
Fail("Error encountered while parsing Global section")
206-
}
207-
208-
By("Creating a CloudStack client")
209-
client := cloudstack.NewAsyncClient(cfg.APIURL, cfg.APIKey, cfg.SecretKey, cfg.VerifySSL)
210-
200+
client := createCloudStackClient()
211201
matcher := clusterName + "-" + machineType
212202

213203
Byf("Listing machines with %q", matcher)
@@ -241,33 +231,41 @@ func DestroyOneMachine(clusterName string, machineType string) {
241231
}
242232
}
243233

244-
func CheckAffinityGroup(clusterName string, affinityType string) {
245-
encodedSecret := os.Getenv("CLOUDSTACK_B64ENCODED_SECRET")
246-
secret, err := base64.StdEncoding.DecodeString(encodedSecret)
247-
if err != nil {
248-
Fail("Failed ")
249-
}
250-
cfg := &cloudConfig{VerifySSL: true}
251-
if rawCfg, err := ini.Load(secret); err != nil {
252-
Fail("Failed to load INI file")
253-
} else if g := rawCfg.Section("Global"); len(g.Keys()) == 0 {
254-
Fail("Global section not found")
255-
} else if err = rawCfg.Section("Global").StrictMapTo(cfg); err != nil {
256-
Fail("Error encountered while parsing Global section")
234+
func CheckAffinityGroupsDeleted(affinityIds []string) error {
235+
client := createCloudStackClient()
236+
237+
for _, affinityId := range affinityIds {
238+
affinity, count, _ := client.AffinityGroup.GetAffinityGroupByID(affinityId)
239+
if count > 0 {
240+
return errors.New("Affinity group " + affinity.Name + " still exists")
241+
}
257242
}
243+
return nil
244+
}
258245

259-
By("Creating a CloudStack client")
260-
client := cloudstack.NewAsyncClient(cfg.APIURL, cfg.APIKey, cfg.SecretKey, cfg.VerifySSL)
246+
func CheckAffinityGroup(clusterName string, affinityType string) []string {
247+
client := createCloudStackClient()
261248

262249
By("Listing all machines")
263250
listResp, err := client.VirtualMachine.ListVirtualMachines(client.VirtualMachine.NewListVirtualMachinesParams())
264251
if err != nil {
265252
Fail("Failed to list machines")
266253
}
267254
affinityTypeString := strings.Title(fmt.Sprintf("%sAffinity", affinityType))
255+
cpHostIdSet := make(map[string]bool)
256+
mdHostIdSet := make(map[string]bool)
257+
affinityIds := []string{}
258+
268259
for _, vm := range listResp.VirtualMachines {
269260
if strings.Contains(vm.Name, clusterName) {
261+
By(vm.Name + " is in host " + vm.Hostname + " (" + vm.Hostid + ")")
262+
err := checkVMHostAssignments(vm, cpHostIdSet, mdHostIdSet, affinityType)
263+
if err != nil {
264+
Fail(err.Error())
265+
}
266+
270267
for _, affinity := range vm.Affinitygroup {
268+
affinityIds = append(affinityIds, affinity.Id)
271269
affinity, _, _ := client.AffinityGroup.GetAffinityGroupByID(affinity.Id)
272270
if err != nil {
273271
Fail("Failed to get affinity group for " + affinity.Id)
@@ -284,6 +282,55 @@ func CheckAffinityGroup(clusterName string, affinityType string) {
284282
}
285283
}
286284
}
285+
return affinityIds
286+
}
287+
288+
func createCloudStackClient() *cloudstack.CloudStackClient {
289+
encodedSecret := os.Getenv("CLOUDSTACK_B64ENCODED_SECRET")
290+
secret, err := base64.StdEncoding.DecodeString(encodedSecret)
291+
if err != nil {
292+
Fail("Failed ")
293+
}
294+
cfg := &cloudConfig{VerifySSL: true}
295+
if rawCfg, err := ini.Load(secret); err != nil {
296+
Fail("Failed to load INI file")
297+
} else if g := rawCfg.Section("Global"); len(g.Keys()) == 0 {
298+
Fail("Global section not found")
299+
} else if err = rawCfg.Section("Global").StrictMapTo(cfg); err != nil {
300+
Fail("Error encountered while parsing Global section")
301+
}
302+
303+
By("Creating a CloudStack client")
304+
client := cloudstack.NewAsyncClient(cfg.APIURL, cfg.APIKey, cfg.SecretKey, cfg.VerifySSL)
305+
return client
306+
}
307+
308+
func checkVMHostAssignments(vm *cloudstack.VirtualMachine, cpHostIdSet map[string]bool, mdHostIdSet map[string]bool, affinityType string) error {
309+
if strings.Contains(vm.Name, ControlPlaneIndicator) {
310+
if len(cpHostIdSet) > 0 {
311+
_, ok := cpHostIdSet[vm.Hostid]
312+
if affinityType == "pro" && !ok {
313+
return errors.New(vm.Name + " is deployed in a different host: " + vm.Hostname + " when affinity type is " + affinityType)
314+
}
315+
if affinityType == "anti" && ok {
316+
return errors.New(vm.Name + " is deployed in the same host: " + vm.Hostname + " when affinity type is " + affinityType)
317+
}
318+
}
319+
cpHostIdSet[vm.Hostid] = true
320+
}
321+
if strings.Contains(vm.Name, MachineDeploymentIndicator) {
322+
if len(mdHostIdSet) > 0 {
323+
_, ok := mdHostIdSet[vm.Hostid]
324+
if affinityType == "pro" && !ok {
325+
return errors.New(vm.Name + " is deployed in a different host: " + vm.Hostname + " when affinity type is " + affinityType)
326+
}
327+
if affinityType == "anti" && ok {
328+
return errors.New(vm.Name + " is deployed in the same host: " + vm.Hostname + " when affinity type is " + affinityType)
329+
}
330+
}
331+
mdHostIdSet[vm.Hostid] = true
332+
}
333+
return nil
287334
}
288335

289336
func WaitForMachineRemediationAfterDestroy(ctx context.Context, proxy framework.ClusterProxy, cluster *clusterv1.Cluster, machineMatcher string, healthyMachineCount int, intervals []interface{}) {

‎test/e2e/machine_remediation.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,9 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() CommonSpecIn
5757
})
5858

5959
It("Should replace a machine when it is destroyed", func() {
60-
cpMatcher := "control-plane"
60+
cpMatcher := ControlPlaneIndicator
6161
cpCount := 3
62-
mdMatcher := "md"
62+
mdMatcher := MachineDeploymentIndicator
6363
mdCount := 3
6464
proxy := input.BootstrapClusterProxy
6565

0 commit comments

Comments
 (0)