Skip to content

Commit 7a1c583

Browse files
wongniwongni
committed
Make more e2e tests pass
1 parent b7187fc commit 7a1c583

29 files changed

+231
-213
lines changed

‎test/e2e/affinity_group.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ func AffinityGroupSpec(ctx context.Context, inputGetter func() CommonSpecInput)
6969
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
7070
dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
7171

72-
err := CheckAffinityGroupsDeleted(affinityIds)
72+
csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
73+
err := CheckAffinityGroupsDeleted(csClient, affinityIds)
7374
if err != nil {
7475
Fail(err.Error())
7576
}
@@ -98,5 +99,6 @@ func executeTest(ctx context.Context, input CommonSpecInput, namespace *corev1.N
9899
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
99100
}, clusterResources)
100101

101-
return CheckAffinityGroup(clusterResources.Cluster.Name, affinityType)
102+
csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
103+
return CheckAffinityGroup(csClient, clusterResources.Cluster.Name, affinityType)
102104
}

‎test/e2e/common.go

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,9 @@ package e2e
1818

1919
import (
2020
"context"
21-
"encoding/base64"
21+
"encoding/json"
2222
"errors"
2323
"fmt"
24-
"os"
2524
"path/filepath"
2625
"strings"
2726
"time"
@@ -31,11 +30,11 @@ import (
3130
"github.com/apache/cloudstack-go/v2/cloudstack"
3231
"github.com/blang/semver"
3332
. "github.com/onsi/ginkgo"
34-
"gopkg.in/ini.v1"
3533
corev1 "k8s.io/api/core/v1"
3634

3735
. "github.com/onsi/gomega"
3836
"github.com/onsi/gomega/types"
37+
"k8s.io/apimachinery/pkg/runtime"
3938
k8stypes "k8s.io/apimachinery/pkg/types"
4039
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
4140
"sigs.k8s.io/cluster-api/test/framework"
@@ -166,6 +165,22 @@ func KubectlExec(ctx context.Context, command string, kubeconfigPath string, arg
166165
return string(stdout), nil
167166
}
168167

168+
func GetK8sObject(ctx context.Context, resourceType, name, namespace, kubeconfigPath string, obj runtime.Object) error {
169+
getArgs := []string{"--ignore-not-found", "--namespace", namespace, resourceType, name, "-o", "json"}
170+
stdOut, err := KubectlExec(ctx, "get", kubeconfigPath, getArgs...)
171+
if err != nil {
172+
return fmt.Errorf("getting %s/%s/%s with kubectl: %v", resourceType, namespace, name, err)
173+
}
174+
if len(stdOut) == 0 {
175+
return fmt.Errorf("not found %s/%s/%s", resourceType, namespace, name)
176+
}
177+
if err = json.Unmarshal([]byte(stdOut), obj); err != nil {
178+
return fmt.Errorf("parsing %s/%s/%s response: %v", resourceType, namespace, name, err)
179+
}
180+
181+
return nil
182+
}
183+
169184
func DeployAppToWorkloadClusterAndWaitForDeploymentReady(ctx context.Context, workloadKubeconfigPath string, appName string, appConfigLink string, timeout int) error {
170185
applyArgs := []string{
171186
"-f", appConfigLink,
@@ -230,15 +245,7 @@ func DownloadMetricsFromCAPCManager(ctx context.Context, bootstrapKubeconfigPath
230245
return result, nil
231246
}
232247

233-
type cloudConfig struct {
234-
APIURL string `ini:"api-url"`
235-
APIKey string `ini:"api-key"`
236-
SecretKey string `ini:"secret-key"`
237-
VerifySSL bool `ini:"verify-ssl"`
238-
}
239-
240-
func DestroyOneMachine(clusterName string, machineType string) {
241-
client := createCloudStackClient()
248+
func DestroyOneMachine(client *cloudstack.CloudStackClient, clusterName string, machineType string) {
242249
matcher := clusterName + "-" + machineType
243250

244251
Byf("Listing machines with %q", matcher)
@@ -266,13 +273,11 @@ func DestroyOneMachine(clusterName string, machineType string) {
266273
}
267274
}
268275

269-
func CheckAffinityGroupsDeleted(affinityIds []string) error {
276+
func CheckAffinityGroupsDeleted(client *cloudstack.CloudStackClient, affinityIds []string) error {
270277
if len(affinityIds) == 0 {
271278
return errors.New("affinityIds are empty")
272279
}
273280

274-
client := createCloudStackClient()
275-
276281
for _, affinityId := range affinityIds {
277282
affinity, count, _ := client.AffinityGroup.GetAffinityGroupByID(affinityId)
278283
if count > 0 {
@@ -282,9 +287,7 @@ func CheckAffinityGroupsDeleted(affinityIds []string) error {
282287
return nil
283288
}
284289

285-
func CheckAffinityGroup(clusterName string, affinityType string) []string {
286-
client := createCloudStackClient()
287-
290+
func CheckAffinityGroup(client *cloudstack.CloudStackClient, clusterName string, affinityType string) []string {
288291
By("Listing all machines")
289292
p := client.VirtualMachine.NewListVirtualMachinesParams()
290293
p.SetListall(true)
@@ -326,39 +329,39 @@ func CheckAffinityGroup(clusterName string, affinityType string) []string {
326329
return affinityIds
327330
}
328331

329-
func CheckNetworkExists(networkName string) (bool, error) {
330-
client := createCloudStackClient()
331-
332+
func CheckNetworkExists(client *cloudstack.CloudStackClient, networkName string) (bool, error) {
332333
_, count, err := client.Network.GetNetworkByName(networkName)
333334
if err != nil {
334335
if strings.Contains(err.Error(), "No match found for") {
335336
return false, nil
336337
}
337338
return false, err
338339
} else if count > 1 {
339-
return false, errors.New(fmt.Sprintf("Expected 0-1 Network with name %s, but got %d.", networkName, count))
340+
return false, fmt.Errorf("Expected 0-1 Network with name %s, but got %d.", networkName, count)
340341
}
341342
return count == 1, nil
342343
}
343344

344-
func createCloudStackClient() *cloudstack.CloudStackClient {
345-
encodedSecret := os.Getenv("CLOUDSTACK_B64ENCODED_SECRET")
346-
secret, err := base64.StdEncoding.DecodeString(encodedSecret)
347-
if err != nil {
348-
Fail("Failed to decode: " + err.Error())
349-
}
350-
cfg := &cloudConfig{VerifySSL: true}
351-
if rawCfg, err := ini.Load(secret); err != nil {
352-
Fail("Failed to load INI file: " + err.Error())
353-
} else if g := rawCfg.Section("Global"); len(g.Keys()) == 0 {
354-
Fail("Global section not found")
355-
} else if err = rawCfg.Section("Global").StrictMapTo(cfg); err != nil {
356-
Fail("Error encountered while parsing Global section")
345+
func CreateCloudStackClient(ctx context.Context, kubeConfigPath string) *cloudstack.CloudStackClient {
346+
By("Getting a CloudStack client secret")
347+
secret := &corev1.Secret{}
348+
name := "secret1"
349+
namepace := "default"
350+
if err := GetK8sObject(ctx, "secret", name, namepace, kubeConfigPath, secret); err != nil {
351+
Fail("Failed to get secret: " + err.Error())
357352
}
358353

359354
By("Creating a CloudStack client")
360-
client := cloudstack.NewAsyncClient(cfg.APIURL, cfg.APIKey, cfg.SecretKey, cfg.VerifySSL)
361-
return client
355+
apiURL := string(secret.Data["api-url"])
356+
apiKey := string(secret.Data["api-key"])
357+
secretKey := string(secret.Data["secret-key"])
358+
verifySSL := string(secret.Data["verify-ssl"])
359+
if apiURL == "" || apiKey == "" || secretKey == "" {
360+
Fail(fmt.Sprintf("Invalid secret: %+v, %s, %s, %s", secret.Data, apiURL, apiKey, secretKey))
361+
}
362+
fmt.Sprintf("from secret: %s, %s, %s", apiURL, apiKey, secretKey)
363+
364+
return cloudstack.NewClient(apiURL, apiKey, secretKey, strings.ToLower(verifySSL) == "true")
362365
}
363366

364367
func checkVMHostAssignments(vm *cloudstack.VirtualMachine, cpHostIdSet map[string]bool, mdHostIdSet map[string]bool, affinityType string) error {
@@ -397,7 +400,8 @@ func WaitForMachineRemediationAfterDestroy(ctx context.Context, proxy framework.
397400
Byf("Current number of healthy %s is %d", machineMatcher, healthyMachineCount)
398401

399402
Byf("Destroying one %s", machineMatcher)
400-
DestroyOneMachine(cluster.Name, machineMatcher)
403+
csClient := CreateCloudStackClient(ctx, proxy.GetKubeconfigPath())
404+
DestroyOneMachine(csClient, cluster.Name, machineMatcher)
401405

402406
Byf("Waiting for the destroyed %s to be unhealthy", machineMatcher)
403407
WaitForHealthyMachineCount(ctx, mgmtClusterClient, workloadClusterClient, cluster, machineMatcher, healthyMachineCount-1, intervals)
@@ -480,9 +484,7 @@ func IsClusterReady(ctx context.Context, mgmtClient client.Client, cluster *clus
480484
return c.Status.ControlPlaneReady && c.Status.InfrastructureReady
481485
}
482486

483-
func CheckDiskOfferingOfVmInstances(clusterName string, diskOfferingName string) {
484-
client := createCloudStackClient()
485-
487+
func CheckDiskOfferingOfVmInstances(client *cloudstack.CloudStackClient, clusterName string, diskOfferingName string) {
486488
Byf("Listing machines with %q", clusterName)
487489
listResp, err := client.VirtualMachine.ListVirtualMachines(client.VirtualMachine.NewListVirtualMachinesParams())
488490
if err != nil {
@@ -494,9 +496,7 @@ func CheckDiskOfferingOfVmInstances(clusterName string, diskOfferingName string)
494496
}
495497
}
496498
}
497-
func CheckVolumeSizeofVmInstances(clusterName string, volumeSize int64) {
498-
client := createCloudStackClient()
499-
499+
func CheckVolumeSizeofVmInstances(client *cloudstack.CloudStackClient, clusterName string, volumeSize int64) {
500500
Byf("Listing machines with %q", clusterName)
501501
listResp, err := client.VirtualMachine.ListVirtualMachines(client.VirtualMachine.NewListVirtualMachinesParams())
502502
if err != nil {

‎test/e2e/config/cloudstack.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ variables:
114114

115115
CLOUDSTACK_FD1_NAME: "fd1"
116116
CLOUDSTACK_FD1_SECRET_NAME: "secret1"
117+
CLOUDSTACK_FD1_SECRET_NAMESPACE: "default"
117118
CLOUDSTACK_ZONE_NAME: zone1
118119
CLOUDSTACK_INVALID_ZONE_NAME: zoneXXXX
119120
CLOUDSTACK_INVALID_NETWORK_NAME: networkXXXX

‎test/e2e/custom_disk_offering.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,9 @@ func CustomDiskOfferingSpec(ctx context.Context, inputGetter func() CommonSpecIn
8181
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
8282
}, clusterResources)
8383

84-
CheckDiskOfferingOfVmInstances(clusterResources.Cluster.Name, diskOfferingName)
85-
CheckVolumeSizeofVmInstances(clusterResources.Cluster.Name, volumeSize)
84+
csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
85+
CheckDiskOfferingOfVmInstances(csClient, clusterResources.Cluster.Name, diskOfferingName)
86+
CheckVolumeSizeofVmInstances(csClient, clusterResources.Cluster.Name, volumeSize)
8687
By("PASSED!")
8788
})
8889

‎test/e2e/data/infrastructure-cloudstack/v1beta2/cluster-template-affinity-group-anti.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ spec:
88
joinConfiguration:
99
nodeRegistration:
1010
kubeletExtraArgs:
11-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
11+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
1212
name: '{{ local_hostname }}'
1313
preKubeadmCommands:
1414
- swapoff -a
@@ -66,12 +66,12 @@ spec:
6666
initConfiguration:
6767
nodeRegistration:
6868
kubeletExtraArgs:
69-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
69+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
7070
name: '{{ local_hostname }}'
7171
joinConfiguration:
7272
nodeRegistration:
7373
kubeletExtraArgs:
74-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
74+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
7575
name: '{{ local_hostname }}'
7676
preKubeadmCommands:
7777
- swapoff -a
@@ -92,10 +92,10 @@ spec:
9292
host: ""
9393
port: 6443
9494
failureDomains:
95-
- name: ${CLOUDSTACK_FD1_NAME}
96-
acsendpoint:
95+
- acsendpoint:
9796
name: ${CLOUDSTACK_FD1_SECRET_NAME}
9897
namespace: default
98+
name: ${CLOUDSTACK_FD1_NAME}
9999
zone:
100100
name: ${CLOUDSTACK_ZONE_NAME}
101101
network:

‎test/e2e/data/infrastructure-cloudstack/v1beta2/cluster-template-affinity-group-pro.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ spec:
88
joinConfiguration:
99
nodeRegistration:
1010
kubeletExtraArgs:
11-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
11+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
1212
name: '{{ local_hostname }}'
1313
preKubeadmCommands:
1414
- swapoff -a
@@ -66,12 +66,12 @@ spec:
6666
initConfiguration:
6767
nodeRegistration:
6868
kubeletExtraArgs:
69-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
69+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
7070
name: '{{ local_hostname }}'
7171
joinConfiguration:
7272
nodeRegistration:
7373
kubeletExtraArgs:
74-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
74+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
7575
name: '{{ local_hostname }}'
7676
preKubeadmCommands:
7777
- swapoff -a
@@ -92,10 +92,10 @@ spec:
9292
host: ""
9393
port: 6443
9494
failureDomains:
95-
- name: ${CLOUDSTACK_FD1_NAME}
96-
acsendpoint:
95+
- acsendpoint:
9796
name: ${CLOUDSTACK_FD1_SECRET_NAME}
9897
namespace: default
98+
name: ${CLOUDSTACK_FD1_NAME}
9999
zone:
100100
name: ${CLOUDSTACK_ZONE_NAME}
101101
network:

‎test/e2e/data/infrastructure-cloudstack/v1beta2/cluster-template-disk-offering.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ spec:
2222
joinConfiguration:
2323
nodeRegistration:
2424
kubeletExtraArgs:
25-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
25+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
2626
name: '{{ local_hostname }}'
2727
mounts:
2828
- - LABEL=${CLOUDSTACK_DISK_OFFERING_LABEL}
@@ -97,12 +97,12 @@ spec:
9797
initConfiguration:
9898
nodeRegistration:
9999
kubeletExtraArgs:
100-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
100+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
101101
name: '{{ local_hostname }}'
102102
joinConfiguration:
103103
nodeRegistration:
104104
kubeletExtraArgs:
105-
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
105+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
106106
name: '{{ local_hostname }}'
107107
mounts:
108108
- - LABEL=${CLOUDSTACK_DISK_OFFERING_LABEL}
@@ -126,10 +126,10 @@ spec:
126126
host: ""
127127
port: 6443
128128
failureDomains:
129-
- name: ${CLOUDSTACK_FD1_NAME}
130-
acsendpoint:
129+
- acsendpoint:
131130
name: ${CLOUDSTACK_FD1_SECRET_NAME}
132131
namespace: default
132+
name: ${CLOUDSTACK_FD1_NAME}
133133
zone:
134134
name: ${CLOUDSTACK_ZONE_NAME}
135135
network:

0 commit comments

Comments
 (0)