Skip to content

Commit 0dc13b1

Browse files
mdboothmdbooth
committed
Add Multi-AZ test
1 parent 5069c3e commit 0dc13b1

File tree

7 files changed

+369
-16
lines changed

7 files changed

+369
-16
lines changed

‎test/e2e/data/e2e_conf.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,3 +121,4 @@ intervals:
121121
default/wait-control-plane: ["30m", "10s"]
122122
default/wait-worker-nodes: ["30m", "10s"]
123123
default/wait-delete-cluster: ["5m", "10s"]
124+
default/wait-alt-az: ["20m", "30s"]

‎test/e2e/data/infrastructure-openstack/cluster-template-multi-az.yaml‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1alpha4
3+
kind: Cluster
4+
metadata:
5+
name: ${CLUSTER_NAME}
6+
labels:
7+
cni: "${CLUSTER_NAME}-crs-0"
8+
spec:
9+
clusterNetwork:
10+
pods:
11+
cidrBlocks: ["192.168.0.0/16"] # CIDR block used by Calico.
12+
serviceDomain: "cluster.local"
13+
infrastructureRef:
14+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
15+
kind: OpenStackCluster
16+
name: ${CLUSTER_NAME}
17+
controlPlaneRef:
18+
kind: KubeadmControlPlane
19+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
20+
name: ${CLUSTER_NAME}-control-plane
21+
---
22+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
23+
kind: OpenStackCluster
24+
metadata:
25+
name: ${CLUSTER_NAME}
26+
spec:
27+
cloudName: ${OPENSTACK_CLOUD}
28+
identityRef:
29+
name: ${CLUSTER_NAME}-cloud-config
30+
kind: Secret
31+
controlPlaneAvailabilityZones:
32+
- ${OPENSTACK_FAILURE_DOMAIN}
33+
- ${OPENSTACK_FAILURE_DOMAIN_ALT}
34+
managedAPIServerLoadBalancer: true
35+
managedSecurityGroups: true
36+
nodeCidr: 10.6.0.0/24
37+
dnsNameservers:
38+
- ${OPENSTACK_DNS_NAMESERVERS}
39+
bastion:
40+
enabled: true
41+
instance:
42+
flavor: ${OPENSTACK_BASTION_MACHINE_FLAVOR}
43+
image: ${OPENSTACK_BASTION_IMAGE_NAME}
44+
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
45+
externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID}
46+
---
47+
kind: KubeadmControlPlane
48+
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
49+
metadata:
50+
name: "${CLUSTER_NAME}-control-plane"
51+
spec:
52+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
53+
machineTemplate:
54+
infrastructureRef:
55+
kind: OpenStackMachineTemplate
56+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
57+
name: "${CLUSTER_NAME}-control-plane"
58+
kubeadmConfigSpec:
59+
initConfiguration:
60+
nodeRegistration:
61+
name: '{{ local_hostname }}'
62+
kubeletExtraArgs:
63+
cloud-provider: openstack
64+
cloud-config: /etc/kubernetes/cloud.conf
65+
clusterConfiguration:
66+
imageRepository: k8s.gcr.io
67+
apiServer:
68+
extraArgs:
69+
cloud-provider: openstack
70+
cloud-config: /etc/kubernetes/cloud.conf
71+
extraVolumes:
72+
- name: cloud
73+
hostPath: /etc/kubernetes/cloud.conf
74+
mountPath: /etc/kubernetes/cloud.conf
75+
readOnly: true
76+
controllerManager:
77+
extraArgs:
78+
cloud-provider: openstack
79+
cloud-config: /etc/kubernetes/cloud.conf
80+
extraVolumes:
81+
- name: cloud
82+
hostPath: /etc/kubernetes/cloud.conf
83+
mountPath: /etc/kubernetes/cloud.conf
84+
readOnly: true
85+
- name: cacerts
86+
hostPath: /etc/certs/cacert
87+
mountPath: /etc/certs/cacert
88+
readOnly: true
89+
joinConfiguration:
90+
nodeRegistration:
91+
name: '{{ local_hostname }}'
92+
kubeletExtraArgs:
93+
cloud-config: /etc/kubernetes/cloud.conf
94+
cloud-provider: openstack
95+
files:
96+
- path: /etc/kubernetes/cloud.conf
97+
owner: root
98+
permissions: "0600"
99+
content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64}
100+
encoding: base64
101+
- path: /etc/certs/cacert
102+
owner: root
103+
permissions: "0600"
104+
content: ${OPENSTACK_CLOUD_CACERT_B64}
105+
encoding: base64
106+
version: "${KUBERNETES_VERSION}"
107+
---
108+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
109+
kind: OpenStackMachineTemplate
110+
metadata:
111+
name: ${CLUSTER_NAME}-control-plane
112+
spec:
113+
template:
114+
spec:
115+
flavor: ${OPENSTACK_CONTROL_PLANE_MACHINE_FLAVOR}
116+
image: ${OPENSTACK_IMAGE_NAME}
117+
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
118+
cloudName: ${OPENSTACK_CLOUD}
119+
identityRef:
120+
name: ${CLUSTER_NAME}-cloud-config
121+
kind: Secret
122+
---
123+
apiVersion: cluster.x-k8s.io/v1alpha4
124+
kind: MachineDeployment
125+
metadata:
126+
name: "${CLUSTER_NAME}-md-0"
127+
spec:
128+
clusterName: "${CLUSTER_NAME}"
129+
replicas: ${WORKER_MACHINE_COUNT}
130+
selector:
131+
matchLabels:
132+
template:
133+
spec:
134+
clusterName: "${CLUSTER_NAME}"
135+
version: "${KUBERNETES_VERSION}"
136+
failureDomain: ${OPENSTACK_FAILURE_DOMAIN_ALT}
137+
bootstrap:
138+
configRef:
139+
name: "${CLUSTER_NAME}-md-0"
140+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
141+
kind: KubeadmConfigTemplate
142+
infrastructureRef:
143+
name: "${CLUSTER_NAME}-md-0"
144+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
145+
kind: OpenStackMachineTemplate
146+
---
147+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
148+
kind: OpenStackMachineTemplate
149+
metadata:
150+
name: ${CLUSTER_NAME}-md-0
151+
spec:
152+
template:
153+
spec:
154+
cloudName: ${OPENSTACK_CLOUD}
155+
identityRef:
156+
name: ${CLUSTER_NAME}-cloud-config
157+
kind: Secret
158+
flavor: ${OPENSTACK_NODE_MACHINE_FLAVOR}
159+
image: ${OPENSTACK_IMAGE_NAME}
160+
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
161+
---
162+
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
163+
kind: KubeadmConfigTemplate
164+
metadata:
165+
name: ${CLUSTER_NAME}-md-0
166+
spec:
167+
template:
168+
spec:
169+
files:
170+
- content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64}
171+
encoding: base64
172+
owner: root
173+
path: /etc/kubernetes/cloud.conf
174+
permissions: "0600"
175+
- content: ${OPENSTACK_CLOUD_CACERT_B64}
176+
encoding: base64
177+
owner: root
178+
path: /etc/certs/cacert
179+
permissions: "0600"
180+
joinConfiguration:
181+
nodeRegistration:
182+
name: '{{ local_hostname }}'
183+
kubeletExtraArgs:
184+
cloud-config: /etc/kubernetes/cloud.conf
185+
cloud-provider: openstack
186+
---
187+
apiVersion: v1
188+
kind: Secret
189+
metadata:
190+
name: ${CLUSTER_NAME}-cloud-config
191+
labels:
192+
clusterctl.cluster.x-k8s.io/move: "true"
193+
data:
194+
clouds.yaml: ${OPENSTACK_CLOUD_YAML_B64}
195+
cacert: ${OPENSTACK_CLOUD_CACERT_B64}
196+
---
197+
apiVersion: v1
198+
kind: ConfigMap
199+
metadata:
200+
name: "cni-${CLUSTER_NAME}-crs-0"
201+
data: ${CNI_RESOURCES}
202+
---
203+
apiVersion: addons.cluster.x-k8s.io/v1alpha4
204+
kind: ClusterResourceSet
205+
metadata:
206+
name: "${CLUSTER_NAME}-crs-0"
207+
spec:
208+
strategy: ApplyOnce
209+
clusterSelector:
210+
matchLabels:
211+
cni: "${CLUSTER_NAME}-crs-0"
212+
resources:
213+
- name: "cni-${CLUSTER_NAME}-crs-0"
214+
kind: ConfigMap

‎test/e2e/shared/common.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ func dumpMachines(ctx context.Context, e2eCtx *E2EContext, namespace *corev1.Nam
9595
_, _ = fmt.Fprintf(GinkgoWriter, "cannot dump machines, could not get machines: %v\n", err)
9696
return
9797
}
98-
srvs, err := getOpenStackServers(e2eCtx, cluster)
98+
srvs, err := GetOpenStackServers(e2eCtx, cluster)
9999
if err != nil {
100100
_, _ = fmt.Fprintf(GinkgoWriter, "cannot dump machines, could not get servers from OpenStack: %v\n", err)
101101
return
@@ -129,7 +129,7 @@ func machinesForSpec(ctx context.Context, clusterProxy framework.ClusterProxy, n
129129
return list, nil
130130
}
131131

132-
func dumpMachine(ctx context.Context, e2eCtx *E2EContext, machine infrav1.OpenStackMachine, srv server, bastionIP string) {
132+
func dumpMachine(ctx context.Context, e2eCtx *E2EContext, machine infrav1.OpenStackMachine, srv ServerExtWithIP, bastionIP string) {
133133
logPath := filepath.Join(e2eCtx.Settings.ArtifactFolder, "clusters", e2eCtx.Environment.BootstrapClusterProxy.GetName())
134134
machineLogBase := path.Join(logPath, "instances", machine.Namespace, machine.Name)
135135
metaLog := path.Join(machineLogBase, "instance.log")
@@ -151,7 +151,7 @@ func dumpMachine(ctx context.Context, e2eCtx *E2EContext, machine infrav1.OpenSt
151151
_, _ = fmt.Fprintf(GinkgoWriter, "error writing server JSON %s: %s", serverJSON, err)
152152
}
153153

154-
_, _ = fmt.Fprintf(f, "instance found: %q\n", srv.id)
154+
_, _ = fmt.Fprintf(f, "instance found: %q\n", srv.ID)
155155
executeCommands(
156156
ctx,
157157
e2eCtx.Settings.ArtifactFolder,

‎test/e2e/shared/defaults.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ const (
4545
FlavorWithoutLB = "without-lb-ci-artifacts"
4646
FlavorExternalCloudProvider = "external-cloud-provider-ci-artifacts"
4747
FlavorMultiNetwork = "multi-network-ci-artifacts"
48+
FlavorMultiAZ = "multi-az-ci-artifacts"
4849
)
4950

5051
// DefaultScheme returns the default scheme to use for testing.

‎test/e2e/shared/exec.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,6 @@ import (
3434
"golang.org/x/crypto/ssh"
3535
)
3636

37-
type server struct {
38-
name string
39-
id string
40-
ip string
41-
}
42-
4337
type command struct {
4438
title string
4539
cmd string

‎test/e2e/shared/openstack.go‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/go-logr/logr"
3434
"github.com/gophercloud/gophercloud"
3535
"github.com/gophercloud/gophercloud/openstack"
36+
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones"
3637
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/keypairs"
3738
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
3839
"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
@@ -50,6 +51,11 @@ import (
5051
"sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/provider"
5152
)
5253

54+
type ServerExtWithIP struct {
55+
compute.ServerExt
56+
ip string
57+
}
58+
5359
// ensureSSHKeyPair ensures A SSH key is present under the name.
5460
func ensureSSHKeyPair(e2eCtx *E2EContext) {
5561
Byf("Ensuring presence of SSH key %q in OpenStack", DefaultSSHKeyPairName)
@@ -154,9 +160,9 @@ func DumpOpenStackPorts(e2eCtx *E2EContext, filter ports.ListOpts) ([]ports.Port
154160
return portsList, nil
155161
}
156162

157-
// getOpenStackServers gets all OpenStack servers at once, to save on DescribeInstances
163+
// GetOpenStackServers gets all OpenStack servers at once, to save on DescribeInstances
158164
// calls.
159-
func getOpenStackServers(e2eCtx *E2EContext, openStackCluster *infrav1.OpenStackCluster) (map[string]server, error) {
165+
func GetOpenStackServers(e2eCtx *E2EContext, openStackCluster *infrav1.OpenStackCluster) (map[string]ServerExtWithIP, error) {
160166
providerClient, clientOpts, err := getProviderClient(e2eCtx)
161167
if err != nil {
162168
_, _ = fmt.Fprintf(GinkgoWriter, "error creating provider client: %s\n", err)
@@ -180,7 +186,7 @@ func getOpenStackServers(e2eCtx *E2EContext, openStackCluster *infrav1.OpenStack
180186
return nil, fmt.Errorf("error extracting server: %v", err)
181187
}
182188

183-
srvs := map[string]server{}
189+
srvs := map[string]ServerExtWithIP{}
184190
for i := range serverList {
185191
srv := &serverList[i]
186192
instanceStatus := compute.NewInstanceStatusFromServer(srv, logr.Discard())
@@ -195,10 +201,9 @@ func getOpenStackServers(e2eCtx *E2EContext, openStackCluster *infrav1.OpenStack
195201
continue
196202
}
197203

198-
srvs[srv.Name] = server{
199-
name: srv.Name,
200-
id: srv.ID,
201-
ip: ip,
204+
srvs[srv.Name] = ServerExtWithIP{
205+
ServerExt: *srv,
206+
ip: ip,
202207
}
203208
}
204209
return srvs, nil
@@ -326,6 +331,28 @@ func getOpenStackCloudYAML(cloudYAML string) []byte {
326331
return cloudYAMLContent
327332
}
328333

334+
func GetComputeAvailabilityZones(e2eCtx *E2EContext) []string {
335+
providerClient, clientOpts, err := getProviderClient(e2eCtx)
336+
Expect(err).NotTo(HaveOccurred())
337+
338+
computeClient, err := openstack.NewComputeV2(providerClient, gophercloud.EndpointOpts{Region: clientOpts.RegionName})
339+
Expect(err).NotTo(HaveOccurred())
340+
341+
allPages, err := availabilityzones.List(computeClient).AllPages()
342+
Expect(err).NotTo(HaveOccurred())
343+
availabilityZoneList, err := availabilityzones.ExtractAvailabilityZones(allPages)
344+
Expect(err).NotTo(HaveOccurred())
345+
346+
var azs []string
347+
for _, az := range availabilityZoneList {
348+
if az.ZoneState.Available {
349+
azs = append(azs, az.ZoneName)
350+
}
351+
}
352+
353+
return azs
354+
}
355+
329356
func CreateOpenStackNetwork(e2eCtx *E2EContext, name, cidr string) (*networks.Network, error) {
330357
providerClient, clientOpts, err := getProviderClient(e2eCtx)
331358
if err != nil {

0 commit comments

Comments
 (0)