Skip to content

Commit 20db3dc

Browse files
wongniwongni
authored
Merge branch 'feature/multi-endpoint-failure-domains' into multiple-endpoints
2 parents 7de1351 + 61f6d61 commit 20db3dc

31 files changed

+712
-124
lines changed

‎Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ cluster-api/tilt-settings.json: hack/tilt-settings.json cluster-api
224224

225225
##@ End-to-End Testing
226226

227-
CLUSTER_TEMPLATES_INPUT_FILES=$(shell find test/e2e/data/infrastructure-cloudstack/*/cluster-template*/* test/e2e/data/infrastructure-cloudstack/*/bases/* -type f)
227+
CLUSTER_TEMPLATES_INPUT_FILES=$(shell find test/e2e/data/infrastructure-cloudstack/v1beta2/*/cluster-template*/* test/e2e/data/infrastructure-cloudstack/*/bases/* -type f)
228228
CLUSTER_TEMPLATES_OUTPUT_FILES=$(shell find test/e2e/data/infrastructure-cloudstack -type d -name "cluster-template*" -exec echo {}.yaml \;)
229229
.PHONY: e2e-cluster-templates
230230
e2e-cluster-templates: $(CLUSTER_TEMPLATES_OUTPUT_FILES) ## Generate cluster template files for e2e testing.
@@ -236,6 +236,7 @@ e2e-essentials: bin/ginkgo_v1 e2e-cluster-templates kind-cluster ## Fulfill esse
236236

237237
JOB ?= .*
238238
run-e2e: e2e-essentials ## Run e2e testing. JOB is an optional REGEXP to select certainn test cases to run. e.g. JOB=PR-Blocking, JOB=Conformance
239+
kubectl apply -f cloud-config.yaml && \
239240
cd test/e2e && \
240241
ginkgo_v1 -v -trace -tags=e2e -focus=$(JOB) -skip=Conformance -nodes=1 -noColor=false ./... -- \
241242
-e2e.artifacts-folder=${PROJECT_DIR}/_artifacts \

‎api/v1beta2/cloudstackcluster_webhook.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
"k8s.io/apimachinery/pkg/api/errors"
2323
"k8s.io/apimachinery/pkg/runtime"
24+
"k8s.io/apimachinery/pkg/util/validation"
2425
"k8s.io/apimachinery/pkg/util/validation/field"
2526
"sigs.k8s.io/cluster-api-provider-cloudstack/pkg/webhookutil"
2627
ctrl "sigs.k8s.io/controller-runtime"
@@ -61,15 +62,19 @@ func (r *CloudStackCluster) ValidateCreate() error {
6162
if len(r.Spec.FailureDomains) <= 0 {
6263
errorList = append(errorList, field.Required(field.NewPath("spec", "FailureDomains"), "FailureDomains"))
6364
} else {
64-
for _, fdSpec := range r.Spec.FailureDomains {
65+
for _, fdSpec := range r.Spec.FailureDomains { // Require failureDomain names meet k8s qualified name spec.
66+
for _, errMsg := range validation.IsDNS1123Subdomain(fdSpec.Name) {
67+
errorList = append(errorList, field.Invalid(
68+
field.NewPath("spec", "failureDomains", "name"), fdSpec.Name, errMsg))
69+
}
6570
if fdSpec.Zone.Network.Name == "" && fdSpec.Zone.Network.ID == "" {
6671
errorList = append(errorList, field.Required(
67-
field.NewPath("spec", "FailureDomains", "Zone", "Network"),
72+
field.NewPath("spec", "failureDomains", "Zone", "Network"),
6873
"each Zone requires a Network specification"))
6974
}
7075
if fdSpec.ACSEndpoint.Name == "" || fdSpec.ACSEndpoint.Namespace == "" {
7176
errorList = append(errorList, field.Required(
72-
field.NewPath("spec", "FailureDomains", "ACSEndpoint"),
77+
field.NewPath("spec", "failureDomains", "ACSEndpoint"),
7378
"Name and Namespace are required"))
7479
}
7580
}

‎controllers/cloudstackfailuredomain_controller.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ func (r *CloudStackFailureDomainReconciliationRunner) Reconcile() (retRes ctrl.R
8484
!csCtrlrUtils.ContainsNoMatchSubstring(err) {
8585
return ctrl.Result{}, errors.Wrap(err, "resolving Cloudstack network information")
8686
}
87+
// Patch Zone ID back before potentially creating an isolated network.
88+
if err := r.Patcher.Patch(r.RequestCtx, r.ReconciliationSubject); err != nil {
89+
return ctrl.Result{}, err
90+
}
8791

8892
// Check if the passed network was an isolated network or the network was missing. In either case, create a
8993
// CloudStackIsolatedNetwork to manage the many intricacies and wait until CloudStackIsolatedNetwork is ready.

‎controllers/utils/failuredomains.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package utils
1818

1919
import (
20+
"fmt"
2021
"strings"
2122

2223
corev1 "k8s.io/api/core/v1"
@@ -85,10 +86,11 @@ func (r *ReconciliationRunner) RemoveExtraneousFailureDomains(fds *infrav1.Cloud
8586
fdPresenceByName[name] = true
8687
}
8788

88-
// Send a deletion request for each FailureDomain no speced for.
89+
// Send a deletion request for each FailureDomain not speced for.
8990
for _, fd := range fds.Items {
9091
if _, present := fdPresenceByName[fd.Name]; !present {
9192
toDelete := fd
93+
r.Log.Info(fmt.Sprintf("Deleting extraneous failure domain: %s.", fd.Name))
9294
if err := r.K8sClient.Delete(r.RequestCtx, &toDelete); err != nil {
9395
return ctrl.Result{}, errors.Wrap(err, "failed to delete obsolete failure domain")
9496
}

‎test/e2e/config/cloudstack.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ variables:
112112
IP_FAMILY: "IPv4"
113113
NODE_DRAIN_TIMEOUT: "60s"
114114

115-
CLOUDSTACK_FD1_NAME: "FD1"
116-
CLOUDSTACK_FD1_SECRET_NAME: "Secret1"
115+
CLOUDSTACK_FD1_NAME: "fd1"
116+
CLOUDSTACK_FD1_SECRET_NAME: "myendpoint"
117117
CLOUDSTACK_ZONE_NAME: zone1
118118
CLOUDSTACK_INVALID_ZONE_NAME: zoneXXXX
119119
CLOUDSTACK_INVALID_NETWORK_NAME: networkXXXX

‎test/e2e/data/infrastructure-cloudstack/v1beta2/bases/cluster-with-kcp.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,13 @@ spec:
2828
port: 6443
2929
failureDomains:
3030
- name: ${CLOUDSTACK_FD1_NAME}
31+
acsendpoint:
32+
name: ${CLOUDSTACK_FD1_SECRET_NAME}
33+
namespace: default
3134
zone:
3235
name : ${CLOUDSTACK_ZONE_NAME}
3336
network:
3437
name: ${CLOUDSTACK_NETWORK_NAME}
35-
acsendpoint:
36-
name: ${CLOUDSTACK_FD1_SECRET_NAME}
37-
namespace: default
3838
---
3939
kind: KubeadmControlPlane
4040
apiVersion: controlplane.cluster.x-k8s.io/v1beta1

‎test/e2e/data/infrastructure-cloudstack/v1beta2/cluster-template-affinity-group-anti.yaml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ spec:
88
joinConfiguration:
99
nodeRegistration:
1010
kubeletExtraArgs:
11-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
11+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
1212
name: '{{ local_hostname }}'
1313
preKubeadmCommands:
1414
- swapoff -a
@@ -66,12 +66,12 @@ spec:
6666
initConfiguration:
6767
nodeRegistration:
6868
kubeletExtraArgs:
69-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
69+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
7070
name: '{{ local_hostname }}'
7171
joinConfiguration:
7272
nodeRegistration:
7373
kubeletExtraArgs:
74-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
74+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
7575
name: '{{ local_hostname }}'
7676
preKubeadmCommands:
7777
- swapoff -a
@@ -93,10 +93,10 @@ spec:
9393
port: 6443
9494
failureDomains:
9595
- name: ${CLOUDSTACK_FD1_NAME}
96+
acsendpoint:
97+
name: ${CLOUDSTACK_FD1_SECRET_NAME}
98+
namespace: default
9699
zone:
97-
acsendpoint:
98-
name: ${CLOUDSTACK_FD1_SECRET_NAME}
99-
namespace: default
100100
name: ${CLOUDSTACK_ZONE_NAME}
101101
network:
102102
name: ${CLOUDSTACK_NETWORK_NAME}

‎test/e2e/data/infrastructure-cloudstack/v1beta2/cluster-template-affinity-group-pro.yaml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ spec:
88
joinConfiguration:
99
nodeRegistration:
1010
kubeletExtraArgs:
11-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
11+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
1212
name: '{{ local_hostname }}'
1313
preKubeadmCommands:
1414
- swapoff -a
@@ -66,12 +66,12 @@ spec:
6666
initConfiguration:
6767
nodeRegistration:
6868
kubeletExtraArgs:
69-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
69+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
7070
name: '{{ local_hostname }}'
7171
joinConfiguration:
7272
nodeRegistration:
7373
kubeletExtraArgs:
74-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
74+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
7575
name: '{{ local_hostname }}'
7676
preKubeadmCommands:
7777
- swapoff -a
@@ -93,10 +93,10 @@ spec:
9393
port: 6443
9494
failureDomains:
9595
- name: ${CLOUDSTACK_FD1_NAME}
96+
acsendpoint:
97+
name: ${CLOUDSTACK_FD1_SECRET_NAME}
98+
namespace: default
9699
zone:
97-
acsendpoint:
98-
name: ${CLOUDSTACK_FD1_SECRET_NAME}
99-
namespace: default
100100
name: ${CLOUDSTACK_ZONE_NAME}
101101
network:
102102
name: ${CLOUDSTACK_NETWORK_NAME}

‎test/e2e/data/infrastructure-cloudstack/v1beta2/cluster-template-custom-disk-offering.yaml

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
2+
kind: KubeadmConfigTemplate
3+
metadata:
4+
name: ${CLUSTER_NAME}-md-0
5+
spec:
6+
template:
7+
spec:
8+
diskSetup:
9+
filesystems:
10+
- device: ${CLOUDSTACK_DISK_OFFERING_DEVICE}1
11+
extraOpts:
12+
- -E
13+
- lazy_itable_init=1,lazy_journal_init=1
14+
filesystem: ${CLOUDSTACK_DISK_OFFERING_FILESYSTEM}
15+
label: data_disk
16+
overwrite: false
17+
partitions:
18+
- device: ${CLOUDSTACK_DISK_OFFERING_DEVICE}
19+
layout: true
20+
overwrite: false
21+
tableType: gpt
22+
joinConfiguration:
23+
nodeRegistration:
24+
kubeletExtraArgs:
25+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
26+
name: '{{ local_hostname }}'
27+
mounts:
28+
- - LABEL=${CLOUDSTACK_DISK_OFFERING_LABEL}
29+
- ${CLOUDSTACK_DISK_OFFERING_MOUNT_PATH}
30+
preKubeadmCommands:
31+
- swapoff -a
32+
---
33+
apiVersion: cluster.x-k8s.io/v1beta1
34+
kind: Cluster
35+
metadata:
36+
name: ${CLUSTER_NAME}
37+
spec:
38+
clusterNetwork:
39+
pods:
40+
cidrBlocks:
41+
- 192.168.0.0/16
42+
serviceDomain: cluster.local
43+
controlPlaneRef:
44+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
45+
kind: KubeadmControlPlane
46+
name: ${CLUSTER_NAME}-control-plane
47+
infrastructureRef:
48+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
49+
kind: CloudStackCluster
50+
name: ${CLUSTER_NAME}
51+
---
52+
apiVersion: cluster.x-k8s.io/v1beta1
53+
kind: MachineDeployment
54+
metadata:
55+
name: ${CLUSTER_NAME}-md-0
56+
spec:
57+
clusterName: ${CLUSTER_NAME}
58+
replicas: ${WORKER_MACHINE_COUNT}
59+
selector:
60+
matchLabels: null
61+
template:
62+
spec:
63+
bootstrap:
64+
configRef:
65+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
66+
kind: KubeadmConfigTemplate
67+
name: ${CLUSTER_NAME}-md-0
68+
clusterName: ${CLUSTER_NAME}
69+
infrastructureRef:
70+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
71+
kind: CloudStackMachineTemplate
72+
name: ${CLUSTER_NAME}-md-0
73+
version: ${KUBERNETES_VERSION}
74+
---
75+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
76+
kind: KubeadmControlPlane
77+
metadata:
78+
name: ${CLUSTER_NAME}-control-plane
79+
spec:
80+
kubeadmConfigSpec:
81+
clusterConfiguration:
82+
imageRepository: k8s.gcr.io
83+
diskSetup:
84+
filesystems:
85+
- device: ${CLOUDSTACK_DISK_OFFERING_DEVICE}1
86+
extraOpts:
87+
- -E
88+
- lazy_itable_init=1,lazy_journal_init=1
89+
filesystem: ${CLOUDSTACK_DISK_OFFERING_FILESYSTEM}
90+
label: data_disk
91+
overwrite: false
92+
partitions:
93+
- device: ${CLOUDSTACK_DISK_OFFERING_DEVICE}
94+
layout: true
95+
overwrite: false
96+
tableType: gpt
97+
initConfiguration:
98+
nodeRegistration:
99+
kubeletExtraArgs:
100+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
101+
name: '{{ local_hostname }}'
102+
joinConfiguration:
103+
nodeRegistration:
104+
kubeletExtraArgs:
105+
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
106+
name: '{{ local_hostname }}'
107+
mounts:
108+
- - LABEL=${CLOUDSTACK_DISK_OFFERING_LABEL}
109+
- ${CLOUDSTACK_DISK_OFFERING_MOUNT_PATH}
110+
preKubeadmCommands:
111+
- swapoff -a
112+
machineTemplate:
113+
infrastructureRef:
114+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
115+
kind: CloudStackMachineTemplate
116+
name: ${CLUSTER_NAME}-control-plane
117+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
118+
version: ${KUBERNETES_VERSION}
119+
---
120+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
121+
kind: CloudStackCluster
122+
metadata:
123+
name: ${CLUSTER_NAME}
124+
spec:
125+
controlPlaneEndpoint:
126+
host: ""
127+
port: 6443
128+
failureDomains:
129+
- acsendpoint:
130+
name: ${CLOUDSTACK_FD1_SECRET_NAME}
131+
namespace: default
132+
name: ${CLOUDSTACK_FD1_NAME}
133+
zone:
134+
name: ${CLOUDSTACK_ZONE_NAME}
135+
network:
136+
name: ${CLOUDSTACK_NETWORK_NAME}
137+
---
138+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
139+
kind: CloudStackMachineTemplate
140+
metadata:
141+
name: ${CLUSTER_NAME}-control-plane
142+
spec:
143+
template:
144+
spec:
145+
diskOffering:
146+
customSizeInGB: ${CLOUDSTACK_DISK_OFFERING_CUSTOM_SIZE}
147+
device: ${CLOUDSTACK_DISK_OFFERING_DEVICE}
148+
filesystem: ${CLOUDSTACK_DISK_OFFERING_FILESYSTEM}
149+
label: ${CLOUDSTACK_DISK_OFFERING_LABEL}
150+
mountPath: ${CLOUDSTACK_DISK_OFFERING_MOUNT_PATH}
151+
name: ${CLOUDSTACK_CUSTOM_DISK_OFFERING_NAME}
152+
offering:
153+
name: ${CLOUDSTACK_CONTROL_PLANE_MACHINE_OFFERING}
154+
sshKey: ${CLOUDSTACK_SSH_KEY_NAME}
155+
template:
156+
name: ${CLOUDSTACK_TEMPLATE_NAME}
157+
---
158+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
159+
kind: CloudStackMachineTemplate
160+
metadata:
161+
name: ${CLUSTER_NAME}-md-0
162+
spec:
163+
template:
164+
spec:
165+
diskOffering:
166+
customSizeInGB: ${CLOUDSTACK_DISK_OFFERING_CUSTOM_SIZE}
167+
device: ${CLOUDSTACK_DISK_OFFERING_DEVICE}
168+
filesystem: ${CLOUDSTACK_DISK_OFFERING_FILESYSTEM}
169+
label: ${CLOUDSTACK_DISK_OFFERING_LABEL}
170+
mountPath: ${CLOUDSTACK_DISK_OFFERING_MOUNT_PATH}
171+
name: ${CLOUDSTACK_CUSTOM_DISK_OFFERING_NAME}
172+
offering:
173+
name: ${CLOUDSTACK_WORKER_MACHINE_OFFERING}
174+
sshKey: ${CLOUDSTACK_SSH_KEY_NAME}
175+
template:
176+
name: ${CLOUDSTACK_TEMPLATE_NAME}

‎test/e2e/data/infrastructure-cloudstack/v1beta2/cluster-template-disk-offering.yaml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ spec:
2222
joinConfiguration:
2323
nodeRegistration:
2424
kubeletExtraArgs:
25-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
25+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
2626
name: '{{ local_hostname }}'
2727
mounts:
2828
- - LABEL=${CLOUDSTACK_DISK_OFFERING_LABEL}
@@ -97,12 +97,12 @@ spec:
9797
initConfiguration:
9898
nodeRegistration:
9999
kubeletExtraArgs:
100-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
100+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
101101
name: '{{ local_hostname }}'
102102
joinConfiguration:
103103
nodeRegistration:
104104
kubeletExtraArgs:
105-
provider-id: cloudstack:///'{{ ds.meta_data.instance_id }}'
105+
provider-id: "cloudstack:///'{{ ds.meta_data.instance_id }}'"
106106
name: '{{ local_hostname }}'
107107
mounts:
108108
- - LABEL=${CLOUDSTACK_DISK_OFFERING_LABEL}
@@ -127,10 +127,10 @@ spec:
127127
port: 6443
128128
failureDomains:
129129
- name: ${CLOUDSTACK_FD1_NAME}
130+
acsendpoint:
131+
name: ${CLOUDSTACK_FD1_SECRET_NAME}
132+
namespace: default
130133
zone:
131-
acsendpoint:
132-
name: ${CLOUDSTACK_FD1_SECRET_NAME}
133-
namespace: default
134134
name: ${CLOUDSTACK_ZONE_NAME}
135135
network:
136136
name: ${CLOUDSTACK_NETWORK_NAME}

0 commit comments

Comments
 (0)