Skip to content

Commit ee2c5f4

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #4733 from nrb/secondary-lb
✨ Add support for a secondary control plane load balancer
2 parents 030638a + c72f25d commit ee2c5f4

24 files changed

+2052
-187
lines changed

‎api/v1beta1/awscluster_conversion.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error {
4444
}
4545
restoreControlPlaneLoadBalancerStatus(&restored.Status.Network.APIServerELB, &dst.Status.Network.APIServerELB)
4646

47+
if restored.Spec.SecondaryControlPlaneLoadBalancer != nil {
48+
if dst.Spec.SecondaryControlPlaneLoadBalancer == nil {
49+
dst.Spec.SecondaryControlPlaneLoadBalancer = &infrav2.AWSLoadBalancerSpec{}
50+
}
51+
restoreControlPlaneLoadBalancer(restored.Spec.SecondaryControlPlaneLoadBalancer, dst.Spec.SecondaryControlPlaneLoadBalancer)
52+
}
53+
restoreControlPlaneLoadBalancerStatus(&restored.Status.Network.SecondaryAPIServerELB, &dst.Status.Network.SecondaryAPIServerELB)
54+
4755
dst.Spec.S3Bucket = restored.Spec.S3Bucket
4856
if restored.Status.Bastion != nil {
4957
dst.Status.Bastion.InstanceMetadataOptions = restored.Status.Bastion.InstanceMetadataOptions
@@ -117,6 +125,16 @@ func restoreControlPlaneLoadBalancerStatus(restored, dst *infrav2.LoadBalancer)
117125
dst.LoadBalancerType = restored.LoadBalancerType
118126
dst.ELBAttributes = restored.ELBAttributes
119127
dst.ELBListeners = restored.ELBListeners
128+
dst.Name = restored.Name
129+
dst.DNSName = restored.DNSName
130+
dst.Scheme = restored.Scheme
131+
dst.SubnetIDs = restored.SubnetIDs
132+
dst.SecurityGroupIDs = restored.SecurityGroupIDs
133+
dst.HealthCheck = restored.HealthCheck
134+
dst.ClassicElbAttributes = restored.ClassicElbAttributes
135+
dst.Tags = restored.Tags
136+
dst.ClassicELBListeners = restored.ClassicELBListeners
137+
dst.AvailabilityZones = restored.AvailabilityZones
120138
}
121139

122140
// restoreIPAMPool manually restores the ipam pool data.
@@ -137,6 +155,10 @@ func restoreControlPlaneLoadBalancer(restored, dst *infrav2.AWSLoadBalancerSpec)
137155
dst.PreserveClientIP = restored.PreserveClientIP
138156
dst.IngressRules = restored.IngressRules
139157
dst.AdditionalListeners = restored.AdditionalListeners
158+
dst.AdditionalSecurityGroups = restored.AdditionalSecurityGroups
159+
dst.Scheme = restored.Scheme
160+
dst.CrossZoneLoadBalancing = restored.CrossZoneLoadBalancing
161+
dst.Subnets = restored.Subnets
140162
}
141163

142164
// ConvertFrom converts the v1beta1 AWSCluster receiver to a v1beta1 AWSCluster.

‎api/v1beta1/zz_generated.conversion.go

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/v1beta2/awscluster_types.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ type AWSClusterSpec struct {
6060
// +optional
6161
ControlPlaneLoadBalancer *AWSLoadBalancerSpec `json:"controlPlaneLoadBalancer,omitempty"`
6262

63+
// SecondaryControlPlaneLoadBalancer is an additional load balancer that can be used for the control plane.
64+
//
65+
// An example use case is to have a separate internal load balancer for internal traffic,
66+
// and a separate external load balancer for external traffic.
67+
//
68+
// +optional
69+
SecondaryControlPlaneLoadBalancer *AWSLoadBalancerSpec `json:"secondaryControlPlaneLoadBalancer,omitempty"`
70+
6371
// ImageLookupFormat is the AMI naming format to look up machine images when
6472
// a machine does not specify an AMI. When set, this will be used for all
6573
// cluster machines unless a machine specifies a different ImageLookupOrg.

‎api/v1beta2/awscluster_webhook.go

Lines changed: 84 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func (r *AWSCluster) ValidateCreate() (admission.Warnings, error) {
5858
allErrs = append(allErrs, r.Spec.AdditionalTags.Validate()...)
5959
allErrs = append(allErrs, r.Spec.S3Bucket.Validate()...)
6060
allErrs = append(allErrs, r.validateNetwork()...)
61-
allErrs = append(allErrs, r.validateControlPlaneLB()...)
61+
allErrs = append(allErrs, r.validateControlPlaneLBs()...)
6262

6363
return nil, aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs)
6464
}
@@ -85,51 +85,18 @@ func (r *AWSCluster) ValidateUpdate(old runtime.Object) (admission.Warnings, err
8585
)
8686
}
8787

88-
newLoadBalancer := &AWSLoadBalancerSpec{}
89-
existingLoadBalancer := &AWSLoadBalancerSpec{}
90-
91-
if r.Spec.ControlPlaneLoadBalancer != nil {
92-
newLoadBalancer = r.Spec.ControlPlaneLoadBalancer.DeepCopy()
88+
// Validate the control plane load balancers.
89+
lbs := map[*AWSLoadBalancerSpec]*AWSLoadBalancerSpec{
90+
oldC.Spec.ControlPlaneLoadBalancer: r.Spec.ControlPlaneLoadBalancer,
91+
oldC.Spec.SecondaryControlPlaneLoadBalancer: r.Spec.SecondaryControlPlaneLoadBalancer,
9392
}
9493

95-
if oldC.Spec.ControlPlaneLoadBalancer != nil {
96-
existingLoadBalancer = oldC.Spec.ControlPlaneLoadBalancer.DeepCopy()
97-
}
98-
if oldC.Spec.ControlPlaneLoadBalancer == nil {
99-
// If old scheme was nil, the only value accepted here is the default value: internet-facing
100-
if newLoadBalancer.Scheme != nil && newLoadBalancer.Scheme.String() != ELBSchemeInternetFacing.String() {
101-
allErrs = append(allErrs,
102-
field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "scheme"),
103-
r.Spec.ControlPlaneLoadBalancer.Scheme, "field is immutable, default value was set to internet-facing"),
104-
)
105-
}
106-
} else {
107-
// If old scheme was not nil, the new scheme should be the same.
108-
if !cmp.Equal(existingLoadBalancer.Scheme, newLoadBalancer.Scheme) {
109-
allErrs = append(allErrs,
110-
field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "scheme"),
111-
r.Spec.ControlPlaneLoadBalancer.Scheme, "field is immutable"),
112-
)
113-
}
114-
// The name must be defined when the AWSCluster is created. If it is not defined,
115-
// then the controller generates a default name at runtime, but does not store it,
116-
// so the name remains nil. In either case, the name cannot be changed.
117-
if !cmp.Equal(existingLoadBalancer.Name, newLoadBalancer.Name) {
118-
allErrs = append(allErrs,
119-
field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "name"),
120-
r.Spec.ControlPlaneLoadBalancer.Name, "field is immutable"),
121-
)
94+
for oldLB, newLB := range lbs {
95+
if oldLB == nil && newLB == nil {
96+
continue
12297
}
123-
}
12498

125-
// Block the update for Protocol :
126-
// - if it was not set in old spec but added in new spec
127-
// - if it was set in old spec but changed in new spec
128-
if !cmp.Equal(newLoadBalancer.HealthCheckProtocol, existingLoadBalancer.HealthCheckProtocol) {
129-
allErrs = append(allErrs,
130-
field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "healthCheckProtocol"),
131-
newLoadBalancer.HealthCheckProtocol, "field is immutable once set"),
132-
)
99+
allErrs = append(allErrs, r.validateControlPlaneLoadBalancerUpdate(oldLB, newLB)...)
133100
}
134101

135102
if !cmp.Equal(oldC.Spec.ControlPlaneEndpoint, clusterv1.APIEndpoint{}) &&
@@ -174,6 +141,49 @@ func (r *AWSCluster) ValidateUpdate(old runtime.Object) (admission.Warnings, err
174141
return nil, aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs)
175142
}
176143

144+
func (r *AWSCluster) validateControlPlaneLoadBalancerUpdate(oldlb, newlb *AWSLoadBalancerSpec) field.ErrorList {
145+
var allErrs field.ErrorList
146+
147+
if oldlb == nil {
148+
// If old scheme was nil, the only value accepted here is the default value: internet-facing
149+
if newlb.Scheme != nil && newlb.Scheme.String() != ELBSchemeInternetFacing.String() {
150+
allErrs = append(allErrs,
151+
field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "scheme"),
152+
newlb.Scheme, "field is immutable, default value was set to internet-facing"),
153+
)
154+
}
155+
} else {
156+
// If old scheme was not nil, the new scheme should be the same.
157+
if !cmp.Equal(oldlb.Scheme, newlb.Scheme) {
158+
allErrs = append(allErrs,
159+
field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "scheme"),
160+
newlb.Scheme, "field is immutable"),
161+
)
162+
}
163+
// The name must be defined when the AWSCluster is created. If it is not defined,
164+
// then the controller generates a default name at runtime, but does not store it,
165+
// so the name remains nil. In either case, the name cannot be changed.
166+
if !cmp.Equal(oldlb.Name, newlb.Name) {
167+
allErrs = append(allErrs,
168+
field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "name"),
169+
newlb.Name, "field is immutable"),
170+
)
171+
}
172+
}
173+
174+
// Block the update for Protocol :
175+
// - if it was not set in old spec but added in new spec
176+
// - if it was set in old spec but changed in new spec
177+
if !cmp.Equal(newlb.HealthCheckProtocol, oldlb.HealthCheckProtocol) {
178+
allErrs = append(allErrs,
179+
field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "healthCheckProtocol"),
180+
newlb.HealthCheckProtocol, "field is immutable once set"),
181+
)
182+
}
183+
184+
return allErrs
185+
}
186+
177187
// Default satisfies the defaulting webhook interface.
178188
func (r *AWSCluster) Default() {
179189
SetObjectDefaults_AWSCluster(r)
@@ -243,26 +253,48 @@ func (r *AWSCluster) validateNetwork() field.ErrorList {
243253
allErrs = append(allErrs, field.Invalid(field.NewPath("additionalControlPlaneIngressRules"), r.Spec.NetworkSpec.AdditionalControlPlaneIngressRules, "CIDR blocks and security group IDs or security group roles cannot be used together"))
244254
}
245255
}
256+
246257
return allErrs
247258
}
248259

249-
func (r *AWSCluster) validateControlPlaneLB() field.ErrorList {
260+
func (r *AWSCluster) validateControlPlaneLBs() field.ErrorList {
250261
var allErrs field.ErrorList
251262

252-
if r.Spec.ControlPlaneLoadBalancer == nil {
253-
return allErrs
263+
// If the secondary is defined, check that the name is not empty and different from the primary.
264+
// Also, ensure that the secondary load balancer is an NLB
265+
if r.Spec.SecondaryControlPlaneLoadBalancer != nil {
266+
if r.Spec.SecondaryControlPlaneLoadBalancer.Name == nil || *r.Spec.SecondaryControlPlaneLoadBalancer.Name == "" {
267+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "secondaryControlPlaneLoadBalancer", "name"), r.Spec.SecondaryControlPlaneLoadBalancer.Name, "secondary controlPlaneLoadBalancer.name cannot be empty"))
268+
}
269+
270+
if r.Spec.SecondaryControlPlaneLoadBalancer.Name == r.Spec.ControlPlaneLoadBalancer.Name {
271+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "secondaryControlPlaneLoadBalancer", "name"), r.Spec.SecondaryControlPlaneLoadBalancer.Name, "field must be different from controlPlaneLoadBalancer.name"))
272+
}
273+
274+
if r.Spec.SecondaryControlPlaneLoadBalancer.Scheme.Equals(r.Spec.ControlPlaneLoadBalancer.Scheme) {
275+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "secondaryControlPlaneLoadBalancer", "scheme"), r.Spec.SecondaryControlPlaneLoadBalancer.Scheme, "control plane load balancers must have different schemes"))
276+
}
277+
278+
if r.Spec.SecondaryControlPlaneLoadBalancer.LoadBalancerType != LoadBalancerTypeNLB {
279+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "secondaryControlPlaneLoadBalancer", "loadBalancerType"), r.Spec.SecondaryControlPlaneLoadBalancer.LoadBalancerType, "secondary control plane load balancer must be a Network Load Balancer"))
280+
}
254281
}
255282

256283
// Additional listeners are only supported for NLBs.
257-
if len(r.Spec.ControlPlaneLoadBalancer.AdditionalListeners) > 0 {
258-
if r.Spec.ControlPlaneLoadBalancer.LoadBalancerType != LoadBalancerTypeNLB {
259-
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "additionalListeners"), r.Spec.ControlPlaneLoadBalancer.AdditionalListeners, "additional listeners are only supported for NLB load balancers"))
260-
}
284+
// Validate the control plane load balancers.
285+
loadBalancers := []*AWSLoadBalancerSpec{
286+
r.Spec.ControlPlaneLoadBalancer,
287+
r.Spec.SecondaryControlPlaneLoadBalancer,
261288
}
289+
for _, cp := range loadBalancers {
290+
if cp == nil {
291+
continue
292+
}
262293

263-
for _, rule := range r.Spec.ControlPlaneLoadBalancer.IngressRules {
264-
if (rule.CidrBlocks != nil || rule.IPv6CidrBlocks != nil) && (rule.SourceSecurityGroupIDs != nil || rule.SourceSecurityGroupRoles != nil) {
265-
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "ingressRules"), r.Spec.ControlPlaneLoadBalancer.IngressRules, "CIDR blocks and security group IDs or security group roles cannot be used together"))
294+
for _, rule := range cp.IngressRules {
295+
if (rule.CidrBlocks != nil || rule.IPv6CidrBlocks != nil) && (rule.SourceSecurityGroupIDs != nil || rule.SourceSecurityGroupRoles != nil) {
296+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "ingressRules"), r.Spec.ControlPlaneLoadBalancer.IngressRules, "CIDR blocks and security group IDs or security group roles cannot be used together"))
297+
}
266298
}
267299
}
268300

‎api/v1beta2/defaults.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,14 @@ func SetDefaults_AWSClusterSpec(s *AWSClusterSpec) { //nolint:golint,stylecheck
6969
if s.ControlPlaneLoadBalancer.LoadBalancerType == "" {
7070
s.ControlPlaneLoadBalancer.LoadBalancerType = LoadBalancerTypeClassic
7171
}
72+
if s.SecondaryControlPlaneLoadBalancer != nil {
73+
if s.SecondaryControlPlaneLoadBalancer.LoadBalancerType == "" {
74+
s.SecondaryControlPlaneLoadBalancer.LoadBalancerType = LoadBalancerTypeNLB
75+
}
76+
if s.SecondaryControlPlaneLoadBalancer.Scheme == nil {
77+
s.SecondaryControlPlaneLoadBalancer.Scheme = &ELBSchemeInternal
78+
}
79+
}
7280
}
7381

7482
// SetDefaults_Labels is used to default cluster scope resources for clusterctl move.

‎api/v1beta2/network_types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ type NetworkStatus struct {
3737
// APIServerELB is the Kubernetes api server load balancer.
3838
APIServerELB LoadBalancer `json:"apiServerElb,omitempty"`
3939

40+
// SecondaryAPIServerELB is the secondary Kubernetes api server load balancer.
41+
SecondaryAPIServerELB LoadBalancer `json:"secondaryAPIServerELB,omitempty"`
42+
4043
// NatGatewaysIPs contains the public IPs of the NAT Gateways
4144
NatGatewaysIPs []string `json:"natGatewaysIPs,omitempty"`
4245
}

‎api/v1beta2/zz_generated.deepcopy.go

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)