Skip to content

Commit e8f327d

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1141 from CecileRobertMichon/lb-reconcile-merge
🐛 Merge load balancer properties with existing properties to avoid overwriting cloud provider rules
2 parents bc3657d + 986ce13 commit e8f327d

File tree

3 files changed

+503
-278
lines changed

3 files changed

+503
-278
lines changed

‎cloud/services/loadbalancers/client.go‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,32 @@ func (ac *AzureClient) CreateOrUpdate(ctx context.Context, resourceGroupName str
6666
ctx, span := tele.Tracer().Start(ctx, "loadbalancers.AzureClient.CreateOrUpdate")
6767
defer span.End()
6868

69-
future, err := ac.loadbalancers.CreateOrUpdate(ctx, resourceGroupName, lbName, lb)
69+
var etag string
70+
if lb.Etag != nil {
71+
etag = *lb.Etag
72+
}
73+
74+
req, err := ac.loadbalancers.CreateOrUpdatePreparer(ctx, resourceGroupName, lbName, lb)
75+
if err != nil {
76+
err = autorest.NewErrorWithError(err, "network.LoadBalancersClient", "CreateOrUpdate", nil, "Failure preparing request")
77+
return err
78+
}
79+
80+
if etag != "" {
81+
req.Header.Add("If-Match", etag)
82+
}
83+
84+
future, err := ac.loadbalancers.CreateOrUpdateSender(req)
7085
if err != nil {
86+
err = autorest.NewErrorWithError(err, "network.LoadBalancersClient", "CreateOrUpdate", future.Response(), "Failure sending request")
7187
return err
7288
}
89+
7390
err = future.WaitForCompletionRef(ctx, ac.loadbalancers.Client)
7491
if err != nil {
7592
return err
7693
}
94+
7795
_, err = future.Result(ac.loadbalancers)
7896
return err
7997
}

‎cloud/services/loadbalancers/loadbalancers.go‎

Lines changed: 210 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ import (
3131
"sigs.k8s.io/cluster-api-provider-azure/util/tele"
3232
)
3333

34+
const (
35+
httpsProbe = "HTTPSProbe"
36+
lbRuleHTTPS = "LBRuleHTTPS"
37+
outboundNAT = "OutboundNATAllProtocols"
38+
)
39+
3440
// LBScope defines the scope interface for a load balancer service.
3541
type LBScope interface {
3642
logr.Logger
@@ -61,14 +67,85 @@ func (s *Service) Reconcile(ctx context.Context) error {
6167
defer span.End()
6268

6369
for _, lbSpec := range s.Scope.LBSpecs() {
64-
s.Scope.V(2).Info("creating load balancer", "load balancer", lbSpec.Name)
70+
var (
71+
etag *string
72+
frontendIDs []network.SubResource
73+
frontendIPConfigs = make([]network.FrontendIPConfiguration, 0)
74+
loadBalancingRules = make([]network.LoadBalancingRule, 0)
75+
backendAddressPools = make([]network.BackendAddressPool, 0)
76+
outboundRules = make([]network.OutboundRule, 0)
77+
probes = make([]network.Probe, 0)
78+
)
6579

66-
frontendIPConfigs, frontendIDs, err := s.getFrontendIPConfigs(lbSpec)
67-
if err != nil {
68-
return err
80+
existingLB, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), lbSpec.Name)
81+
switch {
82+
case err != nil && !azure.ResourceNotFound(err):
83+
return errors.Wrapf(err, "failed to get LB %s in %s", lbSpec.Name, s.Scope.ResourceGroup())
84+
case err == nil:
85+
// LB already exists
86+
s.Scope.V(2).Info("found existing load balancer, checking if updates are needed", "load balancer", lbSpec.Name)
87+
// We append the existing LB etag to the header to ensure we only apply the updates if the LB has not been modified.
88+
etag = existingLB.Etag
89+
update := false
90+
91+
// merge existing LB properties with desired properties
92+
frontendIPConfigs = *existingLB.FrontendIPConfigurations
93+
wantedIPs, wantedFrontendIDs := s.getFrontendIPConfigs(lbSpec)
94+
for _, ip := range wantedIPs {
95+
if !ipExists(frontendIPConfigs, ip) {
96+
update = true
97+
frontendIPConfigs = append(frontendIPConfigs, ip)
98+
}
99+
}
100+
101+
loadBalancingRules = *existingLB.LoadBalancingRules
102+
for _, rule := range s.getLoadBalancingRules(lbSpec, wantedFrontendIDs) {
103+
if !lbRuleExists(loadBalancingRules, rule) {
104+
update = true
105+
loadBalancingRules = append(loadBalancingRules, rule)
106+
}
107+
}
108+
109+
backendAddressPools = *existingLB.BackendAddressPools
110+
for _, pool := range s.getBackendAddressPools(lbSpec) {
111+
if !poolExists(backendAddressPools, pool) {
112+
update = true
113+
backendAddressPools = append(backendAddressPools, pool)
114+
}
115+
}
116+
117+
outboundRules = *existingLB.OutboundRules
118+
for _, rule := range s.getOutboundRules(lbSpec, wantedFrontendIDs) {
119+
if !outboundRuleExists(outboundRules, rule) {
120+
update = true
121+
outboundRules = append(outboundRules, rule)
122+
}
123+
}
124+
125+
probes = *existingLB.Probes
126+
for _, probe := range s.getProbes(lbSpec) {
127+
if !probeExists(probes, probe) {
128+
update = true
129+
probes = append(probes, probe)
130+
}
131+
}
132+
133+
if !update {
134+
// Skip update for LB as the required defaults are present
135+
s.Scope.V(2).Info("LB exists and no defaults are missing, skipping update", "load balancer", lbSpec.Name)
136+
continue
137+
}
138+
default:
139+
s.Scope.V(2).Info("creating load balancer", "load balancer", lbSpec.Name)
140+
frontendIPConfigs, frontendIDs = s.getFrontendIPConfigs(lbSpec)
141+
loadBalancingRules = s.getLoadBalancingRules(lbSpec, frontendIDs)
142+
backendAddressPools = s.getBackendAddressPools(lbSpec)
143+
outboundRules = s.getOutboundRules(lbSpec, frontendIDs)
144+
probes = s.getProbes(lbSpec)
69145
}
70146

71147
lb := network.LoadBalancer{
148+
Etag: etag,
72149
Sku: &network.LoadBalancerSku{Name: converters.SKUtoSDK(lbSpec.SKU)},
73150
Location: to.StringPtr(s.Scope.Location()),
74151
Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{
@@ -79,73 +156,13 @@ func (s *Service) Reconcile(ctx context.Context) error {
79156
})),
80157
LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
81158
FrontendIPConfigurations: &frontendIPConfigs,
82-
BackendAddressPools: &[]network.BackendAddressPool{
83-
{
84-
Name: to.StringPtr(lbSpec.BackendPoolName),
85-
},
86-
},
87-
OutboundRules: &[]network.OutboundRule{
88-
{
89-
Name: to.StringPtr("OutboundNATAllProtocols"),
90-
OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{
91-
Protocol: network.LoadBalancerOutboundRuleProtocolAll,
92-
IdleTimeoutInMinutes: to.Int32Ptr(4),
93-
FrontendIPConfigurations: &frontendIDs,
94-
BackendAddressPool: &network.SubResource{
95-
ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, lbSpec.BackendPoolName)),
96-
},
97-
},
98-
},
99-
},
159+
BackendAddressPools: &backendAddressPools,
160+
OutboundRules: &outboundRules,
161+
Probes: &probes,
162+
LoadBalancingRules: &loadBalancingRules,
100163
},
101164
}
102165

103-
if lbSpec.Role == infrav1.APIServerRole {
104-
probeName := "HTTPSProbe"
105-
lb.LoadBalancerPropertiesFormat.Probes = &[]network.Probe{
106-
{
107-
Name: to.StringPtr(probeName),
108-
ProbePropertiesFormat: &network.ProbePropertiesFormat{
109-
Protocol: network.ProbeProtocolHTTPS,
110-
RequestPath: to.StringPtr("/healthz"),
111-
Port: to.Int32Ptr(lbSpec.APIServerPort),
112-
IntervalInSeconds: to.Int32Ptr(15),
113-
NumberOfProbes: to.Int32Ptr(4),
114-
},
115-
},
116-
}
117-
// We disable outbound SNAT explicitly in the HTTPS LB rule and enable TCP and UDP outbound NAT with an outbound rule.
118-
// For more information on Standard LB outbound connections see https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections.
119-
var frontendIPConfig network.SubResource
120-
if len(frontendIDs) != 0 {
121-
frontendIPConfig = frontendIDs[0]
122-
}
123-
lb.LoadBalancerPropertiesFormat.LoadBalancingRules = &[]network.LoadBalancingRule{
124-
{
125-
Name: to.StringPtr("LBRuleHTTPS"),
126-
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
127-
DisableOutboundSnat: to.BoolPtr(true),
128-
Protocol: network.TransportProtocolTCP,
129-
FrontendPort: to.Int32Ptr(lbSpec.APIServerPort),
130-
BackendPort: to.Int32Ptr(lbSpec.APIServerPort),
131-
IdleTimeoutInMinutes: to.Int32Ptr(4),
132-
EnableFloatingIP: to.BoolPtr(false),
133-
LoadDistribution: network.LoadDistributionDefault,
134-
FrontendIPConfiguration: &frontendIPConfig,
135-
BackendAddressPool: &network.SubResource{
136-
ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, lbSpec.BackendPoolName)),
137-
},
138-
Probe: &network.SubResource{
139-
ID: to.StringPtr(azure.ProbeID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, probeName)),
140-
},
141-
},
142-
},
143-
}
144-
if lbSpec.Type == infrav1.Internal {
145-
lb.LoadBalancerPropertiesFormat.OutboundRules = nil
146-
}
147-
}
148-
149166
err = s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), lbSpec.Name, lb)
150167

151168
if err != nil {
@@ -178,7 +195,7 @@ func (s *Service) Delete(ctx context.Context) error {
178195
return nil
179196
}
180197

181-
func (s *Service) getFrontendIPConfigs(lbSpec azure.LBSpec) ([]network.FrontendIPConfiguration, []network.SubResource, error) {
198+
func (s *Service) getFrontendIPConfigs(lbSpec azure.LBSpec) ([]network.FrontendIPConfiguration, []network.SubResource) {
182199
frontendIPConfigurations := make([]network.FrontendIPConfiguration, 0)
183200
frontendIDs := make([]network.SubResource, 0)
184201
for _, ipConfig := range lbSpec.FrontendIPConfigs {
@@ -206,5 +223,128 @@ func (s *Service) getFrontendIPConfigs(lbSpec azure.LBSpec) ([]network.FrontendI
206223
ID: to.StringPtr(azure.FrontendIPConfigID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, ipConfig.Name)),
207224
})
208225
}
209-
return frontendIPConfigurations, frontendIDs, nil
226+
return frontendIPConfigurations, frontendIDs
227+
}
228+
229+
func (s *Service) getOutboundRules(lbSpec azure.LBSpec, frontendIDs []network.SubResource) []network.OutboundRule {
230+
if lbSpec.Type == infrav1.Internal {
231+
return []network.OutboundRule{}
232+
}
233+
return []network.OutboundRule{
234+
{
235+
Name: to.StringPtr(outboundNAT),
236+
OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{
237+
Protocol: network.LoadBalancerOutboundRuleProtocolAll,
238+
IdleTimeoutInMinutes: to.Int32Ptr(4),
239+
FrontendIPConfigurations: &frontendIDs,
240+
BackendAddressPool: &network.SubResource{
241+
ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, lbSpec.BackendPoolName)),
242+
},
243+
},
244+
},
245+
}
246+
}
247+
248+
func (s *Service) getLoadBalancingRules(lbSpec azure.LBSpec, frontendIDs []network.SubResource) []network.LoadBalancingRule {
249+
if lbSpec.Role == infrav1.APIServerRole {
250+
// We disable outbound SNAT explicitly in the HTTPS LB rule and enable TCP and UDP outbound NAT with an outbound rule.
251+
// For more information on Standard LB outbound connections see https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections.
252+
var frontendIPConfig network.SubResource
253+
if len(frontendIDs) != 0 {
254+
frontendIPConfig = frontendIDs[0]
255+
}
256+
return []network.LoadBalancingRule{
257+
{
258+
Name: to.StringPtr(lbRuleHTTPS),
259+
LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
260+
DisableOutboundSnat: to.BoolPtr(true),
261+
Protocol: network.TransportProtocolTCP,
262+
FrontendPort: to.Int32Ptr(lbSpec.APIServerPort),
263+
BackendPort: to.Int32Ptr(lbSpec.APIServerPort),
264+
IdleTimeoutInMinutes: to.Int32Ptr(4),
265+
EnableFloatingIP: to.BoolPtr(false),
266+
LoadDistribution: network.LoadDistributionDefault,
267+
FrontendIPConfiguration: &frontendIPConfig,
268+
BackendAddressPool: &network.SubResource{
269+
ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, lbSpec.BackendPoolName)),
270+
},
271+
Probe: &network.SubResource{
272+
ID: to.StringPtr(azure.ProbeID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, httpsProbe)),
273+
},
274+
},
275+
},
276+
}
277+
}
278+
return []network.LoadBalancingRule{}
279+
}
280+
281+
func (s *Service) getBackendAddressPools(lbSpec azure.LBSpec) []network.BackendAddressPool {
282+
return []network.BackendAddressPool{
283+
{
284+
Name: to.StringPtr(lbSpec.BackendPoolName),
285+
},
286+
}
287+
}
288+
289+
func (s *Service) getProbes(lbSpec azure.LBSpec) []network.Probe {
290+
if lbSpec.Role == infrav1.APIServerRole {
291+
return []network.Probe{
292+
{
293+
Name: to.StringPtr(httpsProbe),
294+
ProbePropertiesFormat: &network.ProbePropertiesFormat{
295+
Protocol: network.ProbeProtocolHTTPS,
296+
RequestPath: to.StringPtr("/healthz"),
297+
Port: to.Int32Ptr(lbSpec.APIServerPort),
298+
IntervalInSeconds: to.Int32Ptr(15),
299+
NumberOfProbes: to.Int32Ptr(4),
300+
},
301+
},
302+
}
303+
}
304+
return []network.Probe{}
305+
}
306+
307+
func probeExists(probes []network.Probe, probe network.Probe) bool {
308+
for _, p := range probes {
309+
if to.String(p.Name) == to.String(probe.Name) {
310+
return true
311+
}
312+
}
313+
return false
314+
}
315+
316+
func outboundRuleExists(rules []network.OutboundRule, rule network.OutboundRule) bool {
317+
for _, r := range rules {
318+
if to.String(r.Name) == to.String(rule.Name) {
319+
return true
320+
}
321+
}
322+
return false
323+
}
324+
325+
func poolExists(pools []network.BackendAddressPool, pool network.BackendAddressPool) bool {
326+
for _, p := range pools {
327+
if to.String(p.Name) == to.String(pool.Name) {
328+
return true
329+
}
330+
}
331+
return false
332+
}
333+
334+
func lbRuleExists(rules []network.LoadBalancingRule, rule network.LoadBalancingRule) bool {
335+
for _, r := range rules {
336+
if to.String(r.Name) == to.String(rule.Name) {
337+
return true
338+
}
339+
}
340+
return false
341+
}
342+
343+
func ipExists(configs []network.FrontendIPConfiguration, config network.FrontendIPConfiguration) bool {
344+
for _, ip := range configs {
345+
if to.String(ip.Name) == to.String(config.Name) {
346+
return true
347+
}
348+
}
349+
return false
210350
}

0 commit comments

Comments
 (0)