Skip to content

Commit 131180b

Browse files
gossiongossion
committed
Add a config option to azure cloud provider for the pre-configured loadbalancers
1 parent 127c47c commit 131180b

File tree

3 files changed

+155
-21
lines changed

3 files changed

+155
-21
lines changed

‎staging/src/k8s.io/legacy-cloud-providers/azure/azure.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,17 @@ const (
7474
managedByAzureLabel = "kubernetes.azure.com/managed"
7575
)
7676

77+
const (
78+
// PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured
79+
PreConfiguredBackendPoolLoadBalancerTypesNone = ""
80+
// PreConfiguredBackendPoolLoadBalancerTypesInteral means that the `internal` load balancers are pre-configured
81+
PreConfiguredBackendPoolLoadBalancerTypesInteral = "internal"
82+
// PreConfiguredBackendPoolLoadBalancerTypesExternal means that the `external` load balancers are pre-configured
83+
PreConfiguredBackendPoolLoadBalancerTypesExternal = "external"
84+
// PreConfiguredBackendPoolLoadBalancerTypesAll means that all load balancers are pre-configured
85+
PreConfiguredBackendPoolLoadBalancerTypesAll = "all"
86+
)
87+
7788
var (
7889
// Master nodes are not added to standard load balancer by default.
7990
defaultExcludeMasterFromStandardLB = true
@@ -174,6 +185,13 @@ type Config struct {
174185
// LoadBalancerResourceGroup determines the specific resource group of the load balancer user want to use, working
175186
// with LoadBalancerName
176187
LoadBalancerResourceGroup string `json:"loadBalancerResourceGroup,omitempty" yaml:"loadBalancerResourceGroup,omitempty"`
188+
// PreConfiguredBackendPoolLoadBalancerTypes determines whether the LoadBalancer BackendPool has been preconfigured.
189+
// Candidate values are:
190+
// "": exactly with today (not pre-configured for any LBs)
191+
// "internal": for internal LoadBalancer
192+
// "external": for external LoadBalancer
193+
// "all": for both internal and external LoadBalancer
194+
PreConfiguredBackendPoolLoadBalancerTypes string `json:"preConfiguredBackendPoolLoadBalancerTypes,omitempty" yaml:"preConfiguredBackendPoolLoadBalancerTypes,omitempty"`
177195

178196
// AvailabilitySetNodesCacheTTLInSeconds sets the Cache TTL for availabilitySetNodesCache
179197
// if not set, will use default value

‎staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go‎

Lines changed: 51 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -684,6 +684,7 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend
684684
// nodes only used if wantLb is true
685685
func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node, wantLb bool) (*network.LoadBalancer, error) {
686686
isInternal := requiresInternalLoadBalancer(service)
687+
isBackendPoolPreConfigured := az.isBackendPoolPreConfigured(service)
687688
serviceName := getServiceName(service)
688689
klog.V(2).Infof("reconcileLoadBalancer for service(%s) - wantLb(%t): started", serviceName, wantLb)
689690
lb, _, _, err := az.getServiceLoadBalancer(service, clusterName, nodes, wantLb)
@@ -723,6 +724,14 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
723724
}
724725
}
725726
if !foundBackendPool {
727+
if isBackendPoolPreConfigured {
728+
klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - PreConfiguredBackendPoolLoadBalancerTypes %s has been set but can not find corresponding backend pool, ignoring it",
729+
serviceName,
730+
wantLb,
731+
az.PreConfiguredBackendPoolLoadBalancerTypes)
732+
isBackendPoolPreConfigured = false
733+
}
734+
726735
newBackendPools = append(newBackendPools, network.BackendAddressPool{
727736
Name: to.StringPtr(lbBackendPoolName),
728737
})
@@ -928,28 +937,32 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
928937
// If it is not exist, and no change to that, we don't CreateOrUpdate LB
929938
if dirtyLb {
930939
if lb.FrontendIPConfigurations == nil || len(*lb.FrontendIPConfigurations) == 0 {
931-
// When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself,
932-
// because an Azure load balancer cannot have an empty FrontendIPConfigurations collection
933-
klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
934-
935-
// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
936-
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
937-
klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
938-
err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
939-
if err != nil {
940-
klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
941-
return nil, err
942-
}
943-
klog.V(10).Infof("EnsureBackendPoolDeleted(%s) for service %s: end", lbBackendPoolID, serviceName)
940+
if isBackendPoolPreConfigured {
941+
klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - ignore cleanup of dirty lb because the lb is pre-confiruged", serviceName, lbName)
942+
} else {
943+
// When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself,
944+
// because an Azure load balancer cannot have an empty FrontendIPConfigurations collection
945+
klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
946+
947+
// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
948+
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
949+
klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
950+
err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
951+
if err != nil {
952+
klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
953+
return nil, err
954+
}
955+
klog.V(10).Infof("EnsureBackendPoolDeleted(%s) for service %s: end", lbBackendPoolID, serviceName)
944956

945-
// Remove the LB.
946-
klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLB(%q): start", lbName)
947-
err = az.DeleteLB(service, lbName)
948-
if err != nil {
949-
klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
950-
return nil, err
957+
// Remove the LB.
958+
klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLB(%q): start", lbName)
959+
err = az.DeleteLB(service, lbName)
960+
if err != nil {
961+
klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
962+
return nil, err
963+
}
964+
klog.V(10).Infof("az.DeleteLB(%q): end", lbName)
951965
}
952-
klog.V(10).Infof("az.DeleteLB(%q): end", lbName)
953966
} else {
954967
klog.V(2).Infof("reconcileLoadBalancer: reconcileLoadBalancer for service(%s): lb(%s) - updating", serviceName, lbName)
955968
err := az.CreateOrUpdateLB(service, *lb)
@@ -973,7 +986,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
973986
}
974987
}
975988

976-
if wantLb && nodes != nil {
989+
if wantLb && nodes != nil && !isBackendPoolPreConfigured {
977990
// Add the machines to the backend pool if they're not already
978991
vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
979992
// Etag would be changed when updating backend pools, so invalidate lbCache after it.
@@ -1695,6 +1708,23 @@ func (az *Cloud) getPublicIPAddressResourceGroup(service *v1.Service) string {
16951708
return az.ResourceGroup
16961709
}
16971710

1711+
func (az *Cloud) isBackendPoolPreConfigured(service *v1.Service) bool {
1712+
preConfigured := false
1713+
isInternal := requiresInternalLoadBalancer(service)
1714+
1715+
if az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesAll {
1716+
preConfigured = true
1717+
}
1718+
if (az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesInteral) && isInternal {
1719+
preConfigured = true
1720+
}
1721+
if (az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesExternal) && !isInternal {
1722+
preConfigured = true
1723+
}
1724+
1725+
return preConfigured
1726+
}
1727+
16981728
// Check if service requires an internal load balancer.
16991729
func requiresInternalLoadBalancer(service *v1.Service) bool {
17001730
if l, found := service.Annotations[ServiceAnnotationLoadBalancerInternal]; found {

‎staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1421,6 +1421,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
14211421
desc string
14221422
service v1.Service
14231423
loadBalancerSku string
1424+
preConfigLBType string
14241425
disableOutboundSnat *bool
14251426
wantLb bool
14261427
existingLB network.LoadBalancer
@@ -1456,6 +1457,16 @@ func TestReconcileLoadBalancer(t *testing.T) {
14561457
expectedLB: expectedLb1,
14571458
expectedError: nil,
14581459
},
1460+
{
1461+
desc: "reconcileLoadBalancer shall not raise an error",
1462+
loadBalancerSku: "basic",
1463+
service: service3,
1464+
existingLB: modifiedLb1,
1465+
preConfigLBType: "external",
1466+
wantLb: true,
1467+
expectedLB: expectedLb1,
1468+
expectedError: nil,
1469+
},
14591470
{
14601471
desc: "reconcileLoadBalancer shall remove and reconstruct the correspoind field of lb and set enableTcpReset to false in lbRule",
14611472
loadBalancerSku: "standard",
@@ -1500,6 +1511,9 @@ func TestReconcileLoadBalancer(t *testing.T) {
15001511
az := getTestCloud()
15011512
az.Config.LoadBalancerSku = test.loadBalancerSku
15021513
az.DisableOutboundSNAT = test.disableOutboundSnat
1514+
if test.preConfigLBType != "" {
1515+
az.Config.PreConfiguredBackendPoolLoadBalancerTypes = test.preConfigLBType
1516+
}
15031517

15041518
clusterResources := getClusterResources(az, 3, 3)
15051519
test.service.Spec.LoadBalancerIP = "1.2.3.4"
@@ -2064,3 +2078,75 @@ func TestShouldUpdateLoadBalancer(t *testing.T) {
20642078
assert.Equal(t, test.expectedOutput, shouldUpdateLoadBalancer, "TestCase[%d]: %s", i, test.desc)
20652079
}
20662080
}
2081+
2082+
func TestIsBackendPoolPreConfigured(t *testing.T) {
2083+
testCases := []struct {
2084+
desc string
2085+
preConfiguredBackendPoolLoadBalancerTypes string
2086+
isInternalService bool
2087+
expectedOutput bool
2088+
}{
2089+
{
2090+
desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is both for any case",
2091+
preConfiguredBackendPoolLoadBalancerTypes: "all",
2092+
isInternalService: true,
2093+
expectedOutput: true,
2094+
},
2095+
{
2096+
desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is both for any case",
2097+
preConfiguredBackendPoolLoadBalancerTypes: "all",
2098+
isInternalService: false,
2099+
expectedOutput: true,
2100+
},
2101+
{
2102+
desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is external when creating external lb",
2103+
preConfiguredBackendPoolLoadBalancerTypes: "external",
2104+
isInternalService: false,
2105+
expectedOutput: true,
2106+
},
2107+
{
2108+
desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is external when creating internal lb",
2109+
preConfiguredBackendPoolLoadBalancerTypes: "external",
2110+
isInternalService: true,
2111+
expectedOutput: false,
2112+
},
2113+
{
2114+
desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is internal when creating external lb",
2115+
preConfiguredBackendPoolLoadBalancerTypes: "internal",
2116+
isInternalService: false,
2117+
expectedOutput: false,
2118+
},
2119+
{
2120+
desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is internal when creating internal lb",
2121+
preConfiguredBackendPoolLoadBalancerTypes: "internal",
2122+
isInternalService: true,
2123+
expectedOutput: true,
2124+
},
2125+
{
2126+
desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is empty for any case",
2127+
preConfiguredBackendPoolLoadBalancerTypes: "",
2128+
isInternalService: true,
2129+
expectedOutput: false,
2130+
},
2131+
{
2132+
desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is empty for any case",
2133+
preConfiguredBackendPoolLoadBalancerTypes: "",
2134+
isInternalService: false,
2135+
expectedOutput: false,
2136+
},
2137+
}
2138+
2139+
for i, test := range testCases {
2140+
az := getTestCloud()
2141+
az.Config.PreConfiguredBackendPoolLoadBalancerTypes = test.preConfiguredBackendPoolLoadBalancerTypes
2142+
var service v1.Service
2143+
if test.isInternalService {
2144+
service = getInternalTestService("test", 80)
2145+
} else {
2146+
service = getTestService("test", v1.ProtocolTCP, nil, 80)
2147+
}
2148+
2149+
isPreConfigured := az.isBackendPoolPreConfigured(&service)
2150+
assert.Equal(t, test.expectedOutput, isPreConfigured, "TestCase[%d]: %s", i, test.desc)
2151+
}
2152+
}

0 commit comments

Comments
 (0)