Merge pull request kubernetes#86338 from gossion/guwe/k8s

Add a config option to azure cloud provider for the pre-configured loadbalancers

Author: k8s-ci-robot

staging/src/k8s.io/legacy-cloud-providers/azure/azure.go

@@ -74,6 +74,17 @@ const (
 	managedByAzureLabel        = "kubernetes.azure.com/managed"
 )
 
+const (
+	// PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured
+	PreConfiguredBackendPoolLoadBalancerTypesNone = ""
+	// PreConfiguredBackendPoolLoadBalancerTypesInteral means that the `internal` load balancers are pre-configured
+	PreConfiguredBackendPoolLoadBalancerTypesInteral = "internal"
+	// PreConfiguredBackendPoolLoadBalancerTypesExternal means that the `external` load balancers are pre-configured
+	PreConfiguredBackendPoolLoadBalancerTypesExternal = "external"
+	// PreConfiguredBackendPoolLoadBalancerTypesAll means that all load balancers are pre-configured
+	PreConfiguredBackendPoolLoadBalancerTypesAll = "all"
+)
+
 var (
 	// Master nodes are not added to standard load balancer by default.
 	defaultExcludeMasterFromStandardLB = true
@@ -174,6 +185,13 @@ type Config struct {
 	// LoadBalancerResourceGroup determines the specific resource group of the load balancer user want to use, working
 	// with LoadBalancerName
 	LoadBalancerResourceGroup string `json:"loadBalancerResourceGroup,omitempty" yaml:"loadBalancerResourceGroup,omitempty"`
+	// PreConfiguredBackendPoolLoadBalancerTypes determines whether the LoadBalancer BackendPool has been preconfigured.
+	// Candidate values are:
+	//   "": exactly with today (not pre-configured for any LBs)
+	//   "internal": for internal LoadBalancer
+	//   "external": for external LoadBalancer
+	//   "all": for both internal and external LoadBalancer
+	PreConfiguredBackendPoolLoadBalancerTypes string `json:"preConfiguredBackendPoolLoadBalancerTypes,omitempty" yaml:"preConfiguredBackendPoolLoadBalancerTypes,omitempty"`
 
 	// AvailabilitySetNodesCacheTTLInSeconds sets the Cache TTL for availabilitySetNodesCache
 	// if not set, will use default value

staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go

@@ -684,6 +684,7 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend
 // nodes only used if wantLb is true
 func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node, wantLb bool) (*network.LoadBalancer, error) {
 	isInternal := requiresInternalLoadBalancer(service)
+	isBackendPoolPreConfigured := az.isBackendPoolPreConfigured(service)
 	serviceName := getServiceName(service)
 	klog.V(2).Infof("reconcileLoadBalancer for service(%s) - wantLb(%t): started", serviceName, wantLb)
 	lb, _, _, err := az.getServiceLoadBalancer(service, clusterName, nodes, wantLb)
@@ -723,6 +724,14 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 			}
 		}
 		if !foundBackendPool {
+			if isBackendPoolPreConfigured {
+				klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - PreConfiguredBackendPoolLoadBalancerTypes %s has been set but can not find corresponding backend pool, ignoring it",
+					serviceName,
+					wantLb,
+					az.PreConfiguredBackendPoolLoadBalancerTypes)
+				isBackendPoolPreConfigured = false
+			}
+
 			newBackendPools = append(newBackendPools, network.BackendAddressPool{
 				Name: to.StringPtr(lbBackendPoolName),
 			})
@@ -928,28 +937,32 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 	// If it is not exist, and no change to that, we don't CreateOrUpdate LB
 	if dirtyLb {
 		if lb.FrontendIPConfigurations == nil || len(*lb.FrontendIPConfigurations) == 0 {
-			// When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself,
-			// because an Azure load balancer cannot have an empty FrontendIPConfigurations collection
-			klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
-
-			// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
-			vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
-			klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
-			err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
-			if err != nil {
-				klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
-				return nil, err
-			}
-			klog.V(10).Infof("EnsureBackendPoolDeleted(%s) for service %s: end", lbBackendPoolID, serviceName)
+			if isBackendPoolPreConfigured {
+				klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - ignore cleanup of dirty lb because the lb is pre-confiruged", serviceName, lbName)
+			} else {
+				// When FrontendIPConfigurations is empty, we need to delete the Azure load balancer resource itself,
+				// because an Azure load balancer cannot have an empty FrontendIPConfigurations collection
+				klog.V(2).Infof("reconcileLoadBalancer for service(%s): lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
+
+				// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
+				vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
+				klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
+				err := az.vmSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
+				if err != nil {
+					klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
+					return nil, err
+				}
+				klog.V(10).Infof("EnsureBackendPoolDeleted(%s) for service %s: end", lbBackendPoolID, serviceName)
 
-			// Remove the LB.
-			klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLB(%q): start", lbName)
-			err = az.DeleteLB(service, lbName)
-			if err != nil {
-				klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
-				return nil, err
+				// Remove the LB.
+				klog.V(10).Infof("reconcileLoadBalancer: az.DeleteLB(%q): start", lbName)
+				err = az.DeleteLB(service, lbName)
+				if err != nil {
+					klog.V(2).Infof("reconcileLoadBalancer for service(%s) abort backoff: lb(%s) - deleting; no remaining frontendIPConfigurations", serviceName, lbName)
+					return nil, err
+				}
+				klog.V(10).Infof("az.DeleteLB(%q): end", lbName)
 			}
-			klog.V(10).Infof("az.DeleteLB(%q): end", lbName)
 		} else {
 			klog.V(2).Infof("reconcileLoadBalancer: reconcileLoadBalancer for service(%s): lb(%s) - updating", serviceName, lbName)
 			err := az.CreateOrUpdateLB(service, *lb)
@@ -973,7 +986,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 		}
 	}
 
-	if wantLb && nodes != nil {
+	if wantLb && nodes != nil && !isBackendPoolPreConfigured {
 		// Add the machines to the backend pool if they're not already
 		vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
 		// Etag would be changed when updating backend pools, so invalidate lbCache after it.
@@ -1695,6 +1708,23 @@ func (az *Cloud) getPublicIPAddressResourceGroup(service *v1.Service) string {
 	return az.ResourceGroup
 }
 
+func (az *Cloud) isBackendPoolPreConfigured(service *v1.Service) bool {
+	preConfigured := false
+	isInternal := requiresInternalLoadBalancer(service)
+
+	if az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesAll {
+		preConfigured = true
+	}
+	if (az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesInteral) && isInternal {
+		preConfigured = true
+	}
+	if (az.PreConfiguredBackendPoolLoadBalancerTypes == PreConfiguredBackendPoolLoadBalancerTypesExternal) && !isInternal {
+		preConfigured = true
+	}
+
+	return preConfigured
+}
+
 // Check if service requires an internal load balancer.
 func requiresInternalLoadBalancer(service *v1.Service) bool {
 	if l, found := service.Annotations[ServiceAnnotationLoadBalancerInternal]; found {

staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go

@@ -1421,6 +1421,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
 		desc                string
 		service             v1.Service
 		loadBalancerSku     string
+		preConfigLBType     string
 		disableOutboundSnat *bool
 		wantLb              bool
 		existingLB          network.LoadBalancer
@@ -1456,6 +1457,16 @@ func TestReconcileLoadBalancer(t *testing.T) {
 			expectedLB:      expectedLb1,
 			expectedError:   nil,
 		},
+		{
+			desc:            "reconcileLoadBalancer shall not raise an error",
+			loadBalancerSku: "basic",
+			service:         service3,
+			existingLB:      modifiedLb1,
+			preConfigLBType: "external",
+			wantLb:          true,
+			expectedLB:      expectedLb1,
+			expectedError:   nil,
+		},
 		{
 			desc:                "reconcileLoadBalancer shall remove and reconstruct the correspoind field of lb and set enableTcpReset to false in lbRule",
 			loadBalancerSku:     "standard",
@@ -1500,6 +1511,9 @@ func TestReconcileLoadBalancer(t *testing.T) {
 		az := getTestCloud()
 		az.Config.LoadBalancerSku = test.loadBalancerSku
 		az.DisableOutboundSNAT = test.disableOutboundSnat
+		if test.preConfigLBType != "" {
+			az.Config.PreConfiguredBackendPoolLoadBalancerTypes = test.preConfigLBType
+		}
 
 		clusterResources := getClusterResources(az, 3, 3)
 		test.service.Spec.LoadBalancerIP = "1.2.3.4"
@@ -2064,3 +2078,75 @@ func TestShouldUpdateLoadBalancer(t *testing.T) {
 		assert.Equal(t, test.expectedOutput, shouldUpdateLoadBalancer, "TestCase[%d]: %s", i, test.desc)
 	}
 }
+
+func TestIsBackendPoolPreConfigured(t *testing.T) {
+	testCases := []struct {
+		desc                                      string
+		preConfiguredBackendPoolLoadBalancerTypes string
+		isInternalService                         bool
+		expectedOutput                            bool
+	}{
+		{
+			desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is both for any case",
+			preConfiguredBackendPoolLoadBalancerTypes: "all",
+			isInternalService:                         true,
+			expectedOutput:                            true,
+		},
+		{
+			desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is both for any case",
+			preConfiguredBackendPoolLoadBalancerTypes: "all",
+			isInternalService:                         false,
+			expectedOutput:                            true,
+		},
+		{
+			desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is external when creating external lb",
+			preConfiguredBackendPoolLoadBalancerTypes: "external",
+			isInternalService:                         false,
+			expectedOutput:                            true,
+		},
+		{
+			desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is external when creating internal lb",
+			preConfiguredBackendPoolLoadBalancerTypes: "external",
+			isInternalService:                         true,
+			expectedOutput:                            false,
+		},
+		{
+			desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is internal when creating external lb",
+			preConfiguredBackendPoolLoadBalancerTypes: "internal",
+			isInternalService:                         false,
+			expectedOutput:                            false,
+		},
+		{
+			desc: "should return true when preConfiguredBackendPoolLoadBalancerTypes is internal when creating internal lb",
+			preConfiguredBackendPoolLoadBalancerTypes: "internal",
+			isInternalService:                         true,
+			expectedOutput:                            true,
+		},
+		{
+			desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is empty for any case",
+			preConfiguredBackendPoolLoadBalancerTypes: "",
+			isInternalService:                         true,
+			expectedOutput:                            false,
+		},
+		{
+			desc: "should return false when preConfiguredBackendPoolLoadBalancerTypes is empty for any case",
+			preConfiguredBackendPoolLoadBalancerTypes: "",
+			isInternalService:                         false,
+			expectedOutput:                            false,
+		},
+	}
+
+	for i, test := range testCases {
+		az := getTestCloud()
+		az.Config.PreConfiguredBackendPoolLoadBalancerTypes = test.preConfiguredBackendPoolLoadBalancerTypes
+		var service v1.Service
+		if test.isInternalService {
+			service = getInternalTestService("test", 80)
+		} else {
+			service = getTestService("test", v1.ProtocolTCP, nil, 80)
+		}
+
+		isPreConfigured := az.isBackendPoolPreConfigured(&service)
+		assert.Equal(t, test.expectedOutput, isPreConfigured, "TestCase[%d]: %s", i, test.desc)
+	}
+}