Merge pull request #1415 from Ankitasw/backoff-config

Expose cloud provider backoff config

Author: k8s-ci-robot

api/v1alpha4/types.go

@@ -518,6 +518,16 @@ type AddressRecord struct {
 // CloudProviderConfigOverrides represents the fields that can be overridden in azure cloud provider config.
 type CloudProviderConfigOverrides struct {
 	RateLimits []RateLimitSpec `json:"rateLimits,omitempty"`
+	BackOffs   BackOffConfig   `json:"backOffs,omitempty"`
+}
+
+// BackOffConfig indicates the back-off config options.
+type BackOffConfig struct {
+	CloudProviderBackoff         bool               `json:"cloudProviderBackoff,omitempty"`
+	CloudProviderBackoffRetries  int                `json:"cloudProviderBackoffRetries,omitempty"`
+	CloudProviderBackoffExponent *resource.Quantity `json:"cloudProviderBackoffExponent,omitempty"`
+	CloudProviderBackoffDuration int                `json:"cloudProviderBackoffDuration,omitempty"`
+	CloudProviderBackoffJitter   *resource.Quantity `json:"cloudProviderBackoffJitter,omitempty"`
 }
 
 // RateLimitSpec represents the rate limit configuration for a particular kind of resource.

api/v1alpha4/zz_generated.deepcopy.go

@@ -482,6 +482,28 @@ spec:
               cloudProviderConfigOverrides:
                 description: 'CloudProviderConfigOverrides is an optional set of configuration values that can be overridden in azure cloud provider config. This is only a subset of options that are available in azure cloud provider config. Some values for the cloud provider config are inferred from other parts of cluster api provider azure spec, and may not be available for overrides. See: https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs Note: All cloud provider config values can be customized by creating the secret beforehand. CloudProviderConfigOverrides is only used when the secret is managed by the Azure Provider.'
                 properties:
+                  backOffs:
+                    description: BackOffConfig indicates the back-off config options.
+                    properties:
+                      cloudProviderBackoff:
+                        type: boolean
+                      cloudProviderBackoffDuration:
+                        type: integer
+                      cloudProviderBackoffExponent:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        x-kubernetes-int-or-string: true
+                      cloudProviderBackoffJitter:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        x-kubernetes-int-or-string: true
+                      cloudProviderBackoffRetries:
+                        type: integer
+                    type: object
                   rateLimits:
                     items:
                       description: 'RateLimitSpec represents the rate limit configuration for a particular kind of resource. Eg. loadBalancerRateLimit is used to configure rate limits for load balancers. This eventually gets converted to CloudProviderRateLimitConfig that cloud-provider-azure expects. See: https://github.com/kubernetes-sigs/cloud-provider-azure/blob/d585c2031925b39c925624302f22f8856e29e352/pkg/provider/azure_ratelimit.go#L25 We cannot use CloudProviderRateLimitConfig directly because floating point values are not supported in controller-tools. See: https://github.com/kubernetes-sigs/controller-tools/issues/245'

config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml

@@ -301,6 +301,8 @@ func (cpc *CloudProviderConfig) overrideFromSpec(d azure.ClusterScoper) *CloudPr
 			cpc.AvailabilitySetRateLimit = toCloudProviderRateLimitConfig(rateLimit.Config)
 		}
 	}
+
+	cpc.BackOffConfig = toCloudProviderBackOffConfig(d.CloudProviderConfigOverrides().BackOffs)
 	return cpc
 }
 
@@ -341,6 +343,7 @@ type CloudProviderConfig struct {
 	UseInstanceMetadata          bool   `json:"useInstanceMetadata"`
 	UserAssignedIdentityID       string `json:"userAssignedIdentityId,omitempty"`
 	CloudProviderRateLimitConfig
+	BackOffConfig
 }
 
 // CloudProviderRateLimitConfig represents the rate limiting configurations in azure cloud provider config.
@@ -375,6 +378,31 @@ type RateLimitConfig struct {
 	CloudProviderRateLimitBucketWrite int     `json:"cloudProviderRateLimitBucketWrite,omitempty"`
 }
 
+// BackOffConfig indicates the back-off config options.
+// This is a copy of the struct used in cloud-provider-azure: https://github.com/kubernetes-sigs/cloud-provider-azure/blob/d585c2031925b39c925624302f22f8856e29e352/pkg/azureclients/azure_client_config.go#L48
+type BackOffConfig struct {
+	CloudProviderBackoff         bool    `json:"cloudProviderBackoff,omitempty"`
+	CloudProviderBackoffRetries  int     `json:"cloudProviderBackoffRetries,omitempty"`
+	CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"`
+	CloudProviderBackoffDuration int     `json:"cloudProviderBackoffDuration,omitempty"`
+	CloudProviderBackoffJitter   float64 `json:"cloudProviderBackoffJitter,omitempty"`
+}
+
+// toCloudProviderBackOffConfig returns converts infrav1.BackOffConfig to BackOffConfig that is required with the cloud provider.
+func toCloudProviderBackOffConfig(source infrav1.BackOffConfig) BackOffConfig {
+	backOffConfig := BackOffConfig{}
+	backOffConfig.CloudProviderBackoff = source.CloudProviderBackoff
+	if source.CloudProviderBackoffExponent != nil {
+		backOffConfig.CloudProviderBackoffExponent = source.CloudProviderBackoffExponent.AsApproximateFloat64()
+	}
+	backOffConfig.CloudProviderBackoffRetries = source.CloudProviderBackoffRetries
+	if source.CloudProviderBackoffJitter != nil {
+		backOffConfig.CloudProviderBackoffJitter = source.CloudProviderBackoffJitter.AsApproximateFloat64()
+	}
+	backOffConfig.CloudProviderBackoffDuration = source.CloudProviderBackoffDuration
+	return backOffConfig
+}
+
 func reconcileAzureSecret(ctx context.Context, log logr.Logger, kubeclient client.Client, owner metav1.OwnerReference, new *corev1.Secret, clusterName string) error {
 	ctx, span := tele.Tracer().Start(ctx, "controllers.reconcileAzureSecret")
 	defer span.End()

controllers/helpers.go

@@ -139,6 +139,13 @@ func TestGetCloudProviderConfig(t *testing.T) {
 			expectedControlPlaneConfig: rateLimitsControlPlaneCloudConfig,
 			expectedWorkerNodeConfig:   rateLimitsWorkerNodeCloudConfig,
 		},
+		"with back-off config": {
+			cluster:                    cluster,
+			azureCluster:               withbackOffConfig(*azureCluster),
+			identityType:               infrav1.VMIdentityNone,
+			expectedControlPlaneConfig: backOffCloudConfig,
+			expectedWorkerNodeConfig:   backOffCloudConfig,
+		},
 	}
 
 	os.Setenv(auth.ClientID, "fooClient")
@@ -331,6 +338,19 @@ func withRateLimits(ac infrav1.AzureCluster) *infrav1.AzureCluster {
 	return &ac
 }
 
+func withbackOffConfig(ac infrav1.AzureCluster) *infrav1.AzureCluster {
+	cloudProviderBackOffExponent := resource.MustParse("1.2")
+	backOff := infrav1.BackOffConfig{
+		CloudProviderBackoff:         true,
+		CloudProviderBackoffRetries:  1,
+		CloudProviderBackoffExponent: &cloudProviderBackOffExponent,
+		CloudProviderBackoffDuration: 60,
+		CloudProviderBackoffJitter:   &cloudProviderBackOffExponent,
+	}
+	ac.Spec.CloudProviderConfigOverrides = &infrav1.CloudProviderConfigOverrides{BackOffs: backOff}
+	return &ac
+}
+
 func newAzureClusterWithCustomVnet(name, location string) *infrav1.AzureCluster {
 	return &infrav1.AzureCluster{
 		ObjectMeta: metav1.ObjectMeta{
@@ -567,5 +587,30 @@ const (
     "loadBalancerRateLimit": {
         "cloudProviderRateLimitBucket": 10
     }
+}`
+	backOffCloudConfig = `{
+    "cloud": "AzurePublicCloud",
+    "tenantId": "fooTenant",
+    "subscriptionId": "baz",
+    "aadClientId": "fooClient",
+    "aadClientSecret": "fooSecret",
+    "resourceGroup": "bar",
+    "securityGroupName": "foo-node-nsg",
+    "securityGroupResourceGroup": "bar",
+    "location": "bar",
+    "vmType": "vmss",
+    "vnetName": "foo-vnet",
+    "vnetResourceGroup": "bar",
+    "subnetName": "foo-node-subnet",
+    "routeTableName": "foo-node-routetable",
+    "loadBalancerSku": "Standard",
+    "maximumLoadBalancerRuleCount": 250,
+    "useManagedIdentityExtension": false,
+    "useInstanceMetadata": true,
+    "cloudProviderBackoff": true,
+    "cloudProviderBackoffRetries": 1,
+    "cloudProviderBackoffExponent": 1.2000000000000002,
+    "cloudProviderBackoffDuration": 60,
+    "cloudProviderBackoffJitter": 1.2000000000000002
 }`
 )