Skip to content

Commit 92e0de3

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#86515 from feiskyer/separate-limit
Change Azure global rate limit to per client
2 parents 82c7c86 + 32e1c03 commit 92e0de3

File tree

6 files changed

+484
-209
lines changed

6 files changed

+484
-209
lines changed

‎staging/src/k8s.io/legacy-cloud-providers/azure/BUILD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ go_library(
2525
"azure_loadbalancer.go",
2626
"azure_managedDiskController.go",
2727
"azure_metrics.go",
28+
"azure_ratelimit.go",
2829
"azure_routes.go",
2930
"azure_standard.go",
3031
"azure_storage.go",
@@ -93,6 +94,7 @@ go_test(
9394
"azure_instances_test.go",
9495
"azure_loadbalancer_test.go",
9596
"azure_metrics_test.go",
97+
"azure_ratelimit_test.go",
9698
"azure_routes_test.go",
9799
"azure_standard_test.go",
98100
"azure_storage_test.go",

‎staging/src/k8s.io/legacy-cloud-providers/azure/azure.go‎

Lines changed: 39 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ import (
4242
"k8s.io/client-go/pkg/version"
4343
"k8s.io/client-go/tools/cache"
4444
"k8s.io/client-go/tools/record"
45-
"k8s.io/client-go/util/flowcontrol"
4645
cloudprovider "k8s.io/cloud-provider"
4746
"k8s.io/klog"
4847
"k8s.io/legacy-cloud-providers/azure/auth"
@@ -101,6 +100,7 @@ var (
101100
// for more details.
102101
type Config struct {
103102
auth.AzureAuthConfig
103+
CloudProviderRateLimitConfig
104104

105105
// The name of the resource group that the cluster is deployed in
106106
ResourceGroup string `json:"resourceGroup,omitempty" yaml:"resourceGroup,omitempty"`
@@ -149,17 +149,6 @@ type Config struct {
149149
// CloudProviderBackoffJitter are omitted.
150150
// "default" will be used if not specified.
151151
CloudProviderBackoffMode string `json:"cloudProviderBackoffMode,omitempty" yaml:"cloudProviderBackoffMode,omitempty"`
152-
// Enable rate limiting
153-
CloudProviderRateLimit bool `json:"cloudProviderRateLimit,omitempty" yaml:"cloudProviderRateLimit,omitempty"`
154-
// Rate limit QPS (Read)
155-
CloudProviderRateLimitQPS float32 `json:"cloudProviderRateLimitQPS,omitempty" yaml:"cloudProviderRateLimitQPS,omitempty"`
156-
// Rate limit Bucket Size
157-
CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty" yaml:"cloudProviderRateLimitBucket,omitempty"`
158-
// Rate limit QPS (Write)
159-
CloudProviderRateLimitQPSWrite float32 `json:"cloudProviderRateLimitQPSWrite,omitempty" yaml:"cloudProviderRateLimitQPSWrite,omitempty"`
160-
// Rate limit Bucket Size
161-
CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty" yaml:"cloudProviderRateLimitBucketWrite,omitempty"`
162-
163152
// Use instance metadata service where possible
164153
UseInstanceMetadata bool `json:"useInstanceMetadata,omitempty" yaml:"useInstanceMetadata,omitempty"`
165154

@@ -220,22 +209,27 @@ var _ cloudprovider.PVLabeler = (*Cloud)(nil)
220209
// Cloud holds the config and clients
221210
type Cloud struct {
222211
Config
223-
Environment azure.Environment
224-
RoutesClient RoutesClient
225-
SubnetsClient SubnetsClient
226-
InterfacesClient InterfacesClient
227-
RouteTablesClient RouteTablesClient
228-
LoadBalancerClient LoadBalancersClient
229-
PublicIPAddressesClient PublicIPAddressesClient
230-
SecurityGroupsClient SecurityGroupsClient
231-
VirtualMachinesClient VirtualMachinesClient
232-
StorageAccountClient StorageAccountClient
233-
DisksClient DisksClient
234-
SnapshotsClient *compute.SnapshotsClient
235-
FileClient FileClient
236-
ResourceRequestBackoff wait.Backoff
237-
metadata *InstanceMetadataService
238-
vmSet VMSet
212+
Environment azure.Environment
213+
214+
RoutesClient RoutesClient
215+
SubnetsClient SubnetsClient
216+
InterfacesClient InterfacesClient
217+
RouteTablesClient RouteTablesClient
218+
LoadBalancerClient LoadBalancersClient
219+
PublicIPAddressesClient PublicIPAddressesClient
220+
SecurityGroupsClient SecurityGroupsClient
221+
VirtualMachinesClient VirtualMachinesClient
222+
StorageAccountClient StorageAccountClient
223+
DisksClient DisksClient
224+
SnapshotsClient *compute.SnapshotsClient
225+
FileClient FileClient
226+
VirtualMachineScaleSetsClient VirtualMachineScaleSetsClient
227+
VirtualMachineScaleSetVMsClient VirtualMachineScaleSetVMsClient
228+
VirtualMachineSizesClient VirtualMachineSizesClient
229+
230+
ResourceRequestBackoff wait.Backoff
231+
metadata *InstanceMetadataService
232+
vmSet VMSet
239233

240234
// ipv6DualStack allows overriding for unit testing. It's normally initialized from featuregates
241235
ipv6DualStackEnabled bool
@@ -256,13 +250,6 @@ type Cloud struct {
256250
// routeCIDRs holds cache for route CIDRs.
257251
routeCIDRs map[string]string
258252

259-
// Clients for vmss.
260-
VirtualMachineScaleSetsClient VirtualMachineScaleSetsClient
261-
VirtualMachineScaleSetVMsClient VirtualMachineScaleSetVMsClient
262-
263-
// client for vm sizes list
264-
VirtualMachineSizesClient VirtualMachineSizesClient
265-
266253
kubeClient clientset.Interface
267254
eventBroadcaster record.EventBroadcaster
268255
eventRecorder record.EventRecorder
@@ -385,43 +372,8 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
385372
return err
386373
}
387374

388-
// operationPollRateLimiter.Accept() is a no-op if rate limits are configured off.
389-
operationPollRateLimiter := flowcontrol.NewFakeAlwaysRateLimiter()
390-
operationPollRateLimiterWrite := flowcontrol.NewFakeAlwaysRateLimiter()
391-
392-
// If reader is provided (and no writer) we will
393-
// use the same value for both.
394-
if config.CloudProviderRateLimit {
395-
// Assign rate limit defaults if no configuration was passed in
396-
if config.CloudProviderRateLimitQPS == 0 {
397-
config.CloudProviderRateLimitQPS = rateLimitQPSDefault
398-
}
399-
if config.CloudProviderRateLimitBucket == 0 {
400-
config.CloudProviderRateLimitBucket = rateLimitBucketDefault
401-
}
402-
if config.CloudProviderRateLimitQPSWrite == 0 {
403-
config.CloudProviderRateLimitQPSWrite = rateLimitQPSDefault
404-
}
405-
if config.CloudProviderRateLimitBucketWrite == 0 {
406-
config.CloudProviderRateLimitBucketWrite = rateLimitBucketDefault
407-
}
408-
409-
operationPollRateLimiter = flowcontrol.NewTokenBucketRateLimiter(
410-
config.CloudProviderRateLimitQPS,
411-
config.CloudProviderRateLimitBucket)
412-
413-
operationPollRateLimiterWrite = flowcontrol.NewTokenBucketRateLimiter(
414-
config.CloudProviderRateLimitQPSWrite,
415-
config.CloudProviderRateLimitBucketWrite)
416-
417-
klog.V(2).Infof("Azure cloudprovider (read ops) using rate limit config: QPS=%g, bucket=%d",
418-
config.CloudProviderRateLimitQPS,
419-
config.CloudProviderRateLimitBucket)
420-
421-
klog.V(2).Infof("Azure cloudprovider (write ops) using rate limit config: QPS=%g, bucket=%d",
422-
config.CloudProviderRateLimitQPSWrite,
423-
config.CloudProviderRateLimitBucketWrite)
424-
}
375+
// Initialize rate limiting config options.
376+
InitializeCloudProviderRateLimitConfig(&config.CloudProviderRateLimitConfig)
425377

426378
// Conditionally configure resource request backoff
427379
resourceRequestBackoff := wait.Backoff{
@@ -500,26 +452,25 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
500452
subscriptionID: config.SubscriptionID,
501453
resourceManagerEndpoint: env.ResourceManagerEndpoint,
502454
servicePrincipalToken: servicePrincipalToken,
503-
rateLimiterReader: operationPollRateLimiter,
504-
rateLimiterWriter: operationPollRateLimiterWrite,
505455
CloudProviderBackoffRetries: config.CloudProviderBackoffRetries,
506456
CloudProviderBackoffDuration: config.CloudProviderBackoffDuration,
507457
ShouldOmitCloudProviderBackoff: config.shouldOmitCloudProviderBackoff(),
508458
}
509-
az.DisksClient = newAzDisksClient(azClientConfig)
510-
az.SnapshotsClient = newSnapshotsClient(azClientConfig)
511-
az.RoutesClient = newAzRoutesClient(azClientConfig)
512-
az.SubnetsClient = newAzSubnetsClient(azClientConfig)
513-
az.InterfacesClient = newAzInterfacesClient(azClientConfig)
514-
az.RouteTablesClient = newAzRouteTablesClient(azClientConfig)
515-
az.LoadBalancerClient = newAzLoadBalancersClient(azClientConfig)
516-
az.SecurityGroupsClient = newAzSecurityGroupsClient(azClientConfig)
517-
az.StorageAccountClient = newAzStorageAccountClient(azClientConfig)
518-
az.VirtualMachinesClient = newAzVirtualMachinesClient(azClientConfig)
519-
az.PublicIPAddressesClient = newAzPublicIPAddressesClient(azClientConfig)
520-
az.VirtualMachineSizesClient = newAzVirtualMachineSizesClient(azClientConfig)
521-
az.VirtualMachineScaleSetsClient = newAzVirtualMachineScaleSetsClient(azClientConfig)
522-
az.VirtualMachineScaleSetVMsClient = newAzVirtualMachineScaleSetVMsClient(azClientConfig)
459+
az.DisksClient = newAzDisksClient(azClientConfig.WithRateLimiter(config.DiskRateLimit))
460+
az.SnapshotsClient = newSnapshotsClient(azClientConfig.WithRateLimiter(config.SnapshotRateLimit))
461+
az.RoutesClient = newAzRoutesClient(azClientConfig.WithRateLimiter(config.RouteRateLimit))
462+
az.SubnetsClient = newAzSubnetsClient(azClientConfig.WithRateLimiter(config.SubnetsRateLimit))
463+
az.InterfacesClient = newAzInterfacesClient(azClientConfig.WithRateLimiter(config.InterfaceRateLimit))
464+
az.RouteTablesClient = newAzRouteTablesClient(azClientConfig.WithRateLimiter(config.RouteTableRateLimit))
465+
az.LoadBalancerClient = newAzLoadBalancersClient(azClientConfig.WithRateLimiter(config.LoadBalancerRateLimit))
466+
az.SecurityGroupsClient = newAzSecurityGroupsClient(azClientConfig.WithRateLimiter(config.SecurityGroupRateLimit))
467+
az.StorageAccountClient = newAzStorageAccountClient(azClientConfig.WithRateLimiter(config.StorageAccountRateLimit))
468+
az.VirtualMachinesClient = newAzVirtualMachinesClient(azClientConfig.WithRateLimiter(config.VirtualMachineRateLimit))
469+
az.PublicIPAddressesClient = newAzPublicIPAddressesClient(azClientConfig.WithRateLimiter(config.PublicIPAddressRateLimit))
470+
az.VirtualMachineSizesClient = newAzVirtualMachineSizesClient(azClientConfig.WithRateLimiter(config.VirtualMachineSizeRateLimit))
471+
az.VirtualMachineScaleSetsClient = newAzVirtualMachineScaleSetsClient(azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit))
472+
az.VirtualMachineScaleSetVMsClient = newAzVirtualMachineScaleSetVMsClient(azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit))
473+
// TODO(feiskyer): refactor azureFileClient to Interface.
523474
az.FileClient = &azureFileClient{env: *env}
524475

525476
if az.MaximumLoadBalancerRuleCount == 0 {

0 commit comments

Comments
 (0)