Skip to content

Commit 952085e

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#3114 from marc-sensenich/feature-rate-limit-from-env
cluster-autoscaler: Allow Azure Rate Limit Defaults to be set from the environment
2 parents 362aec3 + 1d6f18f commit 952085e

File tree

3 files changed

+154
-14
lines changed

3 files changed

+154
-14
lines changed

‎cluster-autoscaler/cloudprovider/azure/README.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -260,12 +260,12 @@ The new version of [Azure client][] supports rate limit and back-off retries whe
260260
| CloudProviderBackoffDuration | 5 | BACKOFF_DURATION | cloudProviderBackoffDuration |
261261
| CloudProviderBackoffJitter | 1.0 | BACKOFF_JITTER | cloudProviderBackoffJitter |
262262
| CloudProviderRateLimit * | false | CLOUD_PROVIDER_RATE_LIMIT | cloudProviderRateLimit |
263-
| CloudProviderRateLimitQPS * | 1 | | cloudProviderRateLimitQPS |
264-
| CloudProviderRateLimitBucket * | 5 | | cloudProviderRateLimitBucket |
265-
| CloudProviderRateLimitQPSWrite * | 1 | | cloudProviderRateLimitQPSWrite |
266-
| CloudProviderRateLimitBucketWrite * | 5 | | cloudProviderRateLimitBucketWrite |
263+
| CloudProviderRateLimitQPS * | 1 | RATE_LIMIT_READ_QPS | cloudProviderRateLimitQPS |
264+
| CloudProviderRateLimitBucket * | 5 | RATE_LIMIT_READ_BUCKETS | cloudProviderRateLimitBucket |
265+
| CloudProviderRateLimitQPSWrite * | 1 | RATE_LIMIT_WRITE_QPS | cloudProviderRateLimitQPSWrite |
266+
| CloudProviderRateLimitBucketWrite * | 5 | RATE_LIMIT_WRITE_BUCKETS | cloudProviderRateLimitBucketWrite |
267267

268-
> **_NOTE_**: * These rate limit configs can be set per-client. Customizing `QPS` and `Bucket` through environment variables is not supported.
268+
> **_NOTE_**: * These rate limit configs can be set per-client. Customizing `QPS` and `Bucket` through environment variables per client is not supported.
269269

270270
[AKS]: https://docs.microsoft.com/azure/aks/
271271
[AKS autoscaler documentation]: https://docs.microsoft.com/azure/aks/autoscaler

‎cluster-autoscaler/cloudprovider/azure/azure_manager.go‎

Lines changed: 54 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,12 @@ const (
6666
backoffJitterDefault = 1.0
6767

6868
// rate limit
69-
rateLimitQPSDefault = 1.0
69+
rateLimitQPSDefault float32 = 1.0
7070
rateLimitBucketDefault = 5
71+
rateLimitReadQPSEnvVar = "RATE_LIMIT_READ_QPS"
72+
rateLimitReadBucketsEnvVar = "RATE_LIMIT_READ_BUCKETS"
73+
rateLimitWriteQPSEnvVar = "RATE_LIMIT_WRITE_QPS"
74+
rateLimitWriteBucketsEnvVar = "RATE_LIMIT_WRITE_BUCKETS"
7175
)
7276

7377
var validLabelAutoDiscovererKeys = strings.Join([]string{
@@ -148,31 +152,68 @@ type Config struct {
148152
}
149153

150154
// InitializeCloudProviderRateLimitConfig initializes rate limit configs.
151-
func InitializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig) {
155+
func InitializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig) error {
152156
if config == nil {
153-
return
157+
return nil
154158
}
155159

156160
// Assign read rate limit defaults if no configuration was passed in.
157161
if config.CloudProviderRateLimitQPS == 0 {
158-
config.CloudProviderRateLimitQPS = rateLimitQPSDefault
162+
if rateLimitQPSFromEnv := os.Getenv(rateLimitReadQPSEnvVar); rateLimitQPSFromEnv != "" {
163+
rateLimitQPS, err := strconv.ParseFloat(rateLimitQPSFromEnv, 0)
164+
if err != nil {
165+
return fmt.Errorf("failed to parse %s: %q, %v", rateLimitReadQPSEnvVar, rateLimitQPSFromEnv, err)
166+
}
167+
config.CloudProviderRateLimitQPS = float32(rateLimitQPS)
168+
} else {
169+
config.CloudProviderRateLimitQPS = rateLimitQPSDefault
170+
}
159171
}
172+
160173
if config.CloudProviderRateLimitBucket == 0 {
161-
config.CloudProviderRateLimitBucket = rateLimitBucketDefault
174+
if rateLimitBucketFromEnv := os.Getenv(rateLimitReadBucketsEnvVar); rateLimitBucketFromEnv != "" {
175+
rateLimitBucket, err := strconv.ParseInt(rateLimitBucketFromEnv, 10, 0)
176+
if err != nil {
177+
return fmt.Errorf("failed to parse %s: %q, %v", rateLimitReadBucketsEnvVar, rateLimitBucketFromEnv, err)
178+
}
179+
config.CloudProviderRateLimitBucket = int(rateLimitBucket)
180+
} else {
181+
config.CloudProviderRateLimitBucket = rateLimitBucketDefault
182+
}
162183
}
163-
// Assing write rate limit defaults if no configuration was passed in.
184+
185+
// Assign write rate limit defaults if no configuration was passed in.
164186
if config.CloudProviderRateLimitQPSWrite == 0 {
165-
config.CloudProviderRateLimitQPSWrite = config.CloudProviderRateLimitQPS
187+
if rateLimitQPSWriteFromEnv := os.Getenv(rateLimitWriteQPSEnvVar); rateLimitQPSWriteFromEnv != "" {
188+
rateLimitQPSWrite, err := strconv.ParseFloat(rateLimitQPSWriteFromEnv, 0)
189+
if err != nil {
190+
return fmt.Errorf("failed to parse %s: %q, %v", rateLimitWriteQPSEnvVar, rateLimitQPSWriteFromEnv, err)
191+
}
192+
config.CloudProviderRateLimitQPSWrite = float32(rateLimitQPSWrite)
193+
} else {
194+
config.CloudProviderRateLimitQPSWrite = config.CloudProviderRateLimitQPS
195+
}
166196
}
197+
167198
if config.CloudProviderRateLimitBucketWrite == 0 {
168-
config.CloudProviderRateLimitBucketWrite = config.CloudProviderRateLimitBucket
199+
if rateLimitBucketWriteFromEnv := os.Getenv(rateLimitWriteBucketsEnvVar); rateLimitBucketWriteFromEnv != "" {
200+
rateLimitBucketWrite, err := strconv.ParseInt(rateLimitBucketWriteFromEnv, 10, 0)
201+
if err != nil {
202+
return fmt.Errorf("failed to parse %s: %q, %v", rateLimitWriteBucketsEnvVar, rateLimitBucketWriteFromEnv, err)
203+
}
204+
config.CloudProviderRateLimitBucketWrite = int(rateLimitBucketWrite)
205+
} else {
206+
config.CloudProviderRateLimitBucketWrite = config.CloudProviderRateLimitBucket
207+
}
169208
}
170209

171210
config.InterfaceRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.InterfaceRateLimit)
172211
config.VirtualMachineRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineRateLimit)
173212
config.StorageAccountRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.StorageAccountRateLimit)
174213
config.DiskRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.DiskRateLimit)
175214
config.VirtualMachineScaleSetRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineScaleSetRateLimit)
215+
216+
return nil
176217
}
177218

178219
// overrideDefaultRateLimitConfig overrides the default CloudProviderRateLimitConfig.
@@ -358,7 +399,11 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node
358399
return nil, fmt.Errorf("failed to parse CLOUD_PROVIDER_RATE_LIMIT: %q, %v", cloudProviderRateLimit, err)
359400
}
360401
}
361-
InitializeCloudProviderRateLimitConfig(&cfg.CloudProviderRateLimitConfig)
402+
403+
err = InitializeCloudProviderRateLimitConfig(&cfg.CloudProviderRateLimitConfig)
404+
if err != nil {
405+
return nil, err
406+
}
362407

363408
// Defaulting vmType to vmss.
364409
if cfg.VMType == "" {

‎cluster-autoscaler/cloudprovider/azure/azure_manager_test.go‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package azure
1818

1919
import (
2020
"fmt"
21+
"os"
2122
"reflect"
2223
"strings"
2324
"testing"
@@ -366,3 +367,97 @@ func TestFetchAutoAsgsVmss(t *testing.T) {
366367
assert.Equal(t, minVal, asgs[0].MinSize())
367368
assert.Equal(t, maxVal, asgs[0].MaxSize())
368369
}
370+
371+
func TestInitializeCloudProviderRateLimitConfigWithNoConfigReturnsNoError(t *testing.T) {
372+
err := InitializeCloudProviderRateLimitConfig(nil)
373+
assert.Nil(t, err, "err should be nil")
374+
}
375+
376+
func TestInitializeCloudProviderRateLimitConfigWithNoRateLimitSettingsReturnsDefaults(t *testing.T) {
377+
emptyConfig := &CloudProviderRateLimitConfig{}
378+
err := InitializeCloudProviderRateLimitConfig(emptyConfig)
379+
380+
assert.NoError(t, err)
381+
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitQPSDefault)
382+
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitBucketDefault)
383+
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitQPSDefault)
384+
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitBucketDefault)
385+
}
386+
387+
func TestInitializeCloudProviderRateLimitConfigWithReadRateLimitSettingsFromEnv(t *testing.T) {
388+
emptyConfig := &CloudProviderRateLimitConfig{}
389+
var rateLimitReadQPS float32 = 3.0
390+
rateLimitReadBuckets := 10
391+
os.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS))
392+
os.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets))
393+
394+
err := InitializeCloudProviderRateLimitConfig(emptyConfig)
395+
assert.NoError(t, err)
396+
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitReadQPS)
397+
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitReadBuckets)
398+
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitReadQPS)
399+
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets)
400+
401+
os.Unsetenv(rateLimitReadBucketsEnvVar)
402+
os.Unsetenv(rateLimitReadQPSEnvVar)
403+
}
404+
405+
func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitSettingsFromEnv(t *testing.T) {
406+
emptyConfig := &CloudProviderRateLimitConfig{}
407+
var rateLimitReadQPS float32 = 3.0
408+
rateLimitReadBuckets := 10
409+
var rateLimitWriteQPS float32 = 6.0
410+
rateLimitWriteBuckets := 20
411+
412+
os.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS))
413+
os.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets))
414+
os.Setenv(rateLimitWriteQPSEnvVar, fmt.Sprintf("%.1f", rateLimitWriteQPS))
415+
os.Setenv(rateLimitWriteBucketsEnvVar, fmt.Sprintf("%d", rateLimitWriteBuckets))
416+
417+
err := InitializeCloudProviderRateLimitConfig(emptyConfig)
418+
419+
assert.NoError(t, err)
420+
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitReadQPS)
421+
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitReadBuckets)
422+
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS)
423+
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets)
424+
425+
os.Unsetenv(rateLimitReadQPSEnvVar)
426+
os.Unsetenv(rateLimitReadBucketsEnvVar)
427+
os.Unsetenv(rateLimitWriteQPSEnvVar)
428+
os.Unsetenv(rateLimitWriteBucketsEnvVar)
429+
}
430+
431+
func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitAlreadySetInConfig(t *testing.T) {
432+
var rateLimitReadQPS float32 = 3.0
433+
rateLimitReadBuckets := 10
434+
var rateLimitWriteQPS float32 = 6.0
435+
rateLimitWriteBuckets := 20
436+
437+
configWithRateLimits := &CloudProviderRateLimitConfig{
438+
RateLimitConfig: azclients.RateLimitConfig{
439+
CloudProviderRateLimitBucket: rateLimitReadBuckets,
440+
CloudProviderRateLimitBucketWrite: rateLimitWriteBuckets,
441+
CloudProviderRateLimitQPS: rateLimitReadQPS,
442+
CloudProviderRateLimitQPSWrite: rateLimitWriteQPS,
443+
},
444+
}
445+
446+
os.Setenv(rateLimitReadQPSEnvVar, "99")
447+
os.Setenv(rateLimitReadBucketsEnvVar, "99")
448+
os.Setenv(rateLimitWriteQPSEnvVar, "99")
449+
os.Setenv(rateLimitWriteBucketsEnvVar, "99")
450+
451+
err := InitializeCloudProviderRateLimitConfig(configWithRateLimits)
452+
453+
assert.NoError(t, err)
454+
assert.Equal(t, configWithRateLimits.CloudProviderRateLimitQPS, rateLimitReadQPS)
455+
assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucket, rateLimitReadBuckets)
456+
assert.Equal(t, configWithRateLimits.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS)
457+
assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets)
458+
459+
os.Unsetenv(rateLimitReadQPSEnvVar)
460+
os.Unsetenv(rateLimitReadBucketsEnvVar)
461+
os.Unsetenv(rateLimitWriteQPSEnvVar)
462+
os.Unsetenv(rateLimitWriteBucketsEnvVar)
463+
}

0 commit comments

Comments
 (0)