Update vmss client.

Author: t-qini

cluster-autoscaler/cloudprovider/azure/README.md

@@ -244,14 +244,35 @@ Then deploy cluster-autoscaler by running
 kubectl create -f cluster-autoscaler-aks.yaml
 ```
 
-To deploy in AKS with `Helm 3`, please refer to [this tutorial](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler#azure-aks).
+To deploy in AKS with `Helm 3`, please refer to [helm installation tutorial][].
 
 Please see the [AKS autoscaler documentation][] for details.
 
+## Rate limit and back-off retries
+
+The new version of [Azure client][] supports rate limit and back-off retries when the cluster hits the throttling issue. These can be set by environment variables or cloud config file.
+
+| Config Name | Default | Environment Variable | Cloud Config File |
+| ----------- | ------- | -------------------- | ----------------- |
+| CloudProviderBackoff | false | ENABLE_BACKOFF | cloudProviderBackoff |
+| CloudProviderBackoffRetries | 6 | BACKOFF_RETRIES | cloudProviderBackoffRetries |
+| CloudProviderBackoffExponent | 1.5 | BACKOFF_EXPONENT | cloudProviderBackoffExponent |
+| CloudProviderBackoffDuration | 5 | BACKOFF_DURATION | cloudProviderBackoffDuration |
+| CloudProviderBackoffJitter | 1.0 | BACKOFF_JITTER | cloudProviderBackoffJitter |
+| CloudProviderRateLimit * | false | CLOUD_PROVIDER_RATE_LIMIT | cloudProviderRateLimit |
+| CloudProviderRateLimitQPS * | 1 | | cloudProviderRateLimitQPS |
+| CloudProviderRateLimitBucket * | 5 | | cloudProviderRateLimitBucket |
+| CloudProviderRateLimitQPSWrite * | 1 | | cloudProviderRateLimitQPSWrite |
+| CloudProviderRateLimitBucketWrite * | 5 | | cloudProviderRateLimitBucketWrite |
+
+> **_NOTE_**: * These rate limit configs can be set per-client. Customizing  `QPS` and `Bucket` through environment variables is not supported.
+
 [AKS]: https://docs.microsoft.com/azure/aks/
 [AKS autoscaler documentation]: https://docs.microsoft.com/azure/aks/autoscaler
 [aks-engine]: https://github.com/Azure/aks-engine
 [Azure CLI]: https://docs.microsoft.com/cli/azure/install-azure-cli
 [Azure Portal]: https://portal.azure.com
 [Releases]: ../../README.md#releases
 [service principal]: https://docs.microsoft.com/azure/active-directory/develop/app-objects-and-service-principals
+[helm installation tutorial]: https://github.com/helm/charts/tree/master/stable/cluster-autoscaler#azure-aks
+[Azure client]: https://github.com/kubernetes/kubernetes/tree/master/staging/src/k8s.io/legacy-cloud-providers/azure/clients

cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go

@@ -24,7 +24,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources"
 	azStorage "github.com/Azure/azure-sdk-for-go/storage"
 	"github.com/Azure/go-autorest/autorest/to"

cluster-autoscaler/cloudprovider/azure/azure_client.go

@@ -23,31 +23,20 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2018-03-31/containerservice"
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network"
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources"
 	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/Azure/go-autorest/autorest/azure"
+
 	"k8s.io/klog"
+	"k8s.io/legacy-cloud-providers/azure/clients/vmssclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient"
 )
 
-// VirtualMachineScaleSetsClient defines needed functions for azure compute.VirtualMachineScaleSetsClient.
-type VirtualMachineScaleSetsClient interface {
-	Get(ctx context.Context, resourceGroupName string, vmScaleSetName string) (result compute.VirtualMachineScaleSet, err error)
-	CreateOrUpdate(ctx context.Context, resourceGroupName string, name string, parameters compute.VirtualMachineScaleSet) (resp *http.Response, err error)
-	DeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) (resp *http.Response, err error)
-	List(ctx context.Context, resourceGroupName string) (result []compute.VirtualMachineScaleSet, err error)
-}
-
-// VirtualMachineScaleSetVMsClient defines needed functions for azure compute.VirtualMachineScaleSetVMsClient.
-type VirtualMachineScaleSetVMsClient interface {
-	Get(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string) (result compute.VirtualMachineScaleSetVM, err error)
-	List(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result []compute.VirtualMachineScaleSetVM, err error)
-}
-
 // VirtualMachinesClient defines needed functions for azure compute.VirtualMachinesClient.
 type VirtualMachinesClient interface {
 	Get(ctx context.Context, resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (result compute.VirtualMachine, err error)
@@ -79,134 +68,6 @@ type AccountsClient interface {
 	ListKeys(ctx context.Context, resourceGroupName string, accountName string) (result storage.AccountListKeysResult, err error)
 }
 
-// azVirtualMachineScaleSetsClient implements VirtualMachineScaleSetsClient.
-type azVirtualMachineScaleSetsClient struct {
-	client compute.VirtualMachineScaleSetsClient
-}
-
-func newAzVirtualMachineScaleSetsClient(subscriptionID, endpoint string, servicePrincipalToken *adal.ServicePrincipalToken) *azVirtualMachineScaleSetsClient {
-	virtualMachineScaleSetsClient := compute.NewVirtualMachineScaleSetsClient(subscriptionID)
-	virtualMachineScaleSetsClient.BaseURI = endpoint
-	virtualMachineScaleSetsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
-	virtualMachineScaleSetsClient.PollingDelay = 5 * time.Second
-	configureUserAgent(&virtualMachineScaleSetsClient.Client)
-
-	return &azVirtualMachineScaleSetsClient{
-		client: virtualMachineScaleSetsClient,
-	}
-}
-
-func (az *azVirtualMachineScaleSetsClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet) (resp *http.Response, err error) {
-	klog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, VMScaleSetName)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, VMScaleSetName)
-	}()
-
-	future, err := az.client.CreateOrUpdate(ctx, resourceGroupName, VMScaleSetName, parameters)
-	if err != nil {
-		return future.Response(), err
-	}
-
-	err = future.WaitForCompletionRef(ctx, az.client.Client)
-	return future.Response(), err
-}
-
-func (az *azVirtualMachineScaleSetsClient) Get(ctx context.Context, resourceGroupName string, VMScaleSetName string) (result compute.VirtualMachineScaleSet, err error) {
-	klog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): start", resourceGroupName, VMScaleSetName)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): end", resourceGroupName, VMScaleSetName)
-	}()
-
-	return az.client.Get(ctx, resourceGroupName, VMScaleSetName)
-}
-
-func (az *azVirtualMachineScaleSetsClient) List(ctx context.Context, resourceGroupName string) (result []compute.VirtualMachineScaleSet, err error) {
-	klog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q): start", resourceGroupName)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q): end", resourceGroupName)
-	}()
-
-	iterator, err := az.client.ListComplete(ctx, resourceGroupName)
-	if err != nil {
-		return nil, err
-	}
-
-	result = make([]compute.VirtualMachineScaleSet, 0)
-	for ; iterator.NotDone(); err = iterator.Next() {
-		if err != nil {
-			return nil, err
-		}
-
-		result = append(result, iterator.Value())
-	}
-
-	return result, nil
-}
-
-func (az *azVirtualMachineScaleSetsClient) DeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) (resp *http.Response, err error) {
-	klog.V(10).Infof("azVirtualMachineScaleSetsClient.DeleteInstances(%q,%q,%v): start", resourceGroupName, vmScaleSetName, vmInstanceIDs)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachineScaleSetsClient.DeleteInstances(%q,%q,%v): end", resourceGroupName, vmScaleSetName, vmInstanceIDs)
-	}()
-
-	future, err := az.client.DeleteInstances(ctx, resourceGroupName, vmScaleSetName, vmInstanceIDs)
-	if err != nil {
-		return future.Response(), err
-	}
-
-	err = future.WaitForCompletionRef(ctx, az.client.Client)
-	return future.Response(), err
-}
-
-// azVirtualMachineScaleSetVMsClient implements VirtualMachineScaleSetVMsClient.
-type azVirtualMachineScaleSetVMsClient struct {
-	client compute.VirtualMachineScaleSetVMsClient
-}
-
-func newAzVirtualMachineScaleSetVMsClient(subscriptionID, endpoint string, servicePrincipalToken *adal.ServicePrincipalToken) *azVirtualMachineScaleSetVMsClient {
-	virtualMachineScaleSetVMsClient := compute.NewVirtualMachineScaleSetVMsClient(subscriptionID)
-	virtualMachineScaleSetVMsClient.BaseURI = endpoint
-	virtualMachineScaleSetVMsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
-	virtualMachineScaleSetVMsClient.PollingDelay = 5 * time.Second
-	configureUserAgent(&virtualMachineScaleSetVMsClient.Client)
-
-	return &azVirtualMachineScaleSetVMsClient{
-		client: virtualMachineScaleSetVMsClient,
-	}
-}
-
-func (az *azVirtualMachineScaleSetVMsClient) Get(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string) (result compute.VirtualMachineScaleSetVM, err error) {
-	klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID)
-	}()
-
-	return az.client.Get(ctx, resourceGroupName, VMScaleSetName, instanceID)
-}
-
-func (az *azVirtualMachineScaleSetVMsClient) List(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result []compute.VirtualMachineScaleSetVM, err error) {
-	klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): start", resourceGroupName, virtualMachineScaleSetName, filter)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, filter)
-	}()
-
-	iterator, err := az.client.ListComplete(ctx, resourceGroupName, virtualMachineScaleSetName, filter, selectParameter, expand)
-	if err != nil {
-		return nil, err
-	}
-
-	result = make([]compute.VirtualMachineScaleSetVM, 0)
-	for ; iterator.NotDone(); err = iterator.Next() {
-		if err != nil {
-			return nil, err
-		}
-
-		result = append(result, iterator.Value())
-	}
-
-	return result, nil
-}
-
 // azVirtualMachinesClient implements VirtualMachinesClient.
 type azVirtualMachinesClient struct {
 	client compute.VirtualMachinesClient
@@ -446,8 +307,8 @@ func (az *azAccountsClient) ListKeys(ctx context.Context, resourceGroupName stri
 }
 
 type azClient struct {
-	virtualMachineScaleSetsClient   VirtualMachineScaleSetsClient
-	virtualMachineScaleSetVMsClient VirtualMachineScaleSetVMsClient
+	virtualMachineScaleSetsClient   vmssclient.Interface
+	virtualMachineScaleSetVMsClient vmssvmclient.Interface
 	virtualMachinesClient           VirtualMachinesClient
 	deploymentsClient               DeploymentsClient
 	interfacesClient                InterfacesClient
@@ -519,10 +380,14 @@ func newAzClient(cfg *Config, env *azure.Environment) (*azClient, error) {
 		return nil, err
 	}
 
-	scaleSetsClient := newAzVirtualMachineScaleSetsClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, spt)
+	azClientConfig := cfg.getAzureClientConfig(spt, env)
+
+	vmssClientConfig := azClientConfig.WithRateLimiter(cfg.VirtualMachineScaleSetRateLimit)
+	scaleSetsClient := vmssclient.New(vmssClientConfig)
 	klog.V(5).Infof("Created scale set client with authorizer: %v", scaleSetsClient)
 
-	scaleSetVMsClient := newAzVirtualMachineScaleSetVMsClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, spt)
+	vmssVMClientConfig := azClientConfig.WithRateLimiter(cfg.VirtualMachineScaleSetRateLimit)
+	scaleSetVMsClient := vmssvmclient.New(vmssVMClientConfig)
 	klog.V(5).Infof("Created scale set vm client with authorizer: %v", scaleSetVMsClient)
 
 	virtualMachinesClient := newAzVirtualMachinesClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, spt)

cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go

@@ -19,7 +19,7 @@ package azure
 import (
 	"testing"
 
-	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/stretchr/testify/assert"

cluster-autoscaler/cloudprovider/azure/azure_fakes.go

@@ -22,11 +22,12 @@ import (
 	"net/http"
 	"sync"
 
-	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources"
 	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/stretchr/testify/mock"
+	"k8s.io/legacy-cloud-providers/azure/retry"
 )
 
 const (
@@ -41,7 +42,7 @@ type VirtualMachineScaleSetsClientMock struct {
 }
 
 // Get gets the VirtualMachineScaleSet by vmScaleSetName.
-func (client *VirtualMachineScaleSetsClientMock) Get(ctx context.Context, resourceGroupName string, vmScaleSetName string) (result compute.VirtualMachineScaleSet, err error) {
+func (client *VirtualMachineScaleSetsClientMock) Get(ctx context.Context, resourceGroupName string, vmScaleSetName string) (result compute.VirtualMachineScaleSet, rerr *retry.Error) {
 	capacity := int64(2)
 	name := "Standard_D8_V3" // typo to test case-insensitive lookup
 	location := "switzerlandwest"
@@ -58,7 +59,7 @@ func (client *VirtualMachineScaleSetsClientMock) Get(ctx context.Context, resour
 }
 
 // CreateOrUpdate creates or updates the VirtualMachineScaleSet.
-func (client *VirtualMachineScaleSetsClientMock) CreateOrUpdate(ctx context.Context, resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet) (resp *http.Response, err error) {
+func (client *VirtualMachineScaleSetsClientMock) CreateOrUpdate(ctx context.Context, resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet) *retry.Error {
 	client.mutex.Lock()
 	defer client.mutex.Unlock()
 
@@ -67,19 +68,20 @@ func (client *VirtualMachineScaleSetsClientMock) CreateOrUpdate(ctx context.Cont
 	}
 	client.FakeStore[resourceGroupName][VMScaleSetName] = parameters
 
-	return &http.Response{
-		StatusCode: http.StatusOK,
-	}, nil
+	return nil
 }
 
 // DeleteInstances deletes a set of instances for specified VirtualMachineScaleSet.
-func (client *VirtualMachineScaleSetsClientMock) DeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) (resp *http.Response, err error) {
+func (client *VirtualMachineScaleSetsClientMock) DeleteInstances(ctx context.Context, resourceGroupName string, vmScaleSetName string, vmInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs) *retry.Error {
 	args := client.Called(resourceGroupName, vmScaleSetName, vmInstanceIDs)
-	return nil, args.Error(1)
+	if args.Error(1) != nil {
+		return &retry.Error{RawError: args.Error(1)}
+	}
+	return nil
 }
 
 // List gets a list of VirtualMachineScaleSets.
-func (client *VirtualMachineScaleSetsClientMock) List(ctx context.Context, resourceGroupName string) (result []compute.VirtualMachineScaleSet, err error) {
+func (client *VirtualMachineScaleSetsClientMock) List(ctx context.Context, resourceGroupName string) (result []compute.VirtualMachineScaleSet, rerr *retry.Error) {
 	client.mutex.Lock()
 	defer client.mutex.Unlock()
 
@@ -99,7 +101,7 @@ type VirtualMachineScaleSetVMsClientMock struct {
 }
 
 // Get gets a VirtualMachineScaleSetVM by VMScaleSetName and instanceID.
-func (m *VirtualMachineScaleSetVMsClientMock) Get(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string) (result compute.VirtualMachineScaleSetVM, err error) {
+func (m *VirtualMachineScaleSetVMsClientMock) Get(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, expand compute.InstanceViewTypes) (result compute.VirtualMachineScaleSetVM, rerr *retry.Error) {
 	ID := fakeVirtualMachineScaleSetVMID
 	vmID := "123E4567-E89B-12D3-A456-426655440000"
 	properties := compute.VirtualMachineScaleSetVMProperties{
@@ -113,7 +115,7 @@ func (m *VirtualMachineScaleSetVMsClientMock) Get(ctx context.Context, resourceG
 }
 
 // List gets a list of VirtualMachineScaleSetVMs.
-func (m *VirtualMachineScaleSetVMsClientMock) List(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result []compute.VirtualMachineScaleSetVM, err error) {
+func (m *VirtualMachineScaleSetVMsClientMock) List(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, expand string) (result []compute.VirtualMachineScaleSetVM, rerr *retry.Error) {
 	ID := fakeVirtualMachineScaleSetVMID
 	instanceID := "0"
 	vmID := "123E4567-E89B-12D3-A456-426655440000"
@@ -129,6 +131,16 @@ func (m *VirtualMachineScaleSetVMsClientMock) List(ctx context.Context, resource
 	return result, nil
 }
 
+// Update updates a  VirtualMachineScaleSetVM
+func (m *VirtualMachineScaleSetVMsClientMock) Update(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, parameters compute.VirtualMachineScaleSetVM, source string) *retry.Error {
+	return nil
+}
+
+// UpdateVMs updates a list of VirtualMachineScaleSetVM from map[instanceID]compute.VirtualMachineScaleSetVM.
+func (m *VirtualMachineScaleSetVMsClientMock) UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string) *retry.Error {
+	return nil
+}
+
 // VirtualMachinesClientMock mocks for VirtualMachinesClient.
 type VirtualMachinesClientMock struct {
 	mock.Mock