Update vm, storageaccount, interface and disk client.

Author: t-qini

cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/config/dynamic"
 	"k8s.io/klog"
 	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
+	"k8s.io/legacy-cloud-providers/azure/retry"
 )
 
 const (
@@ -149,16 +150,16 @@ func (as *AgentPool) getVirtualMachinesFromCache() ([]compute.VirtualMachine, er
 		return virtualMachinesStatusCache.virtualMachines[as.Id()], nil
 	}
 	klog.V(4).Infof("getVirtualMachinesFromCache: get vms from API")
-	vms, err := as.GetVirtualMachines()
-	klog.V(4).Infof("getVirtualMachinesFromCache: got vms from API %+v", vms)
+	vms, rerr := as.GetVirtualMachines()
+	klog.V(4).Infof("getVirtualMachinesFromCache: got vms from API, len = %d", len(vms))
 
-	if err != nil {
-		if isAzureRequestsThrottled(err) {
-			klog.Warningf("getAllVirtualMachines: throttling with message %v, would return the cached vms", err)
+	if rerr != nil {
+		if isAzureRequestsThrottled(rerr) {
+			klog.Warningf("getAllVirtualMachines: throttling with message %v, would return the cached vms", rerr)
 			return virtualMachinesStatusCache.virtualMachines[as.Id()], nil
 		}
 
-		return []compute.VirtualMachine{}, err
+		return []compute.VirtualMachine{}, rerr.Error()
 	}
 
 	virtualMachinesStatusCache.virtualMachines[as.Id()] = vms
@@ -361,15 +362,16 @@ func (as *AgentPool) IncreaseSize(delta int) error {
 }
 
 // GetVirtualMachines returns list of nodes for the given agent pool.
-func (as *AgentPool) GetVirtualMachines() (instances []compute.VirtualMachine, err error) {
+func (as *AgentPool) GetVirtualMachines() ([]compute.VirtualMachine, *retry.Error) {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
-	result, err := as.manager.azClient.virtualMachinesClient.List(ctx, as.manager.config.ResourceGroup)
-	if err != nil {
-		return nil, err
+	result, rerr := as.manager.azClient.virtualMachinesClient.List(ctx, as.manager.config.ResourceGroup)
+	if rerr != nil {
+		return nil, rerr
 	}
 
+	instances := make([]compute.VirtualMachine, 0)
 	for _, instance := range result {
 		if instance.Tags == nil {
 			continue
@@ -555,9 +557,9 @@ func (as *AgentPool) deleteBlob(accountName, vhdContainer, vhdBlob string) error
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
-	storageKeysResult, err := as.manager.azClient.storageAccountsClient.ListKeys(ctx, as.manager.config.ResourceGroup, accountName)
-	if err != nil {
-		return err
+	storageKeysResult, rerr := as.manager.azClient.storageAccountsClient.ListKeys(ctx, as.manager.config.ResourceGroup, accountName)
+	if rerr != nil {
+		return rerr.Error()
 	}
 
 	keys := *storageKeysResult.Keys
@@ -578,15 +580,15 @@ func (as *AgentPool) deleteVirtualMachine(name string) error {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
-	vm, err := as.manager.azClient.virtualMachinesClient.Get(ctx, as.manager.config.ResourceGroup, name, "")
-	if err != nil {
-		if exists, _ := checkResourceExistsFromError(err); !exists {
+	vm, rerr := as.manager.azClient.virtualMachinesClient.Get(ctx, as.manager.config.ResourceGroup, name, "")
+	if rerr != nil {
+		if exists, _ := checkResourceExistsFromRetryError(rerr); !exists {
 			klog.V(2).Infof("VirtualMachine %s/%s has already been removed", as.manager.config.ResourceGroup, name)
 			return nil
 		}
 
-		klog.Errorf("failed to get VM: %s/%s: %s", as.manager.config.ResourceGroup, name, err.Error())
-		return err
+		klog.Errorf("failed to get VM: %s/%s: %s", as.manager.config.ResourceGroup, name, rerr.Error())
+		return rerr.Error()
 	}
 
 	vhd := vm.VirtualMachineProperties.StorageProfile.OsDisk.Vhd
@@ -602,7 +604,7 @@ func (as *AgentPool) deleteVirtualMachine(name string) error {
 	if nicID == nil {
 		klog.Warningf("NIC ID is not set for VM (%s/%s)", as.manager.config.ResourceGroup, name)
 	} else {
-		nicName, err = resourceName(*nicID)
+		nicName, err := resourceName(*nicID)
 		if err != nil {
 			return err
 		}
@@ -614,8 +616,8 @@ func (as *AgentPool) deleteVirtualMachine(name string) error {
 	defer deleteCancel()
 
 	klog.Infof("waiting for VirtualMachine deletion: %s/%s", as.manager.config.ResourceGroup, name)
-	_, err = as.manager.azClient.virtualMachinesClient.Delete(deleteCtx, as.manager.config.ResourceGroup, name)
-	_, realErr := checkResourceExistsFromError(err)
+	rerr = as.manager.azClient.virtualMachinesClient.Delete(deleteCtx, as.manager.config.ResourceGroup, name)
+	_, realErr := checkResourceExistsFromRetryError(rerr)
 	if realErr != nil {
 		return realErr
 	}
@@ -625,9 +627,9 @@ func (as *AgentPool) deleteVirtualMachine(name string) error {
 		klog.Infof("deleting nic: %s/%s", as.manager.config.ResourceGroup, nicName)
 		interfaceCtx, interfaceCancel := getContextWithCancel()
 		defer interfaceCancel()
-		_, err = as.manager.azClient.interfacesClient.Delete(interfaceCtx, as.manager.config.ResourceGroup, nicName)
+		rerr := as.manager.azClient.interfacesClient.Delete(interfaceCtx, as.manager.config.ResourceGroup, nicName)
 		klog.Infof("waiting for nic deletion: %s/%s", as.manager.config.ResourceGroup, nicName)
-		_, realErr := checkResourceExistsFromError(err)
+		_, realErr := checkResourceExistsFromRetryError(rerr)
 		if realErr != nil {
 			return realErr
 		}
@@ -657,8 +659,8 @@ func (as *AgentPool) deleteVirtualMachine(name string) error {
 			klog.Infof("deleting managed disk: %s/%s", as.manager.config.ResourceGroup, *osDiskName)
 			disksCtx, disksCancel := getContextWithCancel()
 			defer disksCancel()
-			_, err = as.manager.azClient.disksClient.Delete(disksCtx, as.manager.config.ResourceGroup, *osDiskName)
-			_, realErr := checkResourceExistsFromError(err)
+			rerr := as.manager.azClient.disksClient.Delete(disksCtx, as.manager.config.ResourceGroup, *osDiskName)
+			_, realErr := checkResourceExistsFromRetryError(rerr)
 			if realErr != nil {
 				return realErr
 			}

cluster-autoscaler/cloudprovider/azure/azure_cache.go

@@ -148,7 +148,7 @@ func (m *asgCache) FindForInstance(instance *azureRef, vmType string) (cloudprov
 	}
 
 	// Look up caches for the instance.
-	klog.V(4).Infof("FindForInstance: attempting to retrieve instance %v from cache", m.instanceToAsg)
+	klog.V(6).Infof("FindForInstance: attempting to retrieve instance %v from cache", m.instanceToAsg)
 	if asg := m.getInstanceFromCache(inst.Name); asg != nil {
 		klog.V(4).Infof("FindForInstance: found asg %s in cache", asg.Id())
 		return asg, nil

cluster-autoscaler/cloudprovider/azure/azure_client.go

@@ -23,32 +23,22 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2018-03-31/containerservice"
-	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network"
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2017-05-10/resources"
 	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-07-01/storage"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/Azure/go-autorest/autorest/azure"
 
 	"k8s.io/klog"
+	"k8s.io/legacy-cloud-providers/azure/clients/diskclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/interfaceclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/storageaccountclient"
+	"k8s.io/legacy-cloud-providers/azure/clients/vmclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/vmssclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient"
 )
 
-// VirtualMachinesClient defines needed functions for azure compute.VirtualMachinesClient.
-type VirtualMachinesClient interface {
-	Get(ctx context.Context, resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (result compute.VirtualMachine, err error)
-	Delete(ctx context.Context, resourceGroupName string, VMName string) (resp *http.Response, err error)
-	List(ctx context.Context, resourceGroupName string) (result []compute.VirtualMachine, err error)
-}
-
-// InterfacesClient defines needed functions for azure network.InterfacesClient.
-type InterfacesClient interface {
-	Delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) (resp *http.Response, err error)
-}
-
 // DeploymentsClient defines needed functions for azure network.DeploymentsClient.
 type DeploymentsClient interface {
 	Get(ctx context.Context, resourceGroupName string, deploymentName string) (result resources.DeploymentExtended, err error)
@@ -58,111 +48,6 @@ type DeploymentsClient interface {
 	Delete(ctx context.Context, resourceGroupName string, deploymentName string) (resp *http.Response, err error)
 }
 
-// DisksClient defines needed functions for azure disk.DisksClient.
-type DisksClient interface {
-	Delete(ctx context.Context, resourceGroupName string, diskName string) (resp *http.Response, err error)
-}
-
-// AccountsClient defines needed functions for azure storage.AccountsClient.
-type AccountsClient interface {
-	ListKeys(ctx context.Context, resourceGroupName string, accountName string) (result storage.AccountListKeysResult, err error)
-}
-
-// azVirtualMachinesClient implements VirtualMachinesClient.
-type azVirtualMachinesClient struct {
-	client compute.VirtualMachinesClient
-}
-
-func newAzVirtualMachinesClient(subscriptionID, endpoint string, servicePrincipalToken *adal.ServicePrincipalToken) *azVirtualMachinesClient {
-	virtualMachinesClient := compute.NewVirtualMachinesClient(subscriptionID)
-	virtualMachinesClient.BaseURI = endpoint
-	virtualMachinesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
-	virtualMachinesClient.PollingDelay = 5 * time.Second
-	configureUserAgent(&virtualMachinesClient.Client)
-
-	return &azVirtualMachinesClient{
-		client: virtualMachinesClient,
-	}
-}
-
-func (az *azVirtualMachinesClient) Get(ctx context.Context, resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (result compute.VirtualMachine, err error) {
-	klog.V(10).Infof("azVirtualMachinesClient.Get(%q,%q,%q): start", resourceGroupName, VMName, expand)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachinesClient.Get(%q,%q,%q): end", resourceGroupName, VMName, expand)
-	}()
-
-	return az.client.Get(ctx, resourceGroupName, VMName, expand)
-}
-
-func (az *azVirtualMachinesClient) Delete(ctx context.Context, resourceGroupName string, VMName string) (resp *http.Response, err error) {
-	klog.V(10).Infof("azVirtualMachinesClient.Delete(%q,%q): start", resourceGroupName, VMName)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachinesClient.Delete(%q,%q): end", resourceGroupName, VMName)
-	}()
-
-	future, err := az.client.Delete(ctx, resourceGroupName, VMName)
-	if err != nil {
-		return future.Response(), err
-	}
-
-	err = future.WaitForCompletionRef(ctx, az.client.Client)
-	return future.Response(), err
-}
-
-func (az *azVirtualMachinesClient) List(ctx context.Context, resourceGroupName string) (result []compute.VirtualMachine, err error) {
-	klog.V(10).Infof("azVirtualMachinesClient.List(%q): start", resourceGroupName)
-	defer func() {
-		klog.V(10).Infof("azVirtualMachinesClient.List(%q): end", resourceGroupName)
-	}()
-
-	iterator, err := az.client.ListComplete(ctx, resourceGroupName)
-	if err != nil {
-		return nil, err
-	}
-
-	result = make([]compute.VirtualMachine, 0)
-	for ; iterator.NotDone(); err = iterator.Next() {
-		if err != nil {
-			return nil, err
-		}
-
-		result = append(result, iterator.Value())
-	}
-
-	return result, nil
-}
-
-type azInterfacesClient struct {
-	client network.InterfacesClient
-}
-
-func newAzInterfacesClient(subscriptionID, endpoint string, servicePrincipalToken *adal.ServicePrincipalToken) *azInterfacesClient {
-	interfacesClient := network.NewInterfacesClient(subscriptionID)
-	interfacesClient.BaseURI = endpoint
-	interfacesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
-	interfacesClient.PollingDelay = 5 * time.Second
-	configureUserAgent(&interfacesClient.Client)
-
-	return &azInterfacesClient{
-		client: interfacesClient,
-	}
-}
-
-func (az *azInterfacesClient) Delete(ctx context.Context, resourceGroupName string, networkInterfaceName string) (resp *http.Response, err error) {
-	klog.V(10).Infof("azInterfacesClient.Delete(%q,%q): start", resourceGroupName, networkInterfaceName)
-	defer func() {
-		klog.V(10).Infof("azInterfacesClient.Delete(%q,%q): end", resourceGroupName, networkInterfaceName)
-	}()
-
-	future, err := az.client.Delete(ctx, resourceGroupName, networkInterfaceName)
-	if err != nil {
-		return future.Response(), err
-	}
-
-	err = future.WaitForCompletionRef(ctx, az.client.Client)
-	return future.Response(), err
-}
-
 type azDeploymentsClient struct {
 	client resources.DeploymentsClient
 }
@@ -250,70 +135,18 @@ func (az *azDeploymentsClient) Delete(ctx context.Context, resourceGroupName, de
 	return future.Response(), err
 }
 
-type azDisksClient struct {
-	client compute.DisksClient
-}
-
-func newAzDisksClient(subscriptionID, endpoint string, servicePrincipalToken *adal.ServicePrincipalToken) *azDisksClient {
-	disksClient := compute.NewDisksClient(subscriptionID)
-	disksClient.BaseURI = endpoint
-	disksClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
-	disksClient.PollingDelay = 5 * time.Second
-	configureUserAgent(&disksClient.Client)
-
-	return &azDisksClient{
-		client: disksClient,
-	}
-}
-
-func (az *azDisksClient) Delete(ctx context.Context, resourceGroupName string, diskName string) (resp *http.Response, err error) {
-	klog.V(10).Infof("azDisksClient.Delete(%q,%q): start", resourceGroupName, diskName)
-	defer func() {
-		klog.V(10).Infof("azDisksClient.Delete(%q,%q): end", resourceGroupName, diskName)
-	}()
-
-	future, err := az.client.Delete(ctx, resourceGroupName, diskName)
-	if err != nil {
-		return future.Response(), err
-	}
-
-	err = future.WaitForCompletionRef(ctx, az.client.Client)
-	return future.Response(), err
-}
-
 type azAccountsClient struct {
 	client storage.AccountsClient
 }
 
-func newAzAccountsClient(subscriptionID, endpoint string, servicePrincipalToken *adal.ServicePrincipalToken) *azAccountsClient {
-	accountsClient := storage.NewAccountsClient(subscriptionID)
-	accountsClient.BaseURI = endpoint
-	accountsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
-	accountsClient.PollingDelay = 5 * time.Second
-	configureUserAgent(&accountsClient.Client)
-
-	return &azAccountsClient{
-		client: accountsClient,
-	}
-}
-
-func (az *azAccountsClient) ListKeys(ctx context.Context, resourceGroupName string, accountName string) (result storage.AccountListKeysResult, err error) {
-	klog.V(10).Infof("azAccountsClient.ListKeys(%q,%q): start", resourceGroupName, accountName)
-	defer func() {
-		klog.V(10).Infof("azAccountsClient.ListKeys(%q,%q): end", resourceGroupName, accountName)
-	}()
-
-	return az.client.ListKeys(ctx, resourceGroupName, accountName)
-}
-
 type azClient struct {
 	virtualMachineScaleSetsClient   vmssclient.Interface
 	virtualMachineScaleSetVMsClient vmssvmclient.Interface
-	virtualMachinesClient           VirtualMachinesClient
+	virtualMachinesClient           vmclient.Interface
 	deploymentsClient               DeploymentsClient
-	interfacesClient                InterfacesClient
-	disksClient                     DisksClient
-	storageAccountsClient           AccountsClient
+	interfacesClient                interfaceclient.Interface
+	disksClient                     diskclient.Interface
+	storageAccountsClient           storageaccountclient.Interface
 	containerServicesClient         containerservice.ContainerServicesClient
 	managedContainerServicesClient  containerservice.ManagedClustersClient
 }
@@ -390,19 +223,23 @@ func newAzClient(cfg *Config, env *azure.Environment) (*azClient, error) {
 	scaleSetVMsClient := vmssvmclient.New(vmssVMClientConfig)
 	klog.V(5).Infof("Created scale set vm client with authorizer: %v", scaleSetVMsClient)
 
-	virtualMachinesClient := newAzVirtualMachinesClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, spt)
+	vmClientConfig := azClientConfig.WithRateLimiter(cfg.VirtualMachineRateLimit)
+	virtualMachinesClient := vmclient.New(vmClientConfig)
 	klog.V(5).Infof("Created vm client with authorizer: %v", virtualMachinesClient)
 
 	deploymentsClient := newAzDeploymentsClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, spt)
 	klog.V(5).Infof("Created deployments client with authorizer: %v", deploymentsClient)
 
-	interfacesClient := newAzInterfacesClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, spt)
+	interfaceClientConfig := azClientConfig.WithRateLimiter(cfg.InterfaceRateLimit)
+	interfacesClient := interfaceclient.New(interfaceClientConfig)
 	klog.V(5).Infof("Created interfaces client with authorizer: %v", interfacesClient)
 
-	storageAccountsClient := newAzAccountsClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, spt)
+	accountClientConfig := azClientConfig.WithRateLimiter(cfg.StorageAccountRateLimit)
+	storageAccountsClient := storageaccountclient.New(accountClientConfig)
 	klog.V(5).Infof("Created storage accounts client with authorizer: %v", storageAccountsClient)
 
-	disksClient := newAzDisksClient(cfg.SubscriptionID, env.ResourceManagerEndpoint, spt)
+	diskClientConfig := azClientConfig.WithRateLimiter(cfg.DiskRateLimit)
+	disksClient := diskclient.New(diskClientConfig)
 	klog.V(5).Infof("Created disks client with authorizer: %v", disksClient)
 
 	containerServicesClient := containerservice.NewContainerServicesClient(cfg.SubscriptionID)