Skip to content

Commit b046dbc

Browse files
authored
Merge pull request kubernetes#86740 from feiskyer/arm-vmss-clients
Add VMSS and VMSSVM clients with backoff retry
2 parents 59b4933 + e787525 commit b046dbc

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+3113
-386
lines changed

staging/src/k8s.io/legacy-cloud-providers/azure/BUILD

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ go_library(
2424
"azure_instances.go",
2525
"azure_loadbalancer.go",
2626
"azure_managedDiskController.go",
27-
"azure_metrics.go",
2827
"azure_ratelimit.go",
2928
"azure_routes.go",
3029
"azure_standard.go",
@@ -65,10 +64,11 @@ go_library(
6564
"//staging/src/k8s.io/cloud-provider/volume/errors:go_default_library",
6665
"//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library",
6766
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
68-
"//staging/src/k8s.io/component-base/metrics:go_default_library",
69-
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
7067
"//staging/src/k8s.io/legacy-cloud-providers/azure/auth:go_default_library",
7168
"//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library",
69+
"//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient:go_default_library",
70+
"//staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient:go_default_library",
71+
"//staging/src/k8s.io/legacy-cloud-providers/azure/metrics:go_default_library",
7272
"//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library",
7373
"//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute:go_default_library",
7474
"//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network:go_default_library",
@@ -93,7 +93,6 @@ go_test(
9393
"azure_controller_standard_test.go",
9494
"azure_instances_test.go",
9595
"azure_loadbalancer_test.go",
96-
"azure_metrics_test.go",
9796
"azure_ratelimit_test.go",
9897
"azure_routes_test.go",
9998
"azure_standard_test.go",
@@ -143,6 +142,7 @@ filegroup(
143142
":package-srcs",
144143
"//staging/src/k8s.io/legacy-cloud-providers/azure/auth:all-srcs",
145144
"//staging/src/k8s.io/legacy-cloud-providers/azure/clients:all-srcs",
145+
"//staging/src/k8s.io/legacy-cloud-providers/azure/metrics:all-srcs",
146146
"//staging/src/k8s.io/legacy-cloud-providers/azure/retry:all-srcs",
147147
],
148148
tags = ["automanaged"],

staging/src/k8s.io/legacy-cloud-providers/azure/azure.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ import (
4646
"k8s.io/klog"
4747
"k8s.io/legacy-cloud-providers/azure/auth"
4848
azclients "k8s.io/legacy-cloud-providers/azure/clients"
49+
"k8s.io/legacy-cloud-providers/azure/clients/vmssclient"
50+
"k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient"
4951
"k8s.io/legacy-cloud-providers/azure/retry"
5052
"sigs.k8s.io/yaml"
5153
)
@@ -480,8 +482,12 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
480482
az.VirtualMachinesClient = newAzVirtualMachinesClient(azClientConfig.WithRateLimiter(config.VirtualMachineRateLimit))
481483
az.PublicIPAddressesClient = newAzPublicIPAddressesClient(azClientConfig.WithRateLimiter(config.PublicIPAddressRateLimit))
482484
az.VirtualMachineSizesClient = newAzVirtualMachineSizesClient(azClientConfig.WithRateLimiter(config.VirtualMachineSizeRateLimit))
483-
az.VirtualMachineScaleSetsClient = newAzVirtualMachineScaleSetsClient(azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit))
484-
az.VirtualMachineScaleSetVMsClient = newAzVirtualMachineScaleSetVMsClient(azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit))
485+
486+
az.VirtualMachineScaleSetsClient = vmssclient.New(azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit))
487+
vmssVMClientConfig := azClientConfig.WithRateLimiter(config.VirtualMachineScaleSetRateLimit)
488+
vmssVMClientConfig.Backoff = vmssVMClientConfig.Backoff.WithNonRetriableErrors([]string{vmssVMNotActiveErrorMessage})
489+
az.VirtualMachineScaleSetVMsClient = vmssvmclient.New(vmssVMClientConfig)
490+
485491
// TODO(feiskyer): refactor azureFileClient to Interface.
486492
az.FileClient = &azureFileClient{env: *env}
487493

staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go

Lines changed: 24 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -693,77 +693,32 @@ func (az *Cloud) deleteRouteWithRetry(routeName string) error {
693693
})
694694
}
695695

696-
// UpdateVmssVMWithRetry invokes az.VirtualMachineScaleSetVMsClient.Update with exponential backoff retry
697-
func (az *Cloud) UpdateVmssVMWithRetry(resourceGroupName string, VMScaleSetName string, instanceID string, parameters compute.VirtualMachineScaleSetVM, source string) error {
698-
return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
699-
ctx, cancel := getContextWithCancel()
700-
defer cancel()
701-
702-
rerr := az.VirtualMachineScaleSetVMsClient.Update(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters, source)
703-
klog.V(10).Infof("UpdateVmssVMWithRetry: VirtualMachineScaleSetVMsClient.Update(%s,%s): end", VMScaleSetName, instanceID)
704-
if rerr == nil {
705-
return true, nil
706-
}
707-
708-
if strings.Contains(rerr.Error().Error(), vmssVMNotActiveErrorMessage) {
709-
// When instances are under deleting, updating API would report "not an active Virtual Machine Scale Set VM instanceId" error.
710-
// Since they're under deleting, we shouldn't send more update requests for it.
711-
klog.V(3).Infof("UpdateVmssVMWithRetry: VirtualMachineScaleSetVMsClient.Update(%s,%s) gets error message %q, abort backoff because it's probably under deleting", VMScaleSetName, instanceID, vmssVMNotActiveErrorMessage)
712-
return true, nil
713-
}
714-
715-
return !rerr.Retriable, rerr.Error()
716-
})
717-
}
718-
719-
// CreateOrUpdateVmssWithRetry invokes az.VirtualMachineScaleSetsClient.Update with exponential backoff retry
720-
func (az *Cloud) CreateOrUpdateVmssWithRetry(resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet) error {
721-
return wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
722-
ctx, cancel := getContextWithCancel()
723-
defer cancel()
724-
725-
// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
726-
// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
727-
klog.V(3).Infof("CreateOrUpdateVmssWithRetry: verify the status of the vmss being created or updated")
728-
vmss, rerr := az.VirtualMachineScaleSetsClient.Get(ctx, resourceGroupName, VMScaleSetName)
729-
if rerr != nil {
730-
klog.Warningf("CreateOrUpdateVmssWithRetry: error getting vmss: %v", rerr)
731-
}
732-
if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, virtualMachineScaleSetsDeallocating) {
733-
klog.V(3).Infof("CreateOrUpdateVmssWithRetry: found vmss %s being deleted, skipping", VMScaleSetName)
734-
return true, nil
735-
}
736-
737-
rerr = az.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, resourceGroupName, VMScaleSetName, parameters)
738-
klog.V(10).Infof("UpdateVmssVMWithRetry: VirtualMachineScaleSetsClient.CreateOrUpdate(%s): end", VMScaleSetName)
739-
if rerr == nil {
740-
return true, nil
741-
}
742-
743-
return !rerr.Retriable, rerr.Error()
744-
})
745-
}
746-
747-
// GetScaleSetWithRetry gets scale set with exponential backoff retry
748-
func (az *Cloud) GetScaleSetWithRetry(service *v1.Service, resourceGroupName, vmssName string) (compute.VirtualMachineScaleSet, error) {
749-
var result compute.VirtualMachineScaleSet
750-
var retryErr *retry.Error
751-
752-
err := wait.ExponentialBackoff(az.RequestBackoff(), func() (bool, error) {
753-
ctx, cancel := getContextWithCancel()
754-
defer cancel()
696+
// CreateOrUpdateVMSS invokes az.VirtualMachineScaleSetsClient.Update().
697+
func (az *Cloud) CreateOrUpdateVMSS(resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet) *retry.Error {
698+
ctx, cancel := getContextWithCancel()
699+
defer cancel()
700+
701+
// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
702+
// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
703+
klog.V(3).Infof("CreateOrUpdateVMSS: verify the status of the vmss being created or updated")
704+
vmss, rerr := az.VirtualMachineScaleSetsClient.Get(ctx, resourceGroupName, VMScaleSetName)
705+
if rerr != nil {
706+
klog.Errorf("CreateOrUpdateVMSS: error getting vmss(%s): %v", VMScaleSetName, rerr)
707+
return rerr
708+
}
709+
if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, virtualMachineScaleSetsDeallocating) {
710+
klog.V(3).Infof("CreateOrUpdateVMSS: found vmss %s being deleted, skipping", VMScaleSetName)
711+
return nil
712+
}
755713

756-
result, retryErr = az.VirtualMachineScaleSetsClient.Get(ctx, resourceGroupName, vmssName)
757-
if retryErr != nil {
758-
az.Event(service, v1.EventTypeWarning, "GetVirtualMachineScaleSet", retryErr.Error().Error())
759-
klog.Errorf("backoff: failure for scale set %q, will retry,err=%v", vmssName, retryErr)
760-
return false, nil
761-
}
762-
klog.V(4).Infof("backoff: success for scale set %q", vmssName)
763-
return true, nil
764-
})
714+
rerr = az.VirtualMachineScaleSetsClient.CreateOrUpdate(ctx, resourceGroupName, VMScaleSetName, parameters)
715+
klog.V(10).Infof("UpdateVmssVMWithRetry: VirtualMachineScaleSetsClient.CreateOrUpdate(%s): end", VMScaleSetName)
716+
if rerr != nil {
717+
klog.Errorf("CreateOrUpdateVMSS: error CreateOrUpdate vmss(%s): %v", VMScaleSetName, rerr)
718+
return rerr
719+
}
765720

766-
return result, err
721+
return nil
767722
}
768723

769724
func (cfg *Config) shouldOmitCloudProviderBackoff() bool {

0 commit comments

Comments
 (0)