@@ -71,10 +71,14 @@ const (
7171 loadBalancerRuleNameMaxLength = 80
7272)
7373
74- var errNotInVMSet = errors .New ("vm is not in the vmset" )
75- var providerIDRE = regexp .MustCompile (`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$` )
76- var backendPoolIDRE = regexp .MustCompile (`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)` )
77- var nicResourceGroupRE = regexp .MustCompile (`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)` )
74+ var (
75+ errNotInVMSet = errors .New ("vm is not in the vmset" )
76+ providerIDRE = regexp .MustCompile (`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$` )
77+ backendPoolIDRE = regexp .MustCompile (`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)` )
78+ nicResourceGroupRE = regexp .MustCompile (`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)` )
79+ nicIDRE = regexp .MustCompile (`/subscriptions/(?:.*)/resourceGroups/(?:.+)/providers/Microsoft.Network/networkInterfaces/(.+)-nic-(.+)/ipConfigurations/(?:.*)` )
80+ vmasIDRE = regexp .MustCompile (`/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)` )
81+ )
7882
7983// getStandardMachineID returns the full identifier of a virtual machine.
8084func (az * Cloud ) getStandardMachineID (subscriptionID , resourceGroup , machineName string ) string {
@@ -700,7 +704,8 @@ func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node)
700704
701705// GetPrimaryInterface gets machine primary network interface by node name.
702706func (as * availabilitySet ) GetPrimaryInterface (nodeName string ) (network.Interface , error ) {
703- return as .getPrimaryInterfaceWithVMSet (nodeName , "" )
707+ nic , _ , err := as .getPrimaryInterfaceWithVMSet (nodeName , "" )
708+ return nic , err
704709}
705710
706711// extractResourceGroupByNicID extracts the resource group name by nicID.
@@ -714,26 +719,26 @@ func extractResourceGroupByNicID(nicID string) (string, error) {
714719}
715720
716721// getPrimaryInterfaceWithVMSet gets machine primary network interface by node name and vmSet.
717- func (as * availabilitySet ) getPrimaryInterfaceWithVMSet (nodeName , vmSetName string ) (network.Interface , error ) {
722+ func (as * availabilitySet ) getPrimaryInterfaceWithVMSet (nodeName , vmSetName string ) (network.Interface , string , error ) {
718723 var machine compute.VirtualMachine
719724
720725 machine , err := as .GetVirtualMachineWithRetry (types .NodeName (nodeName ), azcache .CacheReadTypeDefault )
721726 if err != nil {
722727 klog .V (2 ).Infof ("GetPrimaryInterface(%s, %s) abort backoff" , nodeName , vmSetName )
723- return network.Interface {}, err
728+ return network.Interface {}, "" , err
724729 }
725730
726731 primaryNicID , err := getPrimaryInterfaceID (machine )
727732 if err != nil {
728- return network.Interface {}, err
733+ return network.Interface {}, "" , err
729734 }
730735 nicName , err := getLastSegment (primaryNicID , "/" )
731736 if err != nil {
732- return network.Interface {}, err
737+ return network.Interface {}, "" , err
733738 }
734739 nodeResourceGroup , err := as .GetNodeResourceGroup (nodeName )
735740 if err != nil {
736- return network.Interface {}, err
741+ return network.Interface {}, "" , err
737742 }
738743
739744 // Check availability set name. Note that vmSetName is empty string when getting
@@ -748,31 +753,35 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
748753 if machine .AvailabilitySet == nil || ! strings .EqualFold (* machine .AvailabilitySet .ID , expectedAvailabilitySetName ) {
749754 klog .V (3 ).Infof (
750755 "GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)" , nicName , vmSetName )
751- return network.Interface {}, errNotInVMSet
756+ return network.Interface {}, "" , errNotInVMSet
752757 }
753758 }
754759
755760 nicResourceGroup , err := extractResourceGroupByNicID (primaryNicID )
756761 if err != nil {
757- return network.Interface {}, err
762+ return network.Interface {}, "" , err
758763 }
759764
760765 ctx , cancel := getContextWithCancel ()
761766 defer cancel ()
762767 nic , rerr := as .InterfacesClient .Get (ctx , nicResourceGroup , nicName , "" )
763768 if rerr != nil {
764- return network.Interface {}, rerr .Error ()
769+ return network.Interface {}, "" , rerr .Error ()
765770 }
766771
767- return nic , nil
772+ var availabilitySetID string
773+ if machine .VirtualMachineProperties != nil && machine .AvailabilitySet != nil {
774+ availabilitySetID = to .String (machine .AvailabilitySet .ID )
775+ }
776+ return nic , availabilitySetID , nil
768777}
769778
770779// EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
771780// participating in the specified LoadBalancer Backend Pool.
772781func (as * availabilitySet ) EnsureHostInPool (service * v1.Service , nodeName types.NodeName , backendPoolID string , vmSetName string , isInternal bool ) (string , string , string , * compute.VirtualMachineScaleSetVM , error ) {
773782 vmName := mapNodeNameToVMName (nodeName )
774783 serviceName := getServiceName (service )
775- nic , err := as .getPrimaryInterfaceWithVMSet (vmName , vmSetName )
784+ nic , _ , err := as .getPrimaryInterfaceWithVMSet (vmName , vmSetName )
776785 if err != nil {
777786 if err == errNotInVMSet {
778787 klog .V (3 ).Infof ("EnsureHostInPool skips node %s because it is not in the vmSet %s" , nodeName , vmSetName )
@@ -895,7 +904,111 @@ func (as *availabilitySet) EnsureHostsInPool(service *v1.Service, nodes []*v1.No
895904
896905// EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes.
897906func (as * availabilitySet ) EnsureBackendPoolDeleted (service * v1.Service , backendPoolID , vmSetName string , backendAddressPools * []network.BackendAddressPool ) error {
898- // Do nothing for availability set.
907+ // Returns nil if backend address pools already deleted.
908+ if backendAddressPools == nil {
909+ return nil
910+ }
911+
912+ mc := metrics .NewMetricContext ("services" , "vmas_ensure_backend_pool_deleted" , as .ResourceGroup , as .SubscriptionID , service .Name )
913+ isOperationSucceeded := false
914+ defer func () {
915+ mc .ObserveOperationWithResult (isOperationSucceeded )
916+ }()
917+
918+ ipConfigurationIDs := []string {}
919+ for _ , backendPool := range * backendAddressPools {
920+ if strings .EqualFold (to .String (backendPool .ID ), backendPoolID ) &&
921+ backendPool .BackendAddressPoolPropertiesFormat != nil &&
922+ backendPool .BackendIPConfigurations != nil {
923+ for _ , ipConf := range * backendPool .BackendIPConfigurations {
924+ if ipConf .ID == nil {
925+ continue
926+ }
927+
928+ ipConfigurationIDs = append (ipConfigurationIDs , * ipConf .ID )
929+ }
930+ }
931+ }
932+ nicUpdaters := make ([]func () error , 0 )
933+ errors := make ([]error , 0 )
934+ for i := range ipConfigurationIDs {
935+ ipConfigurationID := ipConfigurationIDs [i ]
936+ nodeName , err := as .GetNodeNameByIPConfigurationID (ipConfigurationID )
937+ if err != nil {
938+ klog .Errorf ("Failed to GetNodeNameByIPConfigurationID(%s): %v" , ipConfigurationID , err )
939+ errors = append (errors , err )
940+ continue
941+ }
942+
943+ vmName := mapNodeNameToVMName (types .NodeName (nodeName ))
944+ nic , vmasID , err := as .getPrimaryInterfaceWithVMSet (vmName , vmSetName )
945+ if err != nil {
946+ if err == errNotInVMSet {
947+ klog .V (3 ).Infof ("EnsureBackendPoolDeleted skips node %s because it is not in the vmSet %s" , nodeName , vmSetName )
948+ return nil
949+ }
950+
951+ klog .Errorf ("error: az.EnsureBackendPoolDeleted(%s), az.VMSet.GetPrimaryInterface.Get(%s, %s), err=%v" , nodeName , vmName , vmSetName , err )
952+ return err
953+ }
954+ matches := vmasIDRE .FindStringSubmatch (vmasID )
955+ if len (matches ) != 2 {
956+ return fmt .Errorf ("EnsureBackendPoolDeleted: failed to parse the VMAS ID %s: %v" , vmasID , err )
957+ }
958+ vmasName := matches [1 ]
959+ // Only remove nodes belonging to specified vmSet to basic LB backends.
960+ if ! strings .EqualFold (vmasName , vmSetName ) {
961+ klog .V (2 ).Infof ("EnsureBackendPoolDeleted: skipping the node %s belonging to another vm set %s" , nodeName , vmasName )
962+ continue
963+ }
964+
965+ if nic .ProvisioningState != nil && * nic .ProvisioningState == nicFailedState {
966+ klog .Warningf ("EnsureBackendPoolDeleted skips node %s because its primary nic %s is in Failed state" , nodeName , * nic .Name )
967+ return nil
968+ }
969+
970+ if nic .InterfacePropertiesFormat != nil && nic .InterfacePropertiesFormat .IPConfigurations != nil {
971+ newIPConfigs := * nic .IPConfigurations
972+ for j , ipConf := range newIPConfigs {
973+ if ! to .Bool (ipConf .Primary ) {
974+ continue
975+ }
976+ // found primary ip configuration
977+ if ipConf .LoadBalancerBackendAddressPools != nil {
978+ newLBAddressPools := * ipConf .LoadBalancerBackendAddressPools
979+ for k , pool := range newLBAddressPools {
980+ if strings .EqualFold (to .String (pool .ID ), backendPoolID ) {
981+ newLBAddressPools = append (newLBAddressPools [:k ], newLBAddressPools [k + 1 :]... )
982+ break
983+ }
984+ }
985+ newIPConfigs [j ].LoadBalancerBackendAddressPools = & newLBAddressPools
986+ }
987+ }
988+ nic .IPConfigurations = & newIPConfigs
989+ nicUpdaters = append (nicUpdaters , func () error {
990+ ctx , cancel := getContextWithCancel ()
991+ defer cancel ()
992+ klog .V (2 ).Infof ("EnsureBackendPoolDeleted begins to CreateOrUpdate for NIC(%s, %s) with backendPoolID %s" , as .resourceGroup , to .String (nic .Name ), backendPoolID )
993+ rerr := as .InterfacesClient .CreateOrUpdate (ctx , as .ResourceGroup , to .String (nic .Name ), nic )
994+ if rerr != nil {
995+ klog .Errorf ("EnsureBackendPoolDeleted CreateOrUpdate for NIC(%s, %s) failed with error %v" , as .resourceGroup , to .String (nic .Name ), rerr .Error ())
996+ return rerr .Error ()
997+ }
998+ return nil
999+ })
1000+ }
1001+ }
1002+ errs := utilerrors .AggregateGoroutines (nicUpdaters ... )
1003+ if errs != nil {
1004+ return utilerrors .Flatten (errs )
1005+ }
1006+ // Fail if there are other errors.
1007+ if len (errors ) > 0 {
1008+ return utilerrors .Flatten (utilerrors .NewAggregate (errors ))
1009+ }
1010+
1011+ isOperationSucceeded = true
8991012 return nil
9001013}
9011014
@@ -908,3 +1021,16 @@ func generateStorageAccountName(accountNamePrefix string) string {
9081021 }
9091022 return accountName
9101023}
1024+
1025+ // GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
1026+ func (as * availabilitySet ) GetNodeNameByIPConfigurationID (ipConfigurationID string ) (string , error ) {
1027+ matches := nicIDRE .FindStringSubmatch (ipConfigurationID )
1028+ if len (matches ) != 3 {
1029+ klog .V (4 ).Infof ("Can not extract VM name from ipConfigurationID (%s)" , ipConfigurationID )
1030+ return "" , fmt .Errorf ("invalid ip config ID %s" , ipConfigurationID )
1031+ }
1032+
1033+ prefix := matches [1 ]
1034+ suffix := matches [2 ]
1035+ return fmt .Sprintf ("%s-%s" , prefix , suffix ), nil
1036+ }
0 commit comments