Merge pull request kubernetes#94114 from MarcPow/azure-provider-ip-tags

Azure Cloud Provider should support Service annotations that allow for ip-tag control over the public ips created for LoadBalancer Services

Author: k8s-ci-robot

staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -82,6 +83,9 @@ const (
 	// ServiceAnnotationPIPName specifies the pip that will be applied to load balancer
 	ServiceAnnotationPIPName = "service.beta.kubernetes.io/azure-pip-name"
 
+	// ServiceAnnotationIPTagsForPublicIP specifies the iptags used when dynamically creating a public ip
+	ServiceAnnotationIPTagsForPublicIP = "service.beta.kubernetes.io/azure-pip-ip-tags"
+
 	// ServiceAnnotationAllowedServiceTag is the annotation used on the service
 	// to specify a list of allowed service tags separated by comma
 	// Refer https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#service-tags for all supported service tags.
@@ -546,6 +550,7 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai
 		pip.Location = to.StringPtr(az.Location)
 		pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{
 			PublicIPAllocationMethod: network.Static,
+			IPTags:                   getServiceIPTagRequestForPublicIP(service).IPTags,
 		}
 		pip.Tags = map[string]*string{
 			serviceTagKey:  &serviceName,
@@ -604,6 +609,93 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai
 	return &pip, nil
 }
 
+type serviceIPTagRequest struct {
+	IPTagsRequestedByAnnotation bool
+	IPTags                      *[]network.IPTag
+}
+
+// Get the ip tag Request for the public ip from service annotations.
+func getServiceIPTagRequestForPublicIP(service *v1.Service) serviceIPTagRequest {
+	if service != nil {
+		if ipTagString, found := service.Annotations[ServiceAnnotationIPTagsForPublicIP]; found {
+			return serviceIPTagRequest{
+				IPTagsRequestedByAnnotation: true,
+				IPTags:                      convertIPTagMapToSlice(getIPTagMap(ipTagString)),
+			}
+		}
+	}
+
+	return serviceIPTagRequest{
+		IPTagsRequestedByAnnotation: false,
+		IPTags:                      nil,
+	}
+}
+
+func getIPTagMap(ipTagString string) map[string]string {
+	outputMap := make(map[string]string)
+	commaDelimitedPairs := strings.Split(strings.TrimSpace(ipTagString), ",")
+	for _, commaDelimitedPair := range commaDelimitedPairs {
+		splitKeyValue := strings.Split(commaDelimitedPair, "=")
+
+		// Include only valid pairs in the return value
+		// Last Write wins.
+		if len(splitKeyValue) == 2 {
+			tagKey := strings.TrimSpace(splitKeyValue[0])
+			tagValue := strings.TrimSpace(splitKeyValue[1])
+
+			outputMap[tagKey] = tagValue
+		}
+	}
+
+	return outputMap
+}
+
+func sortIPTags(ipTags *[]network.IPTag) {
+	if ipTags != nil {
+		sort.Slice(*ipTags, func(i, j int) bool {
+			ipTag := *ipTags
+			return to.String(ipTag[i].IPTagType) < to.String(ipTag[j].IPTagType) ||
+				to.String(ipTag[i].Tag) < to.String(ipTag[j].Tag)
+		})
+	}
+}
+
+func areIPTagsEquivalent(ipTags1 *[]network.IPTag, ipTags2 *[]network.IPTag) bool {
+	sortIPTags(ipTags1)
+	sortIPTags(ipTags2)
+
+	if ipTags1 == nil {
+		ipTags1 = &[]network.IPTag{}
+	}
+
+	if ipTags2 == nil {
+		ipTags2 = &[]network.IPTag{}
+	}
+
+	return reflect.DeepEqual(ipTags1, ipTags2)
+}
+
+func convertIPTagMapToSlice(ipTagMap map[string]string) *[]network.IPTag {
+	if ipTagMap == nil {
+		return nil
+	}
+
+	if len(ipTagMap) == 0 {
+		return &[]network.IPTag{}
+	}
+
+	outputTags := []network.IPTag{}
+	for k, v := range ipTagMap {
+		ipTag := network.IPTag{
+			IPTagType: to.StringPtr(k),
+			Tag:       to.StringPtr(v),
+		}
+		outputTags = append(outputTags, ipTag)
+	}
+
+	return &outputTags
+}
+
 func getDomainNameLabel(pip *network.PublicIPAddress) string {
 	if pip == nil || pip.PublicIPAddressPropertiesFormat == nil || pip.PublicIPAddressPropertiesFormat.DNSSettings == nil {
 		return ""
@@ -1468,10 +1560,34 @@ func deduplicate(collection *[]string) *[]string {
 	return &result
 }
 
+// Determine if we should release existing owned public IPs
+func shouldReleaseExistingOwnedPublicIP(existingPip *network.PublicIPAddress, lbShouldExist bool, lbIsInternal bool, desiredPipName string, ipTagRequest serviceIPTagRequest) bool {
+	// Latch some variables for readability purposes.
+	pipName := *(*existingPip).Name
+
+	// Assume the current IP Tags are empty by default unless properties specify otherwise.
+	currentIPTags := &[]network.IPTag{}
+	pipPropertiesFormat := (*existingPip).PublicIPAddressPropertiesFormat
+	if pipPropertiesFormat != nil {
+		currentIPTags = (*pipPropertiesFormat).IPTags
+	}
+
+	// Release the ip under the following criteria -
+	// #1 - If we don't actually want a load balancer,
+	return !lbShouldExist ||
+		// #2 - If the load balancer is internal, and thus doesn't require public exposure
+		lbIsInternal ||
+		// #3 - If the name of this public ip does not match the desired name,
+		(pipName != desiredPipName) ||
+		// #4 If the service annotations have specified the ip tags that the public ip must have, but they do not match the ip tags of the existing instance
+		(ipTagRequest.IPTagsRequestedByAnnotation && !areIPTagsEquivalent(currentIPTags, ipTagRequest.IPTags))
+}
+
 // This reconciles the PublicIP resources similar to how the LB is reconciled.
 func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbName string, wantLb bool) (*network.PublicIPAddress, error) {
 	isInternal := requiresInternalLoadBalancer(service)
 	serviceName := getServiceName(service)
+	serviceIPTagRequest := getServiceIPTagRequestForPublicIP(service)
 	var lb *network.LoadBalancer
 	var desiredPipName string
 	var err error
@@ -1498,27 +1614,39 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa
 		return nil, err
 	}
 
-	var found bool
+	var serviceAnnotationRequestsNamedPublicIP bool = shouldPIPExisted
+	var discoveredDesiredPublicIP bool
+	var deletedDesiredPublicIP bool
 	var pipsToBeDeleted []*network.PublicIPAddress
 	for i := range pips {
 		pip := pips[i]
 		pipName := *pip.Name
-		if serviceOwnsPublicIP(&pip, clusterName, serviceName) {
-			// We need to process for pips belong to this service
-			if wantLb && !isInternal && pipName == desiredPipName {
-				// This is the only case we should preserve the
-				// Public ip resource with match service tag
-				found = true
-			} else {
-				pipsToBeDeleted = append(pipsToBeDeleted, &pip)
-			}
-		} else if wantLb && !isInternal && pipName == desiredPipName {
-			found = true
+
+		// If we've been told to use a specific public ip by the client, let's track whether or not it actually existed
+		// when we inspect the set in Azure.
+		discoveredDesiredPublicIP = discoveredDesiredPublicIP || wantLb && !isInternal && pipName == desiredPipName
+
+		// Now, let's perform additional analysis to determine if we should release the public ips we have found.
+		// We can only let them go if (a) they are owned by this service and (b) they meet the criteria for deletion.
+		if serviceOwnsPublicIP(&pip, clusterName, serviceName) &&
+			shouldReleaseExistingOwnedPublicIP(&pip, wantLb, isInternal, desiredPipName, serviceIPTagRequest) {
+
+			// Then, release the public ip
+			pipsToBeDeleted = append(pipsToBeDeleted, &pip)
+
+			// Flag if we deleted the desired public ip
+			deletedDesiredPublicIP = deletedDesiredPublicIP || pipName == desiredPipName
+
+			// An aside: It would be unusual, but possible, for us to delete a public ip referred to explicitly by name
+			// in Service annotations (which is usually reserved for non-service-owned externals), if that IP is tagged as
+			// having been owned by a particular Kubernetes cluster.
 		}
 	}
-	if !isInternal && shouldPIPExisted && !found && wantLb {
+
+	if !isInternal && serviceAnnotationRequestsNamedPublicIP && !discoveredDesiredPublicIP && wantLb {
 		return nil, fmt.Errorf("reconcilePublicIP for service(%s): pip(%s) not found", serviceName, desiredPipName)
 	}
+
 	var deleteFuncs []func() error
 	for _, pip := range pipsToBeDeleted {
 		pipCopy := *pip
@@ -1536,7 +1664,8 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa
 		// Confirm desired public ip resource exists
 		var pip *network.PublicIPAddress
 		domainNameLabel, found := getPublicIPDomainNameLabel(service)
-		if pip, err = az.ensurePublicIPExists(service, desiredPipName, domainNameLabel, clusterName, shouldPIPExisted, found); err != nil {
+		errorIfPublicIPDoesNotExist := serviceAnnotationRequestsNamedPublicIP && discoveredDesiredPublicIP && !deletedDesiredPublicIP
+		if pip, err = az.ensurePublicIPExists(service, desiredPipName, domainNameLabel, clusterName, errorIfPublicIPDoesNotExist, found); err != nil {
 			return nil, err
 		}
 		return pip, nil