Merge pull request #128 from shiftstack/merge-bot-master

Merge https://github.com/kubernetes/cloud-provider-openstack:master into master

Author: openshift-merge-robot

cluster/images/cinder-csi-plugin/Dockerfile.build

@@ -4,6 +4,6 @@ FROM k8s.gcr.io/build-image/debian-base-${DEBIAN_ARCH}:v2.1.3
 ARG ARCH=amd64
 
 # Install e4fsprogs for format
-RUN clean-install ca-certificates e2fsprogs mount xfsprogs udev
+RUN clean-install btrfs-progs ca-certificates e2fsprogs mount udev xfsprogs
 
 ADD cinder-csi-plugin-${ARCH} /bin/cinder-csi-plugin

docs/openstack-cloud-controller-manager/expose-applications-using-loadbalancer-type-service.md

@@ -11,6 +11,7 @@
     - [Restrict Access For LoadBalancer Service](#restrict-access-for-loadbalancer-service)
     - [Use PROXY protocol to preserve client IP](#use-proxy-protocol-to-preserve-client-ip)
     - [Sharing load balancer with multiple Services](#sharing-load-balancer-with-multiple-services)
+    - [IPv4 / IPv6 dual-stack services](#ipv4--ipv6-dual-stack-services)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -580,3 +581,21 @@ $ openstack loadbalancer listener list --loadbalancer 2b224530-9414-4302-8163-5a
 ```
 
 The load balancer will be deleted after `service-2` is deleted. 
+
+### IPv4 / IPv6 dual-stack services
+Since Kubernetes 1.20, Kubernetes clusters can run in dual-stack mode,
+which allows simultaneous usage of both IPv4 and IPv6 addresses in the cluster.
+In dual-stack clusters, services can use IPv4, IPv6, or both address families, which
+can be indicated in service's `spec.ipFamilies`.
+
+If only one address family is specified in service's `spec.ipFamilies`, OCCM will respect
+that and create an IPv4 or IPv6 load balancer based on that.
+
+If two address families are specified in service's `spec.ipFamilies`, OCCM will respect the
+specified order and create an IPv4 or IPv6 load balancer based on the first specified address
+family. Note that creation of two load balancers for services with two `spec.ipFamilies`
+is not yet supported by OCCM.
+
+Internally, OCCM would automatically look for IPv4 or IPv6 subnet to allocate the load balancer
+address from based on the service's address family preference. If the subnet with preferred
+address family is not available, load balancer can not be created.

pkg/openstack/loadbalancer.go

@@ -48,6 +48,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/klog/v2"
+	netutils "k8s.io/utils/net"
 
 	"k8s.io/cloud-provider-openstack/pkg/metrics"
 	cpoutil "k8s.io/cloud-provider-openstack/pkg/util"
@@ -59,10 +60,11 @@ import (
 // Note: when creating a new Loadbalancer (VM), it can take some time before it is ready for use,
 // this timeout is used for waiting until the Loadbalancer provisioning status goes to ACTIVE state.
 const (
-	servicePrefix                   = "kube_service_"
-	defaultLoadBalancerSourceRanges = "0.0.0.0/0"
-	activeStatus                    = "ACTIVE"
-	annotationXForwardedFor         = "X-Forwarded-For"
+	servicePrefix                       = "kube_service_"
+	defaultLoadBalancerSourceRangesIPv4 = "0.0.0.0/0"
+	defaultLoadBalancerSourceRangesIPv6 = "::/0"
+	activeStatus                        = "ACTIVE"
+	annotationXForwardedFor             = "X-Forwarded-For"
 
 	ServiceAnnotationLoadBalancerInternal             = "service.beta.kubernetes.io/openstack-internal-load-balancer"
 	ServiceAnnotationLoadBalancerConnLimit            = "loadbalancer.openstack.org/connection-limit"
@@ -346,6 +348,7 @@ type serviceConfig struct {
 	healthMonitorDelay      int
 	healthMonitorTimeout    int
 	healthMonitorMaxRetries int
+	preferredIPFamily       corev1.IPFamily // preferred (the first) IP family indicated in service's `spec.ipFamilies`
 }
 
 type listenerKey struct {
@@ -651,8 +654,7 @@ func (lbaas *LbaasV2) createFullyPopulatedOctaviaLoadBalancer(name, clusterName
 	}
 
 	// In case subnet ID is not configured
-	if lbaas.opts.SubnetID == "" {
-		lbaas.opts.SubnetID = loadbalancer.VipSubnetID
+	if svcConf.lbMemberSubnetID == "" {
 		svcConf.lbMemberSubnetID = loadbalancer.VipSubnetID
 	}
 
@@ -726,7 +728,8 @@ func cutString(original string) string {
 // In case no InternalIP can be found, ExternalIP is tried.
 // If neither InternalIP nor ExternalIP can be found an error is
 // returned.
-func nodeAddressForLB(node *corev1.Node) (string, error) {
+// If preferredIPFamily is specified, only address of the specified IP family can be returned.
+func nodeAddressForLB(node *corev1.Node, preferredIPFamily corev1.IPFamily) (string, error) {
 	addrs := node.Status.Addresses
 	if len(addrs) == 0 {
 		return "", cpoerrors.ErrNoAddressFound
@@ -737,7 +740,18 @@ func nodeAddressForLB(node *corev1.Node) (string, error) {
 	for _, allowedAddrType := range allowedAddrTypes {
 		for _, addr := range addrs {
 			if addr.Type == allowedAddrType {
-				return addr.Address, nil
+				switch preferredIPFamily {
+				case corev1.IPv4Protocol:
+					if netutils.IsIPv4String(addr.Address) {
+						return addr.Address, nil
+					}
+				case corev1.IPv6Protocol:
+					if netutils.IsIPv6String(addr.Address) {
+						return addr.Address, nil
+					}
+				default:
+					return addr.Address, nil
+				}
 			}
 		}
 	}
@@ -801,8 +815,8 @@ func getBoolFromServiceAnnotation(service *corev1.Service, annotationKey string,
 }
 
 // getSubnetIDForLB returns subnet-id for a specific node
-func getSubnetIDForLB(compute *gophercloud.ServiceClient, node corev1.Node) (string, error) {
-	ipAddress, err := nodeAddressForLB(&node)
+func getSubnetIDForLB(compute *gophercloud.ServiceClient, node corev1.Node, preferredIPFamily corev1.IPFamily) (string, error) {
+	ipAddress, err := nodeAddressForLB(&node, preferredIPFamily)
 	if err != nil {
 		return "", err
 	}
@@ -1308,7 +1322,7 @@ func (lbaas *LbaasV2) buildBatchUpdateMemberOpts(port corev1.ServicePort, nodes
 	newMembers := sets.NewString()
 
 	for _, node := range nodes {
-		addr, err := nodeAddressForLB(node)
+		addr, err := nodeAddressForLB(node, svcConf.preferredIPFamily)
 		if err != nil {
 			if err == cpoerrors.ErrNoAddressFound {
 				// Node failure, do not create member
@@ -1477,6 +1491,12 @@ func (lbaas *LbaasV2) checkServiceUpdate(service *corev1.Service, nodes []*corev
 	}
 	serviceName := fmt.Sprintf("%s/%s", service.Namespace, service.Name)
 
+	if len(service.Spec.IPFamilies) > 0 {
+		// Since OCCM does not support multiple load-balancers per service yet,
+		// the first IP family will determine the IP family of the load-balancer
+		svcConf.preferredIPFamily = service.Spec.IPFamilies[0]
+	}
+
 	svcConf.lbID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerID, "")
 	svcConf.supportLBTags = openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureTags, lbaas.opts.LBProvider)
 
@@ -1497,7 +1517,7 @@ func (lbaas *LbaasV2) checkServiceUpdate(service *corev1.Service, nodes []*corev
 		} else {
 			svcConf.lbMemberSubnetID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerSubnetID, lbaas.opts.SubnetID)
 			if len(svcConf.lbMemberSubnetID) == 0 && len(nodes) > 0 {
-				subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0])
+				subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0], svcConf.preferredIPFamily)
 				if err != nil {
 					return fmt.Errorf("no subnet-id found for service %s: %v", serviceName, err)
 				}
@@ -1549,6 +1569,12 @@ func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node
 		return fmt.Errorf("no service ports provided")
 	}
 
+	if len(service.Spec.IPFamilies) > 0 {
+		// Since OCCM does not support multiple load-balancers per service yet,
+		// the first IP family will determine the IP family of the load-balancer
+		svcConf.preferredIPFamily = service.Spec.IPFamilies[0]
+	}
+
 	svcConf.lbID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerID, "")
 	svcConf.supportLBTags = openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureTags, lbaas.opts.LBProvider)
 
@@ -1558,6 +1584,9 @@ func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node
 			klog.V(3).InfoS("Enforcing internal LB", "annotation", true, "config", false)
 		}
 		svcConf.internal = true
+	} else if svcConf.preferredIPFamily == corev1.IPv6Protocol {
+		// floating IPs are not supported in IPv6 networks
+		svcConf.internal = true
 	} else {
 		svcConf.internal = getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerInternal, lbaas.opts.InternalLB)
 	}
@@ -1590,13 +1619,12 @@ func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node
 		svcConf.lbMemberSubnetID = svcConf.lbSubnetID
 	}
 	if len(svcConf.lbNetworkID) == 0 && len(svcConf.lbSubnetID) == 0 {
-		subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0])
+		subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0], svcConf.preferredIPFamily)
 		if err != nil {
 			return fmt.Errorf("failed to get subnet to create load balancer for service %s: %v", serviceName, err)
 		}
 		svcConf.lbSubnetID = subnetID
 		svcConf.lbMemberSubnetID = subnetID
-		lbaas.opts.SubnetID = subnetID
 	}
 
 	if !svcConf.internal {
@@ -1698,7 +1726,7 @@ func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node
 	}
 
 	var listenerAllowedCIDRs []string
-	sourceRanges, err := GetLoadBalancerSourceRanges(service)
+	sourceRanges, err := GetLoadBalancerSourceRanges(service, svcConf.preferredIPFamily)
 	if err != nil {
 		return fmt.Errorf("failed to get source ranges for loadbalancer service %s: %v", serviceName, err)
 	}
@@ -1932,7 +1960,7 @@ func (lbaas *LbaasV2) ensureOctaviaLoadBalancer(ctx context.Context, clusterName
 	}
 
 	if lbaas.opts.ManageSecurityGroups {
-		err := lbaas.ensureSecurityGroup(clusterName, service, nodes, loadbalancer)
+		err := lbaas.ensureSecurityGroup(clusterName, service, nodes, loadbalancer, svcConf.preferredIPFamily, svcConf.lbMemberSubnetID)
 		if err != nil {
 			return status, fmt.Errorf("failed when reconciling security groups for LB service %v/%v: %v", service.Namespace, service.Name, err)
 		}
@@ -1972,7 +2000,7 @@ func (lbaas *LbaasV2) ensureLoadBalancer(ctx context.Context, clusterName string
 	if len(lbaas.opts.SubnetID) == 0 && len(lbaas.opts.NetworkID) == 0 {
 		// Get SubnetID automatically.
 		// The LB needs to be configured with instance addresses on the same subnet, so get SubnetID by one node.
-		subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0])
+		subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0], "")
 		if err != nil {
 			klog.Warningf("Failed to find subnet-id for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
 			return nil, fmt.Errorf("no subnet-id for service %s/%s : subnet-id not set in cloud provider config, "+
@@ -2063,7 +2091,7 @@ func (lbaas *LbaasV2) ensureLoadBalancer(ctx context.Context, clusterName string
 		}
 	}
 
-	sourceRanges, err := GetLoadBalancerSourceRanges(apiService)
+	sourceRanges, err := GetLoadBalancerSourceRanges(apiService, "")
 	if err != nil {
 		return nil, fmt.Errorf("failed to get source ranges for loadbalancer service %s: %v", serviceName, err)
 	}
@@ -2201,7 +2229,7 @@ func (lbaas *LbaasV2) ensureLoadBalancer(ctx context.Context, clusterName string
 			return nil, fmt.Errorf("error getting pool members %s: %v", pool.ID, err)
 		}
 		for _, node := range nodes {
-			addr, err := nodeAddressForLB(node)
+			addr, err := nodeAddressForLB(node, "")
 			if err != nil {
 				if err == cpoerrors.ErrNotFound {
 					// Node failure, do not create member
@@ -2418,7 +2446,7 @@ func (lbaas *LbaasV2) ensureLoadBalancer(ctx context.Context, clusterName string
 	}
 
 	if lbaas.opts.ManageSecurityGroups {
-		err := lbaas.ensureSecurityGroup(clusterName, apiService, nodes, loadbalancer)
+		err := lbaas.ensureSecurityGroup(clusterName, apiService, nodes, loadbalancer, "", lbaas.opts.SubnetID)
 		if err != nil {
 			return status, fmt.Errorf("failed when reconciling security groups for LB service %v/%v: %v", apiService.Namespace, apiService.Name, err)
 		}
@@ -2476,7 +2504,8 @@ func (lbaas *LbaasV2) getSubnet(subnet string) (*subnets.Subnet, error) {
 
 // ensureSecurityGroup ensures security group exist for specific loadbalancer service.
 // Creating security group for specific loadbalancer service when it does not exist.
-func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1.Service, nodes []*corev1.Node, loadbalancer *loadbalancers.LoadBalancer) error {
+func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1.Service, nodes []*corev1.Node,
+	loadbalancer *loadbalancers.LoadBalancer, preferredIPFamily corev1.IPFamily, memberSubnetID string) error {
 	// find node-security-group for service
 	var err error
 	if len(lbaas.opts.NodeSecurityGroupIDs) == 0 && !lbaas.opts.UseOctavia {
@@ -2495,7 +2524,7 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1
 	}
 
 	// get service source ranges
-	sourceRanges, err := GetLoadBalancerSourceRanges(apiService)
+	sourceRanges, err := GetLoadBalancerSourceRanges(apiService, preferredIPFamily)
 	if err != nil {
 		return fmt.Errorf("failed to get source ranges for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
 	}
@@ -2628,9 +2657,9 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1
 		// traffic from Octavia amphorae to the node port on the worker nodes.
 		if lbaas.opts.UseOctavia {
 			mc := metrics.NewMetricContext("subnet", "get")
-			subnet, err := subnets.Get(lbaas.network, lbaas.opts.SubnetID).Extract()
+			subnet, err := subnets.Get(lbaas.network, memberSubnetID).Extract()
 			if mc.ObserveRequest(err) != nil {
-				return fmt.Errorf("failed to find subnet %s from openstack: %v", lbaas.opts.SubnetID, err)
+				return fmt.Errorf("failed to find subnet %s from openstack: %v", memberSubnetID, err)
 			}
 
 			sgListopts := rules.ListOpts{
@@ -2649,6 +2678,11 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1
 				continue
 			}
 
+			ethertype := rules.EtherType4
+			if netutils.IsIPv6CIDRString(subnet.CIDR) {
+				ethertype = rules.EtherType6
+			}
+
 			// The Octavia amphorae and worker nodes are supposed to be in the same subnet. We allow the ingress traffic
 			// from the amphorae to the specific node port on the nodes.
 			sgRuleCreateOpts := rules.CreateOpts{
@@ -2658,7 +2692,7 @@ func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1
 				Protocol:       toRuleProtocol(port.Protocol),
 				RemoteIPPrefix: subnet.CIDR,
 				SecGroupID:     lbSecGroupID,
-				EtherType:      rules.EtherType4,
+				EtherType:      ethertype,
 			}
 			mc = metrics.NewMetricContext("security_group_rule", "create")
 			_, err = rules.Create(lbaas.network, sgRuleCreateOpts).Extract()
@@ -2797,7 +2831,7 @@ func (lbaas *LbaasV2) updateLoadBalancer(ctx context.Context, clusterName string
 	if len(lbaas.opts.SubnetID) == 0 && len(nodes) > 0 {
 		// Get SubnetID automatically.
 		// The LB needs to be configured with instance addresses on the same subnet, so get SubnetID by one node.
-		subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0])
+		subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0], "")
 		if err != nil {
 			klog.Warningf("Failed to find subnet-id for loadbalancer service %s/%s: %v", service.Namespace, service.Name, err)
 			return fmt.Errorf("no subnet-id for service %s/%s : subnet-id not set in cloud provider config, "+
@@ -2851,7 +2885,7 @@ func (lbaas *LbaasV2) updateLoadBalancer(ctx context.Context, clusterName string
 	// Compose Set of member (addresses) that _should_ exist
 	addrs := make(map[string]*corev1.Node)
 	for _, node := range nodes {
-		addr, err := nodeAddressForLB(node)
+		addr, err := nodeAddressForLB(node, "")
 		if err != nil {
 			return err
 		}
@@ -3332,7 +3366,7 @@ func IsAllowAll(ipnets netsets.IPNet) bool {
 // GetLoadBalancerSourceRanges first try to parse and verify LoadBalancerSourceRanges field from a service.
 // If the field is not specified, turn to parse and verify the AnnotationLoadBalancerSourceRangesKey annotation from a service,
 // extracting the source ranges to allow, and if not present returns a default (allow-all) value.
-func GetLoadBalancerSourceRanges(service *corev1.Service) (netsets.IPNet, error) {
+func GetLoadBalancerSourceRanges(service *corev1.Service, preferredIPFamily corev1.IPFamily) (netsets.IPNet, error) {
 	var ipnets netsets.IPNet
 	var err error
 	// if SourceRange field is specified, ignore sourceRange annotation
@@ -3347,7 +3381,11 @@ func GetLoadBalancerSourceRanges(service *corev1.Service) (netsets.IPNet, error)
 		val := service.Annotations[corev1.AnnotationLoadBalancerSourceRangesKey]
 		val = strings.TrimSpace(val)
 		if val == "" {
-			val = defaultLoadBalancerSourceRanges
+			if preferredIPFamily == corev1.IPv6Protocol {
+				val = defaultLoadBalancerSourceRangesIPv6
+			} else {
+				val = defaultLoadBalancerSourceRangesIPv4
+			}
 		}
 		specs := strings.Split(val, ",")
 		ipnets, err = netsets.ParseIPNets(specs...)

pkg/util/errors/errors.go

@@ -54,6 +54,10 @@ func IsNotFound(err error) bool {
 		}
 	}
 
+	if err == ErrNotFound {
+		return true
+	}
+
 	return false
 }