[OCCM] add support for TLS terminated loadbalancers (kubernetes#1474)

* add support for TLS terminated loadbalancers

* update docs

Author: hamzazafar

docs/openstack-cloud-controller-manager/expose-applications-using-loadbalancer-type-service.md

@@ -132,7 +132,9 @@ Request Body:
 
 - `loadbalancer.openstack.org/x-forwarded-for`
 
-  If 'true', `X-Forwarded-For` is inserted into the HTTP headers which contains the original client IP address so that the backend HTTP service is able to get the real source IP of the request. Only applies when using Octavia.
+  If 'true', `X-Forwarded-For` is inserted into the HTTP headers which contains the original client IP address so that the backend HTTP service is able to get the real source IP of the request. Please note that the cloud provider will force the creation of an Octavia listener of type `HTTP` if this option is set. Only applies when using Octavia.
+
+  This annotation also works in conjunction with the `loadbalancer.openstack.org/default-tls-container-ref` annotation. In this case the cloud provider will create an Octavia listener of type `TERMINATED_HTTPS` instead of an `HTTP` listener.
 
 - `loadbalancer.openstack.org/timeout-client-data`
 
@@ -166,6 +168,11 @@ Request Body:
 
   The name of the loadbalancer availability zone to use. It is ignored if the Octavia version doesn't support availability zones yet.
 
+- `loadbalancer.openstack.org/default-tls-container-ref`
+
+  Reference to a tls container. This option works with Octavia, when this option is set then the cloud provider will create an Octavia Listener of type `TERMINATED_HTTPS` for a TLS Terminated loadbalancer.
+  Format for tls container ref: `https://{keymanager_host}/v1/containers/{uuid}`
+
 ### Switching between Floating Subnets by using preconfigured Classes
 
 If you have multiple `FloatingIPPools` and/or `FloatingIPSubnets` it might be desirable to offer the user logical meanings for `LoadBalancers` like `internetFacing` or `DMZ` instead of requiring the user to select a dedicated network or subnet ID at the service object level as an annotation.

docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md

@@ -244,6 +244,11 @@ Although the openstack-cloud-controller-manager was initially implemented with N
 
   This option is currently a workaround for the issue https://github.com/kubernetes/ingress-nginx/issues/3996, should be removed or refactored after the Kubernetes [KEP-1860](https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1860-kube-proxy-IP-node-binding) is implemented.
 
+* `default-tls-container-ref`
+  Reference to a tls container. This option works with Octavia, when this option is set then the cloud provider will create an Octavia Listener of type TERMINATED_HTTPS for a TLS Terminated loadbalancer.
+
+  Format for tls container ref: `https://{keymanager_host}/v1/containers/{uuid}`
+
 ### Metadata
 
 * `search-order`

pkg/client/service.go

@@ -64,3 +64,12 @@ func NewLoadBalancerV2(provider *gophercloud.ProviderClient, eo *gophercloud.End
 	}
 	return lb, nil
 }
+
+// NewKeyManagerV1 creates a ServiceClient that can be used with KeyManager v1 API
+func NewKeyManagerV1(provider *gophercloud.ProviderClient, eo *gophercloud.EndpointOpts) (*gophercloud.ServiceClient, error) {
+	secret, err := openstack.NewKeyManagerV1(provider, *eo)
+	if err != nil {
+		return nil, fmt.Errorf("unable to initialize keymanager client for region %s: %v", eo.Region, err)
+	}
+	return secret, nil
+}

pkg/openstack/loadbalancer.go

@@ -30,6 +30,7 @@ import (
 	"gopkg.in/godo.v2/glob"
 
 	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack/keymanager/v1/containers"
 	"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners"
 	"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers"
 	v2monitors "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors"
@@ -107,7 +108,7 @@ const (
 	// ServiceAnnotationLoadBalancerEnableHealthMonitor defines whether or not to create health monitor for the load balancer
 	// pool, if not specified, use 'create-monitor' config. The health monitor can be created or deleted dynamically.
 	ServiceAnnotationLoadBalancerEnableHealthMonitor = "loadbalancer.openstack.org/enable-health-monitor"
-
+	ServiceAnnotationTlsContainerRef                 = "loadbalancer.openstack.org/default-tls-container-ref"
 	// See https://nip.io
 	defaultProxyHostnameSuffix = "nip.io"
 )
@@ -345,6 +346,7 @@ type serviceConfig struct {
 	enableMonitor        bool
 	flavorID             string
 	availabilityZone     string
+	tlsContainerRef      string
 }
 
 type listenerKey struct {
@@ -1388,8 +1390,14 @@ func (lbaas *LbaasV2) buildPoolCreateOpt(listenerProtocol string, service *corev
 	poolProto := v2pools.Protocol(listenerProtocol)
 	if svcConf.enableProxyProtocol {
 		poolProto = v2pools.ProtocolPROXY
-	} else if svcConf.keepClientIP && poolProto != v2pools.ProtocolHTTP {
-		klog.V(4).Infof("Forcing to use %q protocol for pool because annotation %q is set", v2pools.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor)
+	} else if (svcConf.keepClientIP || svcConf.tlsContainerRef != "") && poolProto != v2pools.ProtocolHTTP {
+		if svcConf.keepClientIP && svcConf.tlsContainerRef != "" {
+			klog.V(4).Infof("Forcing to use %q protocol for pool because annotations %q %q are set", v2pools.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor, ServiceAnnotationTlsContainerRef)
+		} else if svcConf.keepClientIP {
+			klog.V(4).Infof("Forcing to use %q protocol for pool because annotation %q is set", v2pools.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor)
+		} else {
+			klog.V(4).Infof("Forcing to use %q protocol for pool because annotations %q is set", v2pools.ProtocolHTTP, ServiceAnnotationTlsContainerRef)
+		}
 		poolProto = v2pools.ProtocolHTTP
 	}
 
@@ -1449,7 +1457,9 @@ func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, oldListeners []listener
 	}
 
 	proto := toListenersProtocol(port.Protocol)
-	if svcConf.keepClientIP {
+	if svcConf.tlsContainerRef != "" {
+		proto = listeners.ProtocolTerminatedHTTPS
+	} else if svcConf.keepClientIP {
 		proto = listeners.ProtocolHTTP
 	}
 	listener, ok := lbListeners[listenerKey{
@@ -1478,16 +1488,24 @@ func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, oldListeners []listener
 			updateOpts.ConnLimit = &svcConf.connLimit
 			listenerChanged = true
 		}
-		updateOpts.InsertHeaders = &listener.InsertHeaders
+
 		listenerKeepClientIP := listener.InsertHeaders[annotationXForwardedFor] == "true"
 		if svcConf.keepClientIP != listenerKeepClientIP {
+			updateOpts.InsertHeaders = &listener.InsertHeaders
 			if svcConf.keepClientIP {
+				if *updateOpts.InsertHeaders == nil {
+					*updateOpts.InsertHeaders = make(map[string]string)
+				}
 				(*updateOpts.InsertHeaders)[annotationXForwardedFor] = "true"
 			} else {
 				delete(*updateOpts.InsertHeaders, annotationXForwardedFor)
 			}
 			listenerChanged = true
 		}
+		if svcConf.tlsContainerRef != listener.DefaultTlsContainerRef {
+			updateOpts.DefaultTlsContainerRef = &svcConf.tlsContainerRef
+			listenerChanged = true
+		}
 		if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureTimeout) {
 			if svcConf.timeoutClientData != listener.TimeoutClientData {
 				updateOpts.TimeoutClientData = &svcConf.timeoutClientData
@@ -1542,13 +1560,22 @@ func (lbaas *LbaasV2) buildListenerCreateOpt(port corev1.ServicePort, svcConf *s
 	}
 
 	if svcConf.keepClientIP {
-		if listenerCreateOpt.Protocol != listeners.ProtocolHTTP {
-			klog.V(4).Infof("Forcing to use %q protocol for listener because %q annotation is set", listeners.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor)
-			listenerCreateOpt.Protocol = listeners.ProtocolHTTP
-		}
 		listenerCreateOpt.InsertHeaders = map[string]string{annotationXForwardedFor: "true"}
 	}
 
+	if svcConf.tlsContainerRef != "" {
+		listenerCreateOpt.DefaultTlsContainerRef = svcConf.tlsContainerRef
+	}
+
+	// protocol selection
+	if svcConf.tlsContainerRef != "" && listenerCreateOpt.Protocol != listeners.ProtocolTerminatedHTTPS {
+		klog.V(4).Infof("Forcing to use %q protocol for listener because %q annotation is set", listeners.ProtocolTerminatedHTTPS, ServiceAnnotationTlsContainerRef)
+		listenerCreateOpt.Protocol = listeners.ProtocolTerminatedHTTPS
+	} else if svcConf.keepClientIP && listenerCreateOpt.Protocol != listeners.ProtocolHTTP {
+		klog.V(4).Infof("Forcing to use %q protocol for listener because %q annotation is set", listeners.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor)
+		listenerCreateOpt.Protocol = listeners.ProtocolHTTP
+	}
+
 	if len(svcConf.allowedCIDR) > 0 {
 		listenerCreateOpt.AllowedCIDRs = svcConf.allowedCIDR
 	}
@@ -1577,6 +1604,24 @@ func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node
 		svcConf.internal = internal
 	}
 
+	svcConf.tlsContainerRef = getStringFromServiceAnnotation(service, ServiceAnnotationTlsContainerRef, lbaas.opts.TlsContainerRef)
+	if svcConf.tlsContainerRef != "" {
+		if lbaas.secret == nil {
+			return fmt.Errorf("failed to create a TLS Terminated loadbalancer because openstack keymanager client is not "+
+				"initialized and default-tls-container-ref %q is set", svcConf.tlsContainerRef)
+		}
+
+		// check if container exists
+		// tls container ref has format: https://{keymanager_host}/v1/containers/{uuid}
+		slice := strings.Split(svcConf.tlsContainerRef, "/")
+		containerID := slice[len(slice)-1]
+		container, err := containers.Get(lbaas.secret, containerID).Extract()
+		if err != nil {
+			return fmt.Errorf("failed to get tls container %q: %v", svcConf.tlsContainerRef, err)
+		}
+		klog.V(4).Infof("Default TLS container %q found", container.ContainerRef)
+	}
+
 	svcConf.connLimit = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerConnLimit, -1)
 
 	svcConf.lbNetworkID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerNetworkID, lbaas.opts.NetworkID)

pkg/openstack/openstack.go

@@ -84,6 +84,7 @@ func AddExtraFlags(fs *pflag.FlagSet) {
 
 // LoadBalancer is used for creating and maintaining load balancers
 type LoadBalancer struct {
+	secret  *gophercloud.ServiceClient
 	network *gophercloud.ServiceClient
 	compute *gophercloud.ServiceClient
 	lb      *gophercloud.ServiceClient
@@ -113,7 +114,8 @@ type LoadBalancerOpts struct {
 	CascadeDelete         bool                `gcfg:"cascade-delete"` // applicable only if use-octavia is set to True
 	FlavorID              string              `gcfg:"flavor-id"`
 	AvailabilityZone      string              `gcfg:"availability-zone"`
-	EnableIngressHostname bool                `gcfg:"enable-ingress-hostname"` // Used with proxy protocol by adding a dns suffix to the load balancer IP address. Default false.
+	EnableIngressHostname bool                `gcfg:"enable-ingress-hostname"`   // Used with proxy protocol by adding a dns suffix to the load balancer IP address. Default false.
+	TlsContainerRef       string              `gcfg:"default-tls-container-ref"` //  reference to a tls container
 }
 
 // LBClass defines the corresponding floating network, floating subnet or internal subnet ID
@@ -210,6 +212,7 @@ func ReadConfig(config io.Reader) (Config, error) {
 	cfg.LoadBalancer.MonitorMaxRetries = 1
 	cfg.LoadBalancer.CascadeDelete = true
 	cfg.LoadBalancer.EnableIngressHostname = false
+	cfg.LoadBalancer.TlsContainerRef = ""
 
 	err := gcfg.FatalOnly(gcfg.ReadInto(&cfg, config))
 	if err != nil {
@@ -621,6 +624,12 @@ func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
 		return nil, false
 	}
 
+	// keymanager client is optional
+	secret, err := client.NewKeyManagerV1(os.provider, os.epOpts)
+	if err != nil {
+		klog.Warningf("Failed to create an OpenStack Secret client: %v", err)
+	}
+
 	// LBaaS v1 is deprecated in the OpenStack Liberty release.
 	// Currently kubernetes OpenStack cloud provider just support LBaaS v2.
 	lbVersion := os.lbOpts.LBVersion
@@ -631,7 +640,7 @@ func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
 
 	klog.V(1).Info("Claiming to support LoadBalancer")
 
-	return &LbaasV2{LoadBalancer{network, compute, lb, os.lbOpts}}, true
+	return &LbaasV2{LoadBalancer{secret, network, compute, lb, os.lbOpts}}, true
 }
 
 // Zones indicates that we support zones