Support assigning SSL certs to BackendSets (#243)

Author: MadalinaPatrichi

docs/tutorial-ssl-backendset.md

@@ -0,0 +1,105 @@
+# Tutorial
+
+This example will show you how to use the CCM to create a load balancer with SSL
+termination for BackendSets.
+
+### Load balancer with SSL termination for BackendSets example
+
+When you create a service with --type=LoadBalancer a OCI load balancer will be
+created.
+
+The example below will create an NGINX deployment and expose it via a load
+balancer serving http on port 80, and https on 443. Note that the service
+**type** is set to **LoadBalancer**.
+
+```yaml
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: nginx-service
+  annotations:
+    service.beta.kubernetes.io/oci-load-balancer-ssl-ports: "443"
+    service.beta.kubernetes.io/oci-load-balancer-tls-secret: ssl-certificate-secret
+    service.beta.kubernetes.io/oci-load-balancer-tls-backendset-secret: ssl-certificate-secret
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+  - name: https
+    port: 443
+    targetPort: 80
+```
+
+First, the required Secret needs to be created in Kubernetes. For the purposes
+of this example, we will create a self-signed certificate. However, in
+production you would most likely use a public certificate signed by a
+certificate authority.
+
+Below is an example of a secret configuration file required to be uploaded as a Kubernetes
+generic Secret. The CA certificate, the public certificate and the private key need to be base64 encoded:
+
+***Note: Certificates for BackendSets require a CA certificate to be provided.***
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ssl-certificate-secret
+type: Opaque
+data:
+  ca.crt: LS0tLS1CRUdJTiBDRV(...)
+  tls.crt: LS0tLS1CRUdJTi(...)
+  tls.key: LS0tLS1CRUdJTi(...)
+```
+
+```
+kubectl create -f ssl-certificate-secret.yaml
+```
+
+Create the service:
+
+```
+$ kubectl create -f manifests/demo/nginx-demo-svc-ssl.yaml
+```
+
+Watch the service and await a public IP address. This will be the load balancer
+IP which you can use to connect to your service.
+
+```
+$ kubectl get svc --watch
+NAME            CLUSTER-IP     EXTERNAL-IP      PORT(S)        AGE
+nginx-service   10.96.97.137   129.213.12.174   80:30274/TCP   5m
+```
+
+You can now access your service via the provisioned load balancer using either
+http or https:
+
+```
+curl http://129.213.12.174
+curl --insecure https://129.213.12.174
+```
+
+Note: The `--insecure` flag above is only required due to our use of self-signed
+certificates in this example.

pkg/oci/client/load_balancer.go

@@ -35,7 +35,7 @@ type LoadBalancerInterface interface {
 	DeleteLoadBalancer(ctx context.Context, id string) (string, error)
 
 	GetCertificateByName(ctx context.Context, lbID, name string) (*loadbalancer.Certificate, error)
-	CreateCertificate(ctx context.Context, lbID, certificate, key string) (string, error)
+	CreateCertificate(ctx context.Context, lbID string, cert loadbalancer.CertificateDetails) (string, error)
 
 	CreateBackendSet(ctx context.Context, lbID, name string, details loadbalancer.BackendSetDetails) (string, error)
 	UpdateBackendSet(ctx context.Context, lbID, name string, details loadbalancer.BackendSetDetails) (string, error)
@@ -150,18 +150,19 @@ func (c *client) GetCertificateByName(ctx context.Context, lbID, name string) (*
 	return nil, errors.WithStack(errNotFound)
 }
 
-func (c *client) CreateCertificate(ctx context.Context, lbID, certificate, key string) (string, error) {
+func (c *client) CreateCertificate(ctx context.Context, lbID string, cert loadbalancer.CertificateDetails) (string, error) {
 	if !c.rateLimiter.Writer.TryAccept() {
 		return "", RateLimitError(true, "CreateCertificate")
 	}
 
-	// TODO(apryde): We currently don't have a mechanism for supplying
-	// CreateCertificateDetails.CaCertificate.
 	resp, err := c.loadbalancer.CreateCertificate(ctx, loadbalancer.CreateCertificateRequest{
 		LoadBalancerId: &lbID,
 		CreateCertificateDetails: loadbalancer.CreateCertificateDetails{
-			PublicCertificate: &certificate,
-			PrivateKey:        &key,
+			CertificateName:   cert.CertificateName,
+			CaCertificate:     cert.CaCertificate,
+			PublicCertificate: cert.PublicCertificate,
+			PrivateKey:        cert.PrivateKey,
+			Passphrase:        cert.Passphrase,
 		},
 	})
 	incRequestCounter(err, createVerb, certificateResource)

pkg/oci/load_balancer.go

@@ -63,6 +63,12 @@ const (
 	// See: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls
 	ServiceAnnotationLoadBalancerTLSSecret = "service.beta.kubernetes.io/oci-load-balancer-tls-secret"
 
+	// ServiceAnnotationLoadBalancerTLSBackendSetSecret is a Service annotation for
+	// specifying the generic secret to install on the load balancer listeners which
+	// have SSL enabled.
+	// See: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls
+	ServiceAnnotationLoadBalancerTLSBackendSetSecret = "service.beta.kubernetes.io/oci-load-balancer-tls-backendset-secret"
+
 	// ServiceAnnotationLoadBalancerConnectionIdleTimeout is the annotation used
 	// on the service to specify the idle connection timeout.
 	ServiceAnnotationLoadBalancerConnectionIdleTimeout = "service.beta.kubernetes.io/oci-load-balancer-connection-idle-timeout"
@@ -89,9 +95,10 @@ const (
 	// Fallback value if annotation on service is not set
 	lbDefaultShape = "100Mbps"
 
-	lbNodesHealthCheckPath  = "/healthz"
-	lbNodesHealthCheckPort  = k8sports.ProxyHealthzPort
-	lbNodesHealthCheckProto = "HTTP"
+	lbNodesHealthCheckPath      = "/healthz"
+	lbNodesHealthCheckPort      = k8sports.ProxyHealthzPort
+	lbNodesHealthCheckProtoHTTP = "HTTP"
+	lbNodesHealthCheckProtoTCP  = "TCP"
 )
 
 // GetLoadBalancer returns whether the specified load balancer exists, and if
@@ -189,62 +196,48 @@ func getSubnetsForNodes(ctx context.Context, nodes []*v1.Node, client client.Int
 
 // readSSLSecret returns the certificate and private key from a Kubernetes TLS
 // private key Secret.
-func (cp *CloudProvider) readSSLSecret(svc *v1.Service) (string, string, error) {
-	secretString, ok := svc.Annotations[ServiceAnnotationLoadBalancerTLSSecret]
-	if !ok {
-		return "", "", errors.Errorf("no %q annotation found", ServiceAnnotationLoadBalancerTLSSecret)
-	}
-
-	ns, name := parseSecretString(secretString)
-	if ns == "" {
-		ns = svc.Namespace
-	}
+func (cp *CloudProvider) readSSLSecret(ns, name string) (*certificateData, error) {
 	secret, err := cp.kubeclient.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{})
 	if err != nil {
-		return "", "", err
+		return nil, err
 	}
-
-	var cert, key []byte
-	if cert, ok = secret.Data[sslCertificateFileName]; !ok {
-		return "", "", errors.Errorf("%s not found in secret %s/%s", sslCertificateFileName, ns, name)
+	var ok bool
+	var cacert, cert, key, pass []byte
+	cacert = secret.Data[SSLCAFileName]
+	if cert, ok = secret.Data[SSLCertificateFileName]; !ok {
+		return nil, errors.Errorf("%s not found in secret %s/%s", SSLCertificateFileName, ns, name)
 	}
-	if key, ok = secret.Data[sslPrivateKeyFileName]; !ok {
-		return "", "", errors.Errorf("%s not found in secret %s/%s", sslPrivateKeyFileName, ns, name)
+	if key, ok = secret.Data[SSLPrivateKeyFileName]; !ok {
+		return nil, errors.Errorf("%s not found in secret %s/%s", SSLPrivateKeyFileName, ns, name)
 	}
-
-	return string(cert), string(key), nil
+	pass = secret.Data[SSLPassphrase]
+	return &certificateData{CACert: cacert, PublicCert: cert, PrivateKey: key, Passphrase: pass}, nil
 }
 
 // ensureSSLCertificate creates a OCI SSL certificate to the given load
 // balancer, if it doesn't already exist.
-func (cp *CloudProvider) ensureSSLCertificate(ctx context.Context, lb *loadbalancer.LoadBalancer, spec *LBSpec) error {
-	name := spec.SSLConfig.Name
-	logger := cp.logger.With("loadBalancerID", *lb.Id, "certificateName", name)
-	_, err := cp.client.LoadBalancer().GetCertificateByName(ctx, *lb.Id, name)
-	if err == nil {
-		logger.Debug("Certificate already exists on load balancer. Nothing to do.")
-		return nil
-	}
-	if !client.IsNotFound(err) {
-		return err
-	}
-
-	// Although we iterate here only one certificate is supported at the moment.
+func (cp *CloudProvider) ensureSSLCertificates(ctx context.Context, lb *loadbalancer.LoadBalancer, spec *LBSpec) error {
+	logger := cp.logger.With("loadBalancerID", *lb.Id)
+	// Get all required certificates
 	certs, err := spec.Certificates()
 	if err != nil {
 		return err
 	}
+
 	for _, cert := range certs {
-		wrID, err := cp.client.LoadBalancer().CreateCertificate(ctx, *lb.Id, *cert.PublicCertificate, *cert.PrivateKey)
-		if err != nil {
-			return err
-		}
-		_, err = cp.client.LoadBalancer().AwaitWorkRequest(ctx, wrID)
-		if err != nil {
-			return err
-		}
+		if _, ok := lb.Certificates[*cert.CertificateName]; !ok {
+			logger = cp.logger.With("certificateName", *cert.CertificateName)
+			wrID, err := cp.client.LoadBalancer().CreateCertificate(ctx, *lb.Id, cert)
+			if err != nil {
+				return err
+			}
+			_, err = cp.client.LoadBalancer().AwaitWorkRequest(ctx, wrID)
+			if err != nil {
+				return err
+			}
 
-		logger.Info("Certificate created")
+			logger.Info("Certificate created")
+		}
 	}
 	return nil
 }
@@ -323,16 +316,18 @@ func (cp *CloudProvider) EnsureLoadBalancer(ctx context.Context, clusterName str
 	}
 	exists := !client.IsNotFound(err)
 
-	var ssl *SSLConfig
+	var sslConfig *SSLConfig
 	if requiresCertificate(service) {
 		ports, err := getSSLEnabledPorts(service)
 		if err != nil {
 			return nil, err
 		}
-		ssl = NewSSLConfig(lbName, ports, cp)
+		secretListenerString := service.Annotations[ServiceAnnotationLoadBalancerTLSSecret]
+		secretBackendSetString := service.Annotations[ServiceAnnotationLoadBalancerTLSBackendSetSecret]
+		sslConfig = NewSSLConfig(secretListenerString, secretBackendSetString, ports, cp)
 	}
 	subnets := []string{cp.config.LoadBalancer.Subnet1, cp.config.LoadBalancer.Subnet2}
-	spec, err := NewLBSpec(service, nodes, subnets, ssl, cp.securityListManagerFactory)
+	spec, err := NewLBSpec(service, nodes, subnets, sslConfig, cp.securityListManagerFactory)
 	if err != nil {
 		logger.With(zap.Error(err)).Error("Failed to derive LBSpec")
 		return nil, err
@@ -351,8 +346,8 @@ func (cp *CloudProvider) EnsureLoadBalancer(ctx context.Context, clusterName str
 
 	// If the load balancer needs an SSL cert ensure it is present.
 	if requiresCertificate(service) {
-		if err := cp.ensureSSLCertificate(ctx, lb, spec); err != nil {
-			return nil, errors.Wrap(err, "ensuring ssl certificate")
+		if err := cp.ensureSSLCertificates(ctx, lb, spec); err != nil {
+			return nil, errors.Wrap(err, "ensuring ssl certificates")
 		}
 	}