Merge pull request #22 from appscode-cloud/lka2

 Add test for LoadBalancer Service with a single TLS port

Author: asauber

.travis.yml

@@ -10,7 +10,7 @@ env:
 - GO111MODULE=on
 
 go:
-  - 1.11.x
+  - 1.12.x
   - tip
 
 cache:

Makefile

@@ -2,11 +2,31 @@ IMG ?= linode/linode-cloud-controller-manager:latest
 
 export GO111MODULE=on
 
-all: build
+$(GOPATH)/bin/goimports:
+	GO111MODULE=off go get golang.org/x/tools/cmd/goimports
+
+$(GOPATH)/bin/ginkgo:
+	GO111MODULE=off go get -u github.com/onsi/ginkgo/ginkgo
 
-build: fmt
+vet:
+	go vet -composites=false ./...
+
+fmt: vet $(GOPATH)/bin/goimports
+	# goimports runs a gofmt
+	goimports -w *.go cloud
+
+build: test fmt
 	go build -o dist/linode-cloud-controller-manager github.com/linode/linode-cloud-controller-manager
 
+test: $(GOPATH)/bin/ginkgo
+	ginkgo -r --v --progress --trace --cover --skipPackage=test -- --v=3
+
+docker-build: build
+	docker build . -t ${IMG}
+
+docker-push:
+	docker push ${IMG}
+
 run: build
 	dist/linode-cloud-controller-manager \
 		--logtostderr=true \
@@ -22,26 +42,5 @@ run-debug: build
 		--kubeconfig=${KUBECONFIG} \
 		--linodego-debug
 
-$(GOPATH)/bin/goimports:
-	GO111MODULE=off go get golang.org/x/tools/cmd/goimports
-
-vet:
-	go vet -composites=false ./...
-
-imports: $(GOPATH)/bin/goimports
-	goimports -w *.go cloud
-
-fmt: vet imports
-	gofmt -s -w *.go cloud
-
-$(GOPATH)/bin/ginkgo:
-	GO111MODULE=off go get -u github.com/onsi/ginkgo/ginkgo
-
-test: $(GOPATH)/bin/ginkgo
-	ginkgo -r --v --progress --trace --cover --skipPackage=test -- --v=3
-
-docker-build:
-	docker build . -t ${IMG}
+all: build
 
-docker-push:
-	docker push ${IMG}

README.md

@@ -46,9 +46,7 @@ Annotation (Suffix) | Values | Default | Description
 `protocol` | `tcp`, `http`, `https` | `tcp` | This annotation is used to specify the default protocol for Linode NodeBalancer. For ports specified in the `linode-loadbalancer-tls-ports` annotation, this protocol is overwritten to `https`
 `stickiness` | `none`, `table`, `http_cookie` | `none` | Controls how session stickiness is handled on this port.
 `algorithm` | `round_robin`, `least_connections` | `round_robin` | Specifies which algorithm the Linode NodeBalancer should use
-`tls-ports` | int (e.g. `443,6443,7443`) | | This annotation specifies the ports the NodeBalancer should use for `https`
-`ssl-cert` | string | | The Base64 Encoded SSL certificates for this service. The full certificate chain may be provided. (`base64 -w0 ssl.crt`)
-`ssl-key` | string | | The Base64 Encoded private key corresponding to this port's certificate.  The key can not be passphrase protected. (`base64 -w0 ssl.key`)
+`tls` | json array (e.g. `[ { "tls-secret-name": "prod-app-tls", "port": 443}, {"tls-secret-name": "dev-app-tls", "port": 8443} ]`) | | Specifies TLS ports with their corresponding secrets, the secret type should be `kubernetes.io/tls`
 `check-type` | `none`, `connection`, `http`, `http_body` | | The type of health check to perform against back-ends to ensure they are serving requests
 `check-path` | string | | The URL path to check on each back-end during health checks
 `check-body` | string | | Text which must be present in the response body to pass the NodeBalancer health check

cloud/linode/loadbalancers.go

@@ -2,12 +2,15 @@ package linode
 
 import (
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+
 	"github.com/linode/linodego"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
@@ -20,25 +23,15 @@ const (
 	// is overwritten to https. Options are tcp, http and https. Defaults to tcp.
 	annLinodeProtocol = "service.beta.kubernetes.io/linode-loadbalancer-protocol"
 
-	// annLinodeTLSPorts is the annotation used to specify which ports of the loadbalancer
-	// should use the https protocol. This is a comma separated list of ports
-	// (e.g. 443,6443,7443).
-	annLinodeTLSPorts = "service.beta.kubernetes.io/linode-loadbalancer-tls-ports"
-
 	annLinodeCheckPath       = "service.beta.kubernetes.io/linode-loadbalancer-check-path"
 	annLinodeCheckBody       = "service.beta.kubernetes.io/linode-loadbalancer-check-body"
 	annLinodeHealthCheckType = "service.beta.kubernetes.io/linode-loadbalancer-check-type"
 
-	// annLinodeCertificateID is the annotation specifying the certificate ID
-	// used for https protocol. This annotation is required if annLinodeTLSPorts
-	// is passed.
-	annLinodeSSLCertificate = "service.beta.kubernetes.io/linode-loadbalancer-ssl-cert"
-	annLinodeSSLKey         = "service.beta.kubernetes.io/linode-loadbalancer-ssl-key"
-
 	annLinodeHealthCheckInterval = "service.beta.kubernetes.io/linode-loadbalancer-check-interval"
 	annLinodeHealthCheckTimeout  = "service.beta.kubernetes.io/linode-loadbalancer-check-timeout"
 	annLinodeHealthCheckAttempts = "service.beta.kubernetes.io/linode-loadbalancer-check-attempts"
 	annLinodeHealthCheckPassive  = "service.beta.kubernetes.io/linode-loadbalancer-check-passive"
+	annLinodeLoadBalancerTLS     = "service.beta.kubernetes.io/linode-loadbalancer-tls"
 
 	annLinodeSessionPersistence = "service.beta.kubernetes.io/linode-loadbalancer-stickiness"
 
@@ -58,6 +51,13 @@ var lbNotFound = errors.New("loadbalancer not found")
 type loadbalancers struct {
 	client *linodego.Client
 	zone   string
+
+	kubeClient kubernetes.Interface
+}
+
+type tlsAnnotation struct {
+	TlsSecretName string `json:"tls-secret-name"`
+	Port          int    `json:"port"`
 }
 
 // newLoadbalancers returns a cloudprovider.LoadBalancer whose concrete type is a *loadbalancer.
@@ -346,27 +346,34 @@ func (l *loadbalancers) buildNodeBalancerConfig(service *v1.Service, port int) (
 	config.CheckPassive = checkPassive
 
 	if protocol == linodego.ProtocolHTTPS {
-		isTLS, err := isTLSPort(service, port)
-		if err != nil {
+		if err = l.processHTTPS(service, &config, port); err != nil {
 			return config, err
 		}
-		if isTLS {
-			cert, key := getSSLCertInfo(service)
-			if cert == "" && key == "" {
-				return config, fmt.Errorf("must set %v and %v annotation for https protocol", annLinodeSSLCertificate, annLinodeSSLKey)
-			}
-			if cert != "" {
-				config.SSLCert = cert
-			}
-			if key != "" {
-				config.SSLKey = key
-			}
-		}
 	}
 
 	return config, nil
 }
 
+func (l *loadbalancers) processHTTPS(service *v1.Service, nbConfig *linodego.NodeBalancerConfig, port int) error {
+	if err := l.retrieveKubeClient(); err != nil {
+		return err
+	}
+	tlsAnnotations, err := getTLSAnnotations(service)
+	if err != nil {
+		return err
+	}
+
+	tlsPorts := getTLSPorts(tlsAnnotations)
+	isTLS := isTLSPort(tlsPorts, port)
+	if isTLS {
+		nbConfig.SSLCert, nbConfig.SSLKey, err = getTLSCertInfo(l.kubeClient, tlsAnnotations, service.Namespace, port)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // buildLoadBalancerRequest returns a linodego.NodeBalancer
 // requests for service across nodes.
 func (l *loadbalancers) buildLoadBalancerRequest(ctx context.Context, service *v1.Service, nodes []*v1.Node) (*linodego.NodeBalancer, error) {
@@ -402,6 +409,22 @@ func (l *loadbalancers) buildNodeBalancerNodeCreateOptions(node *v1.Node, nodePo
 	}
 }
 
+func (l *loadbalancers) retrieveKubeClient() error {
+	if l.kubeClient == nil {
+		kubeConfig, err := rest.InClusterConfig()
+		if err != nil {
+			return err
+		}
+
+		l.kubeClient, err = kubernetes.NewForConfig(kubeConfig)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // getProtocol returns the desired protocol of service.
 func getProtocol(service *v1.Service) (linodego.ConfigProtocol, error) {
 	protocol, ok := service.Annotations[annLinodeProtocol]
@@ -427,39 +450,36 @@ func getHealthCheckType(service *v1.Service) (linodego.ConfigCheck, error) {
 	return linodego.ConfigCheck(hType), nil
 }
 
-func isTLSPort(service *v1.Service, port int) (bool, error) {
-	tlsPortsSlice, err := getTLSPorts(service)
-	if err != nil {
-		return false, err
-	}
+func isTLSPort(tlsPortsSlice []int, port int) bool {
 	for _, tlsPort := range tlsPortsSlice {
 		if port == tlsPort {
-			return true, nil
+			return true
 		}
 	}
-	return false, nil
+	return false
 }
 
 // getTLSPorts returns the ports of service that are set to use TLS.
-func getTLSPorts(service *v1.Service) ([]int, error) {
-	tlsPorts, ok := service.Annotations[annLinodeTLSPorts]
-	if !ok {
-		return nil, nil
+func getTLSPorts(tlsAnnotations []*tlsAnnotation) []int {
+	tlsPortsInt := make([]int, 0)
+	for _, tlsAnnotation := range tlsAnnotations {
+		tlsPortsInt = append(tlsPortsInt, tlsAnnotation.Port)
 	}
 
-	tlsPortsSlice := strings.Split(tlsPorts, ",")
-
-	tlsPortsInt := make([]int, len(tlsPortsSlice))
-	for i, tlsPort := range tlsPortsSlice {
-		port, err := strconv.Atoi(tlsPort)
-		if err != nil {
-			return nil, err
-		}
+	return tlsPortsInt
+}
 
-		tlsPortsInt[i] = port
+func getTLSAnnotations(service *v1.Service) ([]*tlsAnnotation, error) {
+	annotationJSON, ok := service.Annotations[annLinodeLoadBalancerTLS]
+	if !ok {
+		return nil, fmt.Errorf("annotation %v must be specified", annLinodeLoadBalancerTLS)
 	}
-
-	return tlsPortsInt, nil
+	tlsAnnotations := make([]*tlsAnnotation, 0)
+	err := json.Unmarshal([]byte(annotationJSON), &tlsAnnotations)
+	if err != nil {
+		return nil, err
+	}
+	return tlsAnnotations, nil
 }
 
 func getNodeInternalIp(node *v1.Node) string {
@@ -503,18 +523,26 @@ func getStickiness(service *v1.Service) linodego.ConfigStickiness {
 	}
 }
 
-func getSSLCertInfo(service *v1.Service) (string, string) {
-	cert := service.Annotations[annLinodeSSLCertificate]
-	if cert != "" {
-		cb, _ := base64.StdEncoding.DecodeString(cert)
-		cert = strings.TrimSpace(string(cb))
-	}
-	key := service.Annotations[annLinodeSSLKey]
-	if key != "" {
-		kb, _ := base64.StdEncoding.DecodeString(key)
-		key = strings.TrimSpace(string(kb))
+func getTLSCertInfo(kubeClient kubernetes.Interface, tlsAnnotations []*tlsAnnotation, namespace string, port int) (string, string, error) {
+	for _, tlsAnnotation := range tlsAnnotations {
+		if tlsAnnotation.Port == port {
+			secret, err := kubeClient.CoreV1().Secrets(namespace).Get(tlsAnnotation.TlsSecretName, metav1.GetOptions{})
+			if err != nil {
+				return "", "", err
+			}
+
+			cert := string(secret.Data[v1.TLSCertKey])
+			cert = strings.TrimSpace(cert)
+
+			key := string(secret.Data[v1.TLSPrivateKeyKey])
+
+			key = strings.TrimSpace(key)
+
+			return cert, key, nil
+		}
 	}
-	return cert, key
+
+	return "", "", fmt.Errorf("cert & key for port %v is not specified in annotation %v", port, annLinodeLoadBalancerTLS)
 }
 
 func getConnectionThrottle(service *v1.Service) int {