Add support for generating and using etcd TLS assets.

This change adds rendering options to allow the apiserver to connect
to etcd using TLS. This applies to both the temporary and self-hosted
control plane.

There are also some options (mostly intended for development) to help
generate the etcd (client/server) certificates. Obviously this is only
useful if etcd is not already up.

Self-hosted etcd is not supported at this time.

Author: Diego Pontoriero

cmd/bootkube/render.go

@@ -18,11 +18,12 @@ import (
 )
 
 const (
-	apiOffset            = 1
-	dnsOffset            = 10
-	etcdOffset           = 15
-	defaultServiceBaseIP = "10.3.0.0"
-	defaultEtcdServers   = "http://127.0.0.1:2379"
+	apiOffset             = 1
+	dnsOffset             = 10
+	etcdOffset            = 15
+	defaultServiceBaseIP  = "10.3.0.0"
+	defaultEtcdServers    = "http://127.0.0.1:2379"
+	defaultEtcdTLSServers = "https://127.0.0.1:2379"
 )
 
 var (
@@ -36,17 +37,21 @@ var (
 	}
 
 	renderOpts struct {
-		assetDir          string
-		caCertificatePath string
-		caPrivateKeyPath  string
-		etcdServers       string
-		apiServers        string
-		altNames          string
-		podCIDR           string
-		serviceCIDR       string
-		selfHostKubelet   bool
-		cloudProvider     string
-		selfHostedEtcd    bool
+		assetDir            string
+		caCertificatePath   string
+		caPrivateKeyPath    string
+		etcdCAPath          string
+		etcdCertificatePath string
+		etcdPrivateKeyPath  string
+		etcdServers         string
+		etcdUseTLS          bool
+		apiServers          string
+		altNames            string
+		podCIDR             string
+		serviceCIDR         string
+		selfHostKubelet     bool
+		cloudProvider       string
+		selfHostedEtcd      bool
 	}
 )
 
@@ -55,6 +60,9 @@ func init() {
 	cmdRender.Flags().StringVar(&renderOpts.assetDir, "asset-dir", "", "Output path for rendered assets")
 	cmdRender.Flags().StringVar(&renderOpts.caCertificatePath, "ca-certificate-path", "", "Path to an existing PEM encoded CA. If provided, TLS assets will be generated using this certificate authority.")
 	cmdRender.Flags().StringVar(&renderOpts.caPrivateKeyPath, "ca-private-key-path", "", "Path to an existing Certificate Authority RSA private key. Required if --ca-certificate is set.")
+	cmdRender.Flags().StringVar(&renderOpts.etcdCAPath, "etcd-ca-path", "", "Path to an existing PEM encoded CA that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-certificate-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRender.Flags().StringVar(&renderOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing server certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRender.Flags().StringVar(&renderOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing server private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.")
 	cmdRender.Flags().StringVar(&renderOpts.etcdServers, "etcd-servers", defaultEtcdServers, "List of etcd servers URLs including host:port, comma separated")
 	cmdRender.Flags().StringVar(&renderOpts.apiServers, "api-servers", "https://127.0.0.1:443", "List of API server URLs including host:port, commma seprated")
 	cmdRender.Flags().StringVar(&renderOpts.altNames, "api-server-alt-names", "", "List of SANs to use in api-server certificate. Example: 'IP=127.0.0.1,IP=127.0.0.2,DNS=localhost'. If empty, SANs will be extracted from the --api-servers flag.")
@@ -63,6 +71,7 @@ func init() {
 	cmdRender.Flags().BoolVar(&renderOpts.selfHostKubelet, "experimental-self-hosted-kubelet", false, "(Experimental) Create a self-hosted kubelet daemonset.")
 	cmdRender.Flags().StringVar(&renderOpts.cloudProvider, "cloud-provider", "", "The provider for cloud services.  Empty string for no provider")
 	cmdRender.Flags().BoolVar(&renderOpts.selfHostedEtcd, "experimental-self-hosted-etcd", false, "(Experimental) Create self-hosted etcd assets.")
+	cmdRender.Flags().BoolVar(&renderOpts.etcdUseTLS, "etcd-use-tls", false, "If true, uses TLS for etcd. Implicitly true if --etcd-ca-path,--etcd-certificate-path,--etcd-private-key-path are set. If true but those flags are not set etcd TLS certificates will be generated. Not supported if --experimental-self-hosted-etcd=true.")
 }
 
 func runCmdRender(cmd *cobra.Command, args []string) error {
@@ -86,6 +95,16 @@ func validateRenderOpts(cmd *cobra.Command, args []string) error {
 	if renderOpts.caPrivateKeyPath != "" && renderOpts.caCertificatePath == "" {
 		return errors.New("You must provide the --ca-certificate-path flag when --ca-private-key-path is provided.")
 	}
+	if (renderOpts.etcdCAPath != "" || renderOpts.etcdCertificatePath != "" || renderOpts.etcdPrivateKeyPath != "") && (renderOpts.etcdCAPath == "" || renderOpts.etcdCertificatePath == "" || renderOpts.etcdPrivateKeyPath == "") {
+		return errors.New("You must specify either all or none of --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path")
+	}
+	if renderOpts.etcdCAPath != "" && !renderOpts.etcdUseTLS {
+		bootkube.UserOutput("etcd TLS certificates specified. Overriding --etcd-use-tls=true\n")
+		renderOpts.etcdUseTLS = true
+	}
+	if renderOpts.etcdUseTLS && renderOpts.selfHostedEtcd {
+		return errors.New("Cannot use --etcd-use-tls with --experimental-self-hosted-etcd")
+	}
 	if renderOpts.assetDir == "" {
 		return errors.New("Missing required flag: --asset-dir")
 	}
@@ -163,19 +182,50 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 			bootkube.UserOutput("--experimental-self-hosted-etcd and --service-cidr set. Overriding --etcd-servers setting with %s\n", etcdServers)
 		}
 	} else {
+		if renderOpts.etcdUseTLS && renderOpts.etcdServers == defaultEtcdServers {
+			renderOpts.etcdServers = defaultEtcdTLSServers
+		}
 		etcdServers, err = parseURLs(renderOpts.etcdServers)
 		if err != nil {
 			return nil, err
 		}
 	}
 
+	if renderOpts.etcdUseTLS {
+		for _, url := range etcdServers {
+			if url.Scheme != "https" {
+				return nil, fmt.Errorf("--etcd-use-tls=true but insecure etcd server endpoint specified: %s\n", url)
+			}
+		}
+	}
+
+	var etcdCACert *x509.Certificate
+	if renderOpts.etcdCAPath != "" {
+		etcdCACert, err = parseCertFromDisk(renderOpts.etcdCAPath)
+		if err != nil {
+			return nil, err
+		}
+	}
+	var etcdServerCert *x509.Certificate
+	var etcdServerKey *rsa.PrivateKey
+	if renderOpts.etcdCertificatePath != "" {
+		etcdServerKey, etcdServerCert, err = parseCertAndPrivateKeyFromDisk(renderOpts.etcdCertificatePath, renderOpts.etcdPrivateKeyPath)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// TODO: Find better option than asking users to make manual changes
 	if serviceNet.IP.String() != defaultServiceBaseIP {
 		fmt.Printf("You have selected a non-default service CIDR %s - be sure your kubelet service file uses --cluster-dns=%s\n", serviceNet.String(), dnsServiceIP.String())
 	}
 
 	return &asset.Config{
+		EtcdCACert:      etcdCACert,
+		EtcdServerCert:  etcdServerCert,
+		EtcdServerKey:   etcdServerKey,
 		EtcdServers:     etcdServers,
+		EtcdUseTLS:      renderOpts.etcdUseTLS,
 		CACert:          caCert,
 		CAPrivKey:       caPrivKey,
 		APIServers:      apiServers,
@@ -202,15 +252,23 @@ func parseCertAndPrivateKeyFromDisk(caCertPath, privKeyPath string) (*rsa.Privat
 		return nil, nil, fmt.Errorf("unable to parse CA private key: %v", err)
 	}
 	// Parse CA Cert.
+	cert, err := parseCertFromDisk(caCertPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	return key, cert, nil
+}
+
+func parseCertFromDisk(caCertPath string) (*x509.Certificate, error) {
 	capem, err := ioutil.ReadFile(caCertPath)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error reading ca cert file at %s: %v", caCertPath, err)
+		return nil, fmt.Errorf("error reading ca cert file at %s: %v", caCertPath, err)
 	}
 	cert, err := tlsutil.ParsePEMEncodedCACert(capem)
 	if err != nil {
-		return nil, nil, fmt.Errorf("unable to parse CA Cert: %v", err)
+		return nil, fmt.Errorf("unable to parse CA Cert: %v", err)
 	}
-	return key, cert, nil
+	return cert, nil
 }
 
 func parseURLs(s string) ([]*url.URL, error) {

hack/quickstart/init-master.sh

@@ -16,16 +16,25 @@ function usage() {
 
 function configure_etcd() {
     [ -f "/etc/systemd/system/etcd-member.service.d/10-etcd-member.conf" ] || {
+        mkdir -p /etc/etcd/tls
+        cp /home/core/assets/tls/etcd* /etc/etcd/tls
+        chown -R etcd:etcd /etc/etcd
+        chmod -R u=rX,g=,o= /etc/etcd
         mkdir -p /etc/systemd/system/etcd-member.service.d
         cat << EOF > /etc/systemd/system/etcd-member.service.d/10-etcd-member.conf
 [Service]
 Environment="ETCD_IMAGE_TAG=v3.1.0"
 Environment="ETCD_NAME=controller"
 Environment="ETCD_INITIAL_CLUSTER=controller=http://${COREOS_PRIVATE_IPV4}:2380"
 Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://${COREOS_PRIVATE_IPV4}:2380"
-Environment="ETCD_ADVERTISE_CLIENT_URLS=http://${COREOS_PRIVATE_IPV4}:2379"
-Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379"
+Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${COREOS_PRIVATE_IPV4}:2379"
+Environment="ETCD_SSL_DIR=/etc/etcd/tls"
+Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
 Environment="ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380"
+Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
+Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-server.crt"
+Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-server.key"
+Environment="ETCD_CLIENT_CERT_AUTH=true"
 EOF
     }
 }
@@ -35,26 +44,29 @@ function init_master_node() {
     systemctl daemon-reload
     systemctl stop update-engine; systemctl mask update-engine
 
-    etcd_render_flags=""
-
-    # Start etcd.
     if [ "$SELF_HOST_ETCD" = true ] ; then
         echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
         etcd_render_flags="--experimental-self-hosted-etcd"
     else
-        configure_etcd
-        systemctl enable etcd-member; sudo systemctl start etcd-member
+        etcd_render_flags="--etcd-use-tls --etcd-servers=https://${COREOS_PRIVATE_IPV4}:2379"
     fi
 
     # Render cluster assets
-    /home/core/bootkube render --asset-dir=/home/core/assets --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443 ${etcd_render_flags}
+    /home/core/bootkube render --asset-dir=/home/core/assets ${etcd_render_flags} \
+      --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443
 
     # Move the local kubeconfig into expected location
     chown -R core:core /home/core/assets
     mkdir -p /etc/kubernetes
     cp /home/core/assets/auth/kubeconfig /etc/kubernetes/
     cp /home/core/assets/tls/ca.crt /etc/kubernetes/ca.crt
 
+    # Start etcd.
+    if [ "$SELF_HOST_ETCD" = false ] ; then
+        configure_etcd
+        systemctl enable etcd-member; sudo systemctl start etcd-member
+    fi
+
     # Start the kubelet
     systemctl enable kubelet; sudo systemctl start kubelet
 

pkg/asset/asset.go

@@ -19,6 +19,9 @@ const (
 	AssetPathCACert                      = "tls/ca.crt"
 	AssetPathAPIServerKey                = "tls/apiserver.key"
 	AssetPathAPIServerCert               = "tls/apiserver.crt"
+	AssetPathEtcdCA                      = "tls/etcd-ca.crt"
+	AssetPathEtcdServerCert              = "tls/etcd-server.crt"
+	AssetPathEtcdServerKey               = "tls/etcd-server.key"
 	AssetPathServiceAccountPrivKey       = "tls/service-account.key"
 	AssetPathServiceAccountPubKey        = "tls/service-account.pub"
 	AssetPathKubeletKey                  = "tls/kubelet.key"
@@ -55,7 +58,11 @@ const (
 // AssetConfig holds all configuration needed when generating
 // the default set of assets.
 type Config struct {
+	EtcdCACert          *x509.Certificate
+	EtcdServerCert      *x509.Certificate
+	EtcdServerKey       *rsa.PrivateKey
 	EtcdServers         []*url.URL
+	EtcdUseTLS          bool
 	APIServers          []*url.URL
 	CACert              *x509.Certificate
 	CAPrivKey           *rsa.PrivateKey
@@ -83,13 +90,31 @@ func NewDefaultAssets(conf Config) (Assets, error) {
 	// Add kube-apiserver service IP
 	conf.AltNames.IPs = append(conf.AltNames.IPs, conf.APIServiceIP)
 
+	// Create a CA if none was provided.
+	if conf.CACert == nil {
+		var err error
+		conf.CAPrivKey, conf.CACert, err = newCACert()
+		if err != nil {
+			return Assets{}, err
+		}
+	}
+
 	// TLS assets
 	tlsAssets, err := newTLSAssets(conf.CACert, conf.CAPrivKey, *conf.AltNames)
 	if err != nil {
 		return Assets{}, err
 	}
 	as = append(as, tlsAssets...)
 
+	// etcd TLS assets.
+	if conf.EtcdUseTLS {
+		etcdTLSAssets, err := newEtcdTLSAssets(conf.EtcdCACert, conf.EtcdServerCert, conf.EtcdServerKey, conf.CACert, conf.CAPrivKey, conf.EtcdServers)
+		if err != nil {
+			return Assets{}, err
+		}
+		as = append(as, etcdTLSAssets...)
+	}
+
 	// K8S kubeconfig
 	kubeConfig, err := newKubeConfigAsset(as, conf)
 	if err != nil {
@@ -98,7 +123,7 @@ func NewDefaultAssets(conf Config) (Assets, error) {
 	as = append(as, kubeConfig)
 
 	// K8S APIServer secret
-	apiSecret, err := newAPIServerSecretAsset(as)
+	apiSecret, err := newAPIServerSecretAsset(as, conf.EtcdUseTLS)
 	if err != nil {
 		return Assets{}, err
 	}

pkg/asset/internal/templates.go

@@ -171,6 +171,11 @@ spec:
         - --bind-address=0.0.0.0
         - --client-ca-file=/etc/kubernetes/secrets/ca.crt
         - --cloud-provider={{ .CloudProvider  }}
+{{- if .EtcdUseTLS }}
+        - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt
+        - --etcd-certfile=/etc/kubernetes/secrets/etcd-server.crt
+        - --etcd-keyfile=/etc/kubernetes/secrets/etcd-server.key
+{{- end }}
         - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}
         - --insecure-port=8080
         - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
@@ -231,6 +236,11 @@ spec:
     - --authorization-mode=RBAC
     - --bind-address=0.0.0.0
     - --client-ca-file=/etc/kubernetes/secrets/ca.crt
+{{- if .EtcdUseTLS }}
+    - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt
+    - --etcd-certfile=/etc/kubernetes/secrets/etcd-server.crt
+    - --etcd-keyfile=/etc/kubernetes/secrets/etcd-server.key
+{{- end }}
     - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}{{ if .SelfHostedEtcd }},http://127.0.0.1:12379{{end}}
     - --insecure-port=8080
     - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt

pkg/asset/k8s.go

@@ -87,13 +87,20 @@ func newKubeConfigAsset(assets Assets, conf Config) (Asset, error) {
 	})
 }
 
-func newAPIServerSecretAsset(assets Assets) (Asset, error) {
+func newAPIServerSecretAsset(assets Assets, etcdUseTLS bool) (Asset, error) {
 	secretAssets := []string{
 		AssetPathAPIServerKey,
 		AssetPathAPIServerCert,
 		AssetPathServiceAccountPubKey,
 		AssetPathCACert,
 	}
+	if etcdUseTLS {
+		secretAssets = append(secretAssets, []string{
+			AssetPathEtcdCA,
+			AssetPathEtcdServerCert,
+			AssetPathEtcdServerKey,
+		}...)
+	}
 
 	secretYAML, err := secretFromAssets(secretAPIServerName, secretNamespace, secretAssets, assets)
 	if err != nil {