Always secure etcd using TLS, except when self-hosting.

Author: Diego Pontoriero

cmd/bootkube/render.go

@@ -18,12 +18,12 @@ import (
 )
 
 const (
-	apiOffset             = 1
-	dnsOffset             = 10
-	etcdOffset            = 15
-	defaultServiceBaseIP  = "10.3.0.0"
-	defaultEtcdServers    = "http://127.0.0.1:2379"
-	defaultEtcdTLSServers = "https://127.0.0.1:2379"
+	apiOffset                    = 1
+	dnsOffset                    = 10
+	etcdOffset                   = 15
+	defaultServiceBaseIP         = "10.3.0.0"
+	defaultEtcdServers           = "https://127.0.0.1:2379"
+	defaultSelfHostedEtcdServers = "http://127.0.0.1:2379"
 )
 
 var (
@@ -44,7 +44,6 @@ var (
 		etcdCertificatePath string
 		etcdPrivateKeyPath  string
 		etcdServers         string
-		etcdUseTLS          bool
 		apiServers          string
 		altNames            string
 		podCIDR             string
@@ -61,8 +60,8 @@ func init() {
 	cmdRender.Flags().StringVar(&renderOpts.caCertificatePath, "ca-certificate-path", "", "Path to an existing PEM encoded CA. If provided, TLS assets will be generated using this certificate authority.")
 	cmdRender.Flags().StringVar(&renderOpts.caPrivateKeyPath, "ca-private-key-path", "", "Path to an existing Certificate Authority RSA private key. Required if --ca-certificate is set.")
 	cmdRender.Flags().StringVar(&renderOpts.etcdCAPath, "etcd-ca-path", "", "Path to an existing PEM encoded CA that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-certificate-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
-	cmdRender.Flags().StringVar(&renderOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing server certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
-	cmdRender.Flags().StringVar(&renderOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing server private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRender.Flags().StringVar(&renderOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRender.Flags().StringVar(&renderOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.")
 	cmdRender.Flags().StringVar(&renderOpts.etcdServers, "etcd-servers", defaultEtcdServers, "List of etcd servers URLs including host:port, comma separated")
 	cmdRender.Flags().StringVar(&renderOpts.apiServers, "api-servers", "https://127.0.0.1:443", "List of API server URLs including host:port, commma seprated")
 	cmdRender.Flags().StringVar(&renderOpts.altNames, "api-server-alt-names", "", "List of SANs to use in api-server certificate. Example: 'IP=127.0.0.1,IP=127.0.0.2,DNS=localhost'. If empty, SANs will be extracted from the --api-servers flag.")
@@ -71,7 +70,6 @@ func init() {
 	cmdRender.Flags().BoolVar(&renderOpts.selfHostKubelet, "experimental-self-hosted-kubelet", false, "(Experimental) Create a self-hosted kubelet daemonset.")
 	cmdRender.Flags().StringVar(&renderOpts.cloudProvider, "cloud-provider", "", "The provider for cloud services.  Empty string for no provider")
 	cmdRender.Flags().BoolVar(&renderOpts.selfHostedEtcd, "experimental-self-hosted-etcd", false, "(Experimental) Create self-hosted etcd assets.")
-	cmdRender.Flags().BoolVar(&renderOpts.etcdUseTLS, "etcd-use-tls", false, "If true, uses TLS for etcd. Implicitly true if --etcd-ca-path,--etcd-certificate-path,--etcd-private-key-path are set. If true but those flags are not set etcd TLS certificates will be generated. Not supported if --experimental-self-hosted-etcd=true.")
 }
 
 func runCmdRender(cmd *cobra.Command, args []string) error {
@@ -98,12 +96,8 @@ func validateRenderOpts(cmd *cobra.Command, args []string) error {
 	if (renderOpts.etcdCAPath != "" || renderOpts.etcdCertificatePath != "" || renderOpts.etcdPrivateKeyPath != "") && (renderOpts.etcdCAPath == "" || renderOpts.etcdCertificatePath == "" || renderOpts.etcdPrivateKeyPath == "") {
 		return errors.New("You must specify either all or none of --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path")
 	}
-	if renderOpts.etcdCAPath != "" && !renderOpts.etcdUseTLS {
-		bootkube.UserOutput("etcd TLS certificates specified. Overriding --etcd-use-tls=true\n")
-		renderOpts.etcdUseTLS = true
-	}
-	if renderOpts.etcdUseTLS && renderOpts.selfHostedEtcd {
-		return errors.New("Cannot use --etcd-use-tls with --experimental-self-hosted-etcd")
+	if renderOpts.etcdCertificatePath != "" && renderOpts.selfHostedEtcd {
+		return errors.New("Cannot specify --etcd-certificate-path with --experimental-self-hosted-etcd")
 	}
 	if renderOpts.assetDir == "" {
 		return errors.New("Missing required flag: --asset-dir")
@@ -175,26 +169,18 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		if err != nil {
 			return nil, err
 		}
-
 		etcdServers = append(etcdServers, etcdServerUrl)
-
-		if renderOpts.etcdServers != defaultEtcdServers {
+		if renderOpts.etcdServers != defaultSelfHostedEtcdServers {
 			bootkube.UserOutput("--experimental-self-hosted-etcd and --service-cidr set. Overriding --etcd-servers setting with %s\n", etcdServers)
 		}
 	} else {
-		if renderOpts.etcdUseTLS && renderOpts.etcdServers == defaultEtcdServers {
-			renderOpts.etcdServers = defaultEtcdTLSServers
-		}
 		etcdServers, err = parseURLs(renderOpts.etcdServers)
 		if err != nil {
 			return nil, err
 		}
-	}
-
-	if renderOpts.etcdUseTLS {
 		for _, url := range etcdServers {
 			if url.Scheme != "https" {
-				return nil, fmt.Errorf("--etcd-use-tls=true but insecure etcd server endpoint specified: %s\n", url)
+				return nil, fmt.Errorf("etcd endpoints must use https, got: %s\n", url)
 			}
 		}
 	}
@@ -206,10 +192,10 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 			return nil, err
 		}
 	}
-	var etcdServerCert *x509.Certificate
-	var etcdServerKey *rsa.PrivateKey
+	var etcdClientCert *x509.Certificate
+	var etcdClientKey *rsa.PrivateKey
 	if renderOpts.etcdCertificatePath != "" {
-		etcdServerKey, etcdServerCert, err = parseCertAndPrivateKeyFromDisk(renderOpts.etcdCertificatePath, renderOpts.etcdPrivateKeyPath)
+		etcdClientKey, etcdClientCert, err = parseCertAndPrivateKeyFromDisk(renderOpts.etcdCertificatePath, renderOpts.etcdPrivateKeyPath)
 		if err != nil {
 			return nil, err
 		}
@@ -222,10 +208,9 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 
 	return &asset.Config{
 		EtcdCACert:      etcdCACert,
-		EtcdServerCert:  etcdServerCert,
-		EtcdServerKey:   etcdServerKey,
+		EtcdClientCert:  etcdClientCert,
+		EtcdClientKey:   etcdClientKey,
 		EtcdServers:     etcdServers,
-		EtcdUseTLS:      renderOpts.etcdUseTLS,
 		CACert:          caCert,
 		CAPrivKey:       caPrivKey,
 		APIServers:      apiServers,
@@ -234,7 +219,7 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		ServiceCIDR:     serviceNet,
 		APIServiceIP:    apiServiceIP,
 		DNSServiceIP:    dnsServiceIP,
-		ETCDServiceIP:   etcdServiceIP,
+		EtcdServiceIP:   etcdServiceIP,
 		SelfHostKubelet: renderOpts.selfHostKubelet,
 		CloudProvider:   renderOpts.cloudProvider,
 		SelfHostedEtcd:  renderOpts.selfHostedEtcd,

hack/quickstart/init-master.sh

@@ -25,16 +25,19 @@ function configure_etcd() {
 [Service]
 Environment="ETCD_IMAGE_TAG=v3.1.0"
 Environment="ETCD_NAME=controller"
-Environment="ETCD_INITIAL_CLUSTER=controller=http://${COREOS_PRIVATE_IPV4}:2380"
-Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://${COREOS_PRIVATE_IPV4}:2380"
+Environment="ETCD_INITIAL_CLUSTER=controller=https://${COREOS_PRIVATE_IPV4}:2380"
+Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${COREOS_PRIVATE_IPV4}:2380"
 Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${COREOS_PRIVATE_IPV4}:2379"
-Environment="ETCD_SSL_DIR=/etc/etcd/tls"
 Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
-Environment="ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380"
+Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+Environment="ETCD_SSL_DIR=/etc/etcd/tls"
 Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
-Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-server.crt"
-Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-server.key"
+Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-client.crt"
+Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-client.key"
 Environment="ETCD_CLIENT_CERT_AUTH=true"
+Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
+Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd-peer.crt"
+Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd-peer.key"
 EOF
     }
 }
@@ -48,7 +51,7 @@ function init_master_node() {
         echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
         etcd_render_flags="--experimental-self-hosted-etcd"
     else
-        etcd_render_flags="--etcd-use-tls --etcd-servers=https://${COREOS_PRIVATE_IPV4}:2379"
+        etcd_render_flags="--etcd-servers=https://${COREOS_PRIVATE_IPV4}:2379"
     fi
 
     # Render cluster assets

pkg/asset/asset.go

@@ -20,8 +20,10 @@ const (
 	AssetPathAPIServerKey                = "tls/apiserver.key"
 	AssetPathAPIServerCert               = "tls/apiserver.crt"
 	AssetPathEtcdCA                      = "tls/etcd-ca.crt"
-	AssetPathEtcdServerCert              = "tls/etcd-server.crt"
-	AssetPathEtcdServerKey               = "tls/etcd-server.key"
+	AssetPathEtcdClientCert              = "tls/etcd-client.crt"
+	AssetPathEtcdClientKey               = "tls/etcd-client.key"
+	AssetPathEtcdPeerCert                = "tls/etcd-peer.crt"
+	AssetPathEtcdPeerKey                 = "tls/etcd-peer.key"
 	AssetPathServiceAccountPrivKey       = "tls/service-account.key"
 	AssetPathServiceAccountPubKey        = "tls/service-account.pub"
 	AssetPathKubeletKey                  = "tls/kubelet.key"
@@ -59,10 +61,9 @@ const (
 // the default set of assets.
 type Config struct {
 	EtcdCACert          *x509.Certificate
-	EtcdServerCert      *x509.Certificate
-	EtcdServerKey       *rsa.PrivateKey
+	EtcdClientCert      *x509.Certificate
+	EtcdClientKey       *rsa.PrivateKey
 	EtcdServers         []*url.URL
-	EtcdUseTLS          bool
 	APIServers          []*url.URL
 	CACert              *x509.Certificate
 	CAPrivKey           *rsa.PrivateKey
@@ -71,7 +72,7 @@ type Config struct {
 	ServiceCIDR         *net.IPNet
 	APIServiceIP        net.IP
 	DNSServiceIP        net.IP
-	ETCDServiceIP       net.IP
+	EtcdServiceIP       net.IP
 	SelfHostKubelet     bool
 	SelfHostedEtcd      bool
 	CloudProvider       string
@@ -107,8 +108,8 @@ func NewDefaultAssets(conf Config) (Assets, error) {
 	as = append(as, tlsAssets...)
 
 	// etcd TLS assets.
-	if conf.EtcdUseTLS {
-		etcdTLSAssets, err := newEtcdTLSAssets(conf.EtcdCACert, conf.EtcdServerCert, conf.EtcdServerKey, conf.CACert, conf.CAPrivKey, conf.EtcdServers)
+	if !conf.SelfHostedEtcd {
+		etcdTLSAssets, err := newEtcdTLSAssets(conf.EtcdCACert, conf.EtcdClientCert, conf.EtcdClientKey, conf.CACert, conf.CAPrivKey, conf.EtcdServers)
 		if err != nil {
 			return Assets{}, err
 		}
@@ -123,7 +124,7 @@ func NewDefaultAssets(conf Config) (Assets, error) {
 	as = append(as, kubeConfig)
 
 	// K8S APIServer secret
-	apiSecret, err := newAPIServerSecretAsset(as, conf.EtcdUseTLS)
+	apiSecret, err := newAPIServerSecretAsset(as, conf.SelfHostedEtcd)
 	if err != nil {
 		return Assets{}, err
 	}

pkg/asset/internal/templates.go

@@ -171,10 +171,10 @@ spec:
         - --bind-address=0.0.0.0
         - --client-ca-file=/etc/kubernetes/secrets/ca.crt
         - --cloud-provider={{ .CloudProvider  }}
-{{- if .EtcdUseTLS }}
+{{- if not .SelfHostedEtcd }}
         - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt
-        - --etcd-certfile=/etc/kubernetes/secrets/etcd-server.crt
-        - --etcd-keyfile=/etc/kubernetes/secrets/etcd-server.key
+        - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
+        - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
 {{- end }}
         - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}
         - --insecure-port=8080
@@ -236,10 +236,10 @@ spec:
     - --authorization-mode=RBAC
     - --bind-address=0.0.0.0
     - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-{{- if .EtcdUseTLS }}
+{{- if not .SelfHostedEtcd }}
     - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt
-    - --etcd-certfile=/etc/kubernetes/secrets/etcd-server.crt
-    - --etcd-keyfile=/etc/kubernetes/secrets/etcd-server.key
+    - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
+    - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
 {{- end }}
     - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}{{ if .SelfHostedEtcd }},http://127.0.0.1:12379{{end}}
     - --insecure-port=8080
@@ -844,7 +844,7 @@ spec:
   selector:
     app: etcd
     etcd_cluster: kube-etcd
-  clusterIP: {{ .ETCDServiceIP }}
+  clusterIP: {{ .EtcdServiceIP }}
   ports:
   - name: client
     port: 2379

pkg/asset/k8s.go

@@ -87,18 +87,18 @@ func newKubeConfigAsset(assets Assets, conf Config) (Asset, error) {
 	})
 }
 
-func newAPIServerSecretAsset(assets Assets, etcdUseTLS bool) (Asset, error) {
+func newAPIServerSecretAsset(assets Assets, selfHostedEtcd bool) (Asset, error) {
 	secretAssets := []string{
 		AssetPathAPIServerKey,
 		AssetPathAPIServerCert,
 		AssetPathServiceAccountPubKey,
 		AssetPathCACert,
 	}
-	if etcdUseTLS {
+	if !selfHostedEtcd {
 		secretAssets = append(secretAssets, []string{
 			AssetPathEtcdCA,
-			AssetPathEtcdServerCert,
-			AssetPathEtcdServerKey,
+			AssetPathEtcdClientCert,
+			AssetPathEtcdClientKey,
 		}...)
 	}
 

pkg/asset/tls.go

@@ -115,24 +115,40 @@ func newKubeletKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey) (
 	return key, cert, err
 }
 
-func newEtcdTLSAssets(etcdCACert, etcdServerCert *x509.Certificate, etcdServerKey *rsa.PrivateKey, caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, etcdServers []*url.URL) ([]Asset, error) {
+func newEtcdTLSAssets(etcdCACert, etcdClientCert *x509.Certificate, etcdClientKey *rsa.PrivateKey, caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, etcdServers []*url.URL) ([]Asset, error) {
+	var assets []Asset
 	if etcdCACert == nil {
+		// Use the master CA to generate etcd assets.
+		etcdCACert = caCert
+
+		// Create an etcd client cert.
 		var err error
-		etcdServerKey, etcdServerCert, err = newEtcdServerKeyAndCert(caCert, caPrivKey, etcdServers)
+		etcdClientKey, etcdClientCert, err = newEtcdKeyAndCert(caCert, caPrivKey, "etcd-client", etcdServers)
 		if err != nil {
 			return nil, err
 		}
-		etcdCACert = caCert
+
+		// Create an etcd peer cert (not consumed by self-hosted components).
+		etcdPeerKey, etcdPeerCert, err := newEtcdKeyAndCert(caCert, caPrivKey, "etcd-peer", etcdServers)
+		if err != nil {
+			return nil, err
+		}
+		assets = append(assets, []Asset{
+			{Name: AssetPathEtcdPeerKey, Data: tlsutil.EncodePrivateKeyPEM(etcdPeerKey)},
+			{Name: AssetPathEtcdPeerCert, Data: tlsutil.EncodeCertificatePEM(etcdPeerCert)},
+		}...)
 	}
 
-	return []Asset{
+	assets = append(assets, []Asset{
 		{Name: AssetPathEtcdCA, Data: tlsutil.EncodeCertificatePEM(etcdCACert)},
-		{Name: AssetPathEtcdServerKey, Data: tlsutil.EncodePrivateKeyPEM(etcdServerKey)},
-		{Name: AssetPathEtcdServerCert, Data: tlsutil.EncodeCertificatePEM(etcdServerCert)},
-	}, nil
+		{Name: AssetPathEtcdClientKey, Data: tlsutil.EncodePrivateKeyPEM(etcdClientKey)},
+		{Name: AssetPathEtcdClientCert, Data: tlsutil.EncodeCertificatePEM(etcdClientCert)},
+	}...)
+
+	return assets, nil
 }
 
-func newEtcdServerKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, etcdServers []*url.URL) (*rsa.PrivateKey, *x509.Certificate, error) {
+func newEtcdKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, commonName string, etcdServers []*url.URL) (*rsa.PrivateKey, *x509.Certificate, error) {
 	key, err := tlsutil.NewPrivateKey()
 	if err != nil {
 		return nil, nil, err
@@ -147,7 +163,7 @@ func newEtcdServerKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey
 		}
 	}
 	config := tlsutil.CertConfig{
-		CommonName:   "etcd-server",
+		CommonName:   commonName,
 		Organization: []string{"etcd"},
 		AltNames:     altNames,
 	}