Merge pull request #433 from diegs/etcd-tls

Add support for generating and using etcd TLS assets.

Author: diegs

cmd/bootkube/render.go

@@ -18,11 +18,12 @@ import (
 )
 
 const (
-	apiOffset            = 1
-	dnsOffset            = 10
-	etcdOffset           = 15
-	defaultServiceBaseIP = "10.3.0.0"
-	defaultEtcdServers   = "http://127.0.0.1:2379"
+	apiOffset                    = 1
+	dnsOffset                    = 10
+	etcdOffset                   = 15
+	defaultServiceBaseIP         = "10.3.0.0"
+	defaultEtcdServers           = "https://127.0.0.1:2379"
+	defaultSelfHostedEtcdServers = "http://127.0.0.1:2379"
 )
 
 var (
@@ -36,17 +37,20 @@ var (
 	}
 
 	renderOpts struct {
-		assetDir          string
-		caCertificatePath string
-		caPrivateKeyPath  string
-		etcdServers       string
-		apiServers        string
-		altNames          string
-		podCIDR           string
-		serviceCIDR       string
-		selfHostKubelet   bool
-		cloudProvider     string
-		selfHostedEtcd    bool
+		assetDir            string
+		caCertificatePath   string
+		caPrivateKeyPath    string
+		etcdCAPath          string
+		etcdCertificatePath string
+		etcdPrivateKeyPath  string
+		etcdServers         string
+		apiServers          string
+		altNames            string
+		podCIDR             string
+		serviceCIDR         string
+		selfHostKubelet     bool
+		cloudProvider       string
+		selfHostedEtcd      bool
 	}
 )
 
@@ -55,6 +59,9 @@ func init() {
 	cmdRender.Flags().StringVar(&renderOpts.assetDir, "asset-dir", "", "Output path for rendered assets")
 	cmdRender.Flags().StringVar(&renderOpts.caCertificatePath, "ca-certificate-path", "", "Path to an existing PEM encoded CA. If provided, TLS assets will be generated using this certificate authority.")
 	cmdRender.Flags().StringVar(&renderOpts.caPrivateKeyPath, "ca-private-key-path", "", "Path to an existing Certificate Authority RSA private key. Required if --ca-certificate is set.")
+	cmdRender.Flags().StringVar(&renderOpts.etcdCAPath, "etcd-ca-path", "", "Path to an existing PEM encoded CA that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-certificate-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRender.Flags().StringVar(&renderOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRender.Flags().StringVar(&renderOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.")
 	cmdRender.Flags().StringVar(&renderOpts.etcdServers, "etcd-servers", defaultEtcdServers, "List of etcd servers URLs including host:port, comma separated")
 	cmdRender.Flags().StringVar(&renderOpts.apiServers, "api-servers", "https://127.0.0.1:443", "List of API server URLs including host:port, commma seprated")
 	cmdRender.Flags().StringVar(&renderOpts.altNames, "api-server-alt-names", "", "List of SANs to use in api-server certificate. Example: 'IP=127.0.0.1,IP=127.0.0.2,DNS=localhost'. If empty, SANs will be extracted from the --api-servers flag.")
@@ -86,6 +93,12 @@ func validateRenderOpts(cmd *cobra.Command, args []string) error {
 	if renderOpts.caPrivateKeyPath != "" && renderOpts.caCertificatePath == "" {
 		return errors.New("You must provide the --ca-certificate-path flag when --ca-private-key-path is provided.")
 	}
+	if (renderOpts.etcdCAPath != "" || renderOpts.etcdCertificatePath != "" || renderOpts.etcdPrivateKeyPath != "") && (renderOpts.etcdCAPath == "" || renderOpts.etcdCertificatePath == "" || renderOpts.etcdPrivateKeyPath == "") {
+		return errors.New("You must specify either all or none of --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path")
+	}
+	if renderOpts.etcdCertificatePath != "" && renderOpts.selfHostedEtcd {
+		return errors.New("Cannot specify --etcd-certificate-path with --experimental-self-hosted-etcd")
+	}
 	if renderOpts.assetDir == "" {
 		return errors.New("Missing required flag: --asset-dir")
 	}
@@ -156,10 +169,8 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		if err != nil {
 			return nil, err
 		}
-
 		etcdServers = append(etcdServers, etcdServerUrl)
-
-		if renderOpts.etcdServers != defaultEtcdServers {
+		if renderOpts.etcdServers != defaultSelfHostedEtcdServers {
 			bootkube.UserOutput("--experimental-self-hosted-etcd and --service-cidr set. Overriding --etcd-servers setting with %s\n", etcdServers)
 		}
 	} else {
@@ -169,13 +180,44 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		}
 	}
 
+	etcdUseTLS := false
+	for _, url := range etcdServers {
+		if url.Scheme == "https" {
+			etcdUseTLS = true
+		}
+	}
+
+	var etcdCACert *x509.Certificate
+	if renderOpts.etcdCAPath != "" {
+		etcdCACert, err = parseCertFromDisk(renderOpts.etcdCAPath)
+		if err != nil {
+			return nil, err
+		}
+	}
+	var etcdClientCert *x509.Certificate
+	var etcdClientKey *rsa.PrivateKey
+	if renderOpts.etcdCertificatePath != "" {
+		etcdClientKey, etcdClientCert, err = parseCertAndPrivateKeyFromDisk(renderOpts.etcdCertificatePath, renderOpts.etcdPrivateKeyPath)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if etcdUseTLS && etcdCACert == nil {
+		bootkube.UserOutput("NOTE: --etcd-servers=%s but -etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path were not set. Bootkube will create etcd certificates under '%s/tls'. You must configure etcd to use these certificates before invoking 'bootkube run'.\n", renderOpts.etcdServers, renderOpts.assetDir)
+	}
+
 	// TODO: Find better option than asking users to make manual changes
 	if serviceNet.IP.String() != defaultServiceBaseIP {
 		fmt.Printf("You have selected a non-default service CIDR %s - be sure your kubelet service file uses --cluster-dns=%s\n", serviceNet.String(), dnsServiceIP.String())
 	}
 
 	return &asset.Config{
+		EtcdCACert:      etcdCACert,
+		EtcdClientCert:  etcdClientCert,
+		EtcdClientKey:   etcdClientKey,
 		EtcdServers:     etcdServers,
+		EtcdUseTLS:      etcdUseTLS,
 		CACert:          caCert,
 		CAPrivKey:       caPrivKey,
 		APIServers:      apiServers,
@@ -184,7 +226,7 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		ServiceCIDR:     serviceNet,
 		APIServiceIP:    apiServiceIP,
 		DNSServiceIP:    dnsServiceIP,
-		ETCDServiceIP:   etcdServiceIP,
+		EtcdServiceIP:   etcdServiceIP,
 		SelfHostKubelet: renderOpts.selfHostKubelet,
 		CloudProvider:   renderOpts.cloudProvider,
 		SelfHostedEtcd:  renderOpts.selfHostedEtcd,
@@ -202,15 +244,23 @@ func parseCertAndPrivateKeyFromDisk(caCertPath, privKeyPath string) (*rsa.Privat
 		return nil, nil, fmt.Errorf("unable to parse CA private key: %v", err)
 	}
 	// Parse CA Cert.
+	cert, err := parseCertFromDisk(caCertPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	return key, cert, nil
+}
+
+func parseCertFromDisk(caCertPath string) (*x509.Certificate, error) {
 	capem, err := ioutil.ReadFile(caCertPath)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error reading ca cert file at %s: %v", caCertPath, err)
+		return nil, fmt.Errorf("error reading ca cert file at %s: %v", caCertPath, err)
 	}
 	cert, err := tlsutil.ParsePEMEncodedCACert(capem)
 	if err != nil {
-		return nil, nil, fmt.Errorf("unable to parse CA Cert: %v", err)
+		return nil, fmt.Errorf("unable to parse CA Cert: %v", err)
 	}
-	return key, cert, nil
+	return cert, nil
 }
 
 func parseURLs(s string) ([]*url.URL, error) {

hack/multi-node/Vagrantfile

@@ -22,6 +22,7 @@ CONTROLLER_USER_DATA_PATH = File.expand_path("./cluster/user-data-controller")
 WORKER_USER_DATA_PATH = File.expand_path("./cluster/user-data-worker")
 KUBECONFIG_PATH = File.expand_path("cluster/auth/kubeconfig")
 CA_CERT_PATH = File.expand_path("cluster/tls/ca.crt")
+ETCD_CERT_GLOB = File.expand_path("cluster/tls/etcd-*")
 
 def etcdIP(num)
   return "172.17.4.#{num+50}"
@@ -41,7 +42,7 @@ if !$self_host_etcd
     $etcd_vm_memory = 512
     ETCD_CLOUD_CONFIG_PATH = File.expand_path("./etcd-cloud-config.yaml")
     etcdIPs = [*1..$etcd_count].map{ |i| etcdIP(i) }
-    initial_etcd_cluster = etcdIPs.map.with_index{ |ip, i| "e#{i+1}=http://#{ip}:2380" }.join(",")
+    initial_etcd_cluster = etcdIPs.map.with_index{ |ip, i| "e#{i+1}=https://#{ip}:2380" }.join(",")
 end
 
 Vagrant.configure("2") do |config|
@@ -109,6 +110,14 @@ Vagrant.configure("2") do |config|
 
         etcd.vm.provision :file, source: etcd_config_file.path, destination: "/tmp/vagrantfile-user-data"
         etcd.vm.provision :shell, inline: "mv /tmp/vagrantfile-user-data /var/lib/coreos-vagrant/", privileged: true
+
+        etcd.vm.provision :shell, :inline => "mkdir -p /etc/etcd/tls", :privileged => true
+        Dir.glob(ETCD_CERT_GLOB) do |etcd_cert_file|
+          etcd.vm.provision :file, :source => etcd_cert_file, :destination => "/tmp/#{File.basename(etcd_cert_file)}"
+          etcd.vm.provision :shell, :inline => "mv /tmp/#{File.basename(etcd_cert_file)} /etc/etcd/tls/", :privileged => true
+        end
+        etcd.vm.provision :shell, :inline => "chown -R etcd:etcd /etc/etcd", :privileged => true
+        etcd.vm.provision :shell, :inline => "chmod -R u=rX,g=,o= /etc/etcd", :privileged => true
       end
     end
   end

hack/multi-node/bootkube-up

@@ -14,7 +14,8 @@ if [ ${SELF_HOST_ETCD} = "true" ]; then
     echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
     etcd_render_flags="--experimental-self-hosted-etcd"
 else
-    etcd_render_flags="--etcd-servers=http://172.17.4.51:2379"
+    # Note: if you increase the number of etcd servers in the Vagrantfile you must also add them here.
+    etcd_render_flags="--etcd-servers=https://172.17.4.51:2379"
 fi
 
 # Render assets

hack/multi-node/etcd-cloud-config.yaml

@@ -15,8 +15,16 @@ coreos:
           [Service]
           Environment="ETCD_IMAGE_TAG=v3.1.0"
           Environment="ETCD_NAME={{ETCD_NODE_NAME}}"
-          Environment="ETCD_ADVERTISE_CLIENT_URLS=http://$private_ipv4:2379"
-          Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://$private_ipv4:2380"
-          Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379"
-          Environment="ETCD_LISTEN_PEER_URLS=http://$private_ipv4:2380"
+          Environment="ETCD_ADVERTISE_CLIENT_URLS=https://$private_ipv4:2379"
+          Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://$private_ipv4:2380"
+          Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+          Environment="ETCD_LISTEN_PEER_URLS=https://$private_ipv4:2380"
           Environment="ETCD_INITIAL_CLUSTER={{ETCD_INITIAL_CLUSTER}}"
+          Environment="ETCD_SSL_DIR=/etc/etcd/tls"
+          Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
+          Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-client.crt"
+          Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-client.key"
+          Environment="ETCD_CLIENT_CERT_AUTH=true"
+          Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
+          Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd-peer.crt"
+          Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd-peer.key"

hack/quickstart/init-master.sh

@@ -16,16 +16,28 @@ function usage() {
 
 function configure_etcd() {
     [ -f "/etc/systemd/system/etcd-member.service.d/10-etcd-member.conf" ] || {
+        mkdir -p /etc/etcd/tls
+        cp /home/core/assets/tls/etcd* /etc/etcd/tls
+        chown -R etcd:etcd /etc/etcd
+        chmod -R u=rX,g=,o= /etc/etcd
         mkdir -p /etc/systemd/system/etcd-member.service.d
         cat << EOF > /etc/systemd/system/etcd-member.service.d/10-etcd-member.conf
 [Service]
 Environment="ETCD_IMAGE_TAG=v3.1.0"
 Environment="ETCD_NAME=controller"
-Environment="ETCD_INITIAL_CLUSTER=controller=http://${COREOS_PRIVATE_IPV4}:2380"
-Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://${COREOS_PRIVATE_IPV4}:2380"
-Environment="ETCD_ADVERTISE_CLIENT_URLS=http://${COREOS_PRIVATE_IPV4}:2379"
-Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379"
-Environment="ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380"
+Environment="ETCD_INITIAL_CLUSTER=controller=https://${COREOS_PRIVATE_IPV4}:2380"
+Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${COREOS_PRIVATE_IPV4}:2380"
+Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${COREOS_PRIVATE_IPV4}:2379"
+Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+Environment="ETCD_SSL_DIR=/etc/etcd/tls"
+Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
+Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-client.crt"
+Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-client.key"
+Environment="ETCD_CLIENT_CERT_AUTH=true"
+Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
+Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd-peer.crt"
+Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd-peer.key"
 EOF
     }
 }
@@ -35,26 +47,29 @@ function init_master_node() {
     systemctl daemon-reload
     systemctl stop update-engine; systemctl mask update-engine
 
-    etcd_render_flags=""
-
-    # Start etcd.
     if [ "$SELF_HOST_ETCD" = true ] ; then
         echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
         etcd_render_flags="--experimental-self-hosted-etcd"
     else
-        configure_etcd
-        systemctl enable etcd-member; sudo systemctl start etcd-member
+        etcd_render_flags="--etcd-servers=https://${COREOS_PRIVATE_IPV4}:2379"
     fi
 
     # Render cluster assets
-    /home/core/bootkube render --asset-dir=/home/core/assets --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443 ${etcd_render_flags}
+    /home/core/bootkube render --asset-dir=/home/core/assets ${etcd_render_flags} \
+      --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443
 
     # Move the local kubeconfig into expected location
     chown -R core:core /home/core/assets
     mkdir -p /etc/kubernetes
     cp /home/core/assets/auth/kubeconfig /etc/kubernetes/
     cp /home/core/assets/tls/ca.crt /etc/kubernetes/ca.crt
 
+    # Start etcd.
+    if [ "$SELF_HOST_ETCD" = false ] ; then
+        configure_etcd
+        systemctl enable etcd-member; sudo systemctl start etcd-member
+    fi
+
     # Start the kubelet
     systemctl enable kubelet; sudo systemctl start kubelet
 

hack/single-node/Vagrantfile

@@ -14,6 +14,7 @@ NODE_IP = "172.17.4.100"
 USER_DATA_PATH = File.expand_path("cluster/user-data")
 KUBECONFIG_PATH = File.expand_path("cluster/auth/kubeconfig")
 CA_CERT_PATH = File.expand_path("cluster/tls/ca.crt")
+ETCD_CERT_GLOB = File.expand_path("cluster/tls/etcd-*")
 
 Vagrant.configure("2") do |config|
   # always use Vagrant's insecure key
@@ -61,4 +62,12 @@ Vagrant.configure("2") do |config|
 
   config.vm.provision :file, :source => CA_CERT_PATH, :destination => "/tmp/ca.crt"
   config.vm.provision :shell, :inline => "mv /tmp/ca.crt /etc/kubernetes/ca.crt", :privileged => true
+
+  config.vm.provision :shell, :inline => "mkdir -p /etc/etcd/tls", :privileged => true
+  Dir.glob(ETCD_CERT_GLOB) do |etcd_cert_file|
+    config.vm.provision :file, :source => etcd_cert_file, :destination => "/tmp/#{File.basename(etcd_cert_file)}"
+    config.vm.provision :shell, :inline => "mv /tmp/#{File.basename(etcd_cert_file)} /etc/etcd/tls/", :privileged => true
+  end
+  config.vm.provision :shell, :inline => "chown -R etcd:etcd /etc/etcd", :privileged => true
+  config.vm.provision :shell, :inline => "chmod -R u=rX,g=,o= /etc/etcd", :privileged => true
 end

hack/single-node/user-data-etcd.sample

@@ -5,4 +5,18 @@
           content: |
             [Service]
             Environment="ETCD_IMAGE_TAG=v3.1.0"
+            Environment="ETCD_NAME=default"
+            Environment="ETCD_INITIAL_CLUSTER=default=https://127.0.0.1:2380"
+            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://127.0.0.1:2380"
+            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://127.0.0.1:2379"
+            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_SSL_DIR=/etc/etcd/tls"
+            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
+            Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-client.crt"
+            Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-client.key"
+            Environment="ETCD_CLIENT_CERT_AUTH=true"
+            Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt"
+            Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd-peer.crt"
+            Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd-peer.key"
       command: start