Merge pull request #585 from hongchaodeng/flag

hongchaodeng · web-flow · commit ab1b6b66197d · 2017-06-15T16:34:44.000-07:00
selfHostedEtcd: autogen certs from CA cert
diff --git a/cmd/bootkube/render.go b/cmd/bootkube/render.go
@@ -100,7 +100,8 @@ func validateRenderOpts(cmd *cobra.Command, args []string) error {
 		return errors.New("You must specify either all or none of --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path")
 	}
 	if renderOpts.etcdCertificatePath != "" && renderOpts.selfHostedEtcd {
-		return errors.New("Cannot specify --etcd-certificate-path with --experimental-self-hosted-etcd")
+		return errors.New("Cannot specify --etcd-certificate-path with --experimental-self-hosted-etcd." +
+			" Self-hosted etcd + TLS will auto-generate certs based on root CA cert.")
 	}
 	if renderOpts.assetDir == "" {
 		return errors.New("Missing required flag: --asset-dir")
@@ -211,7 +212,7 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		}
 	}
 
-	if etcdUseTLS && etcdCACert == nil {
+	if etcdUseTLS && etcdCACert == nil && !renderOpts.selfHostedEtcd {
 		bootkube.UserOutput("NOTE: --etcd-servers=%s but --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path were not set. Bootkube will create etcd certificates under '%s/tls'. You must configure etcd to use these certificates before invoking 'bootkube run'.\n", renderOpts.etcdServers, renderOpts.assetDir)
 	}
 
diff --git a/pkg/asset/asset.go b/pkg/asset/asset.go
@@ -15,48 +15,57 @@ import (
 )
 
 const (
-	AssetPathSecrets                     = "tls"
-	AssetPathCAKey                       = "tls/ca.key"
-	AssetPathCACert                      = "tls/ca.crt"
-	AssetPathAPIServerKey                = "tls/apiserver.key"
-	AssetPathAPIServerCert               = "tls/apiserver.crt"
-	AssetPathEtcdCA                      = "tls/etcd-ca.crt"
-	AssetPathEtcdClientCert              = "tls/etcd-client.crt"
-	AssetPathEtcdClientKey               = "tls/etcd-client.key"
-	AssetPathEtcdPeerCert                = "tls/etcd-peer.crt"
-	AssetPathEtcdPeerKey                 = "tls/etcd-peer.key"
-	AssetPathServiceAccountPrivKey       = "tls/service-account.key"
-	AssetPathServiceAccountPubKey        = "tls/service-account.pub"
-	AssetPathKubeletKey                  = "tls/kubelet.key"
-	AssetPathKubeletCert                 = "tls/kubelet.crt"
-	AssetPathKubeConfig                  = "auth/kubeconfig"
-	AssetPathManifests                   = "manifests"
-	AssetPathKubelet                     = "manifests/kubelet.yaml"
-	AssetPathProxy                       = "manifests/kube-proxy.yaml"
-	AssetPathKubeFlannel                 = "manifests/kube-flannel.yaml"
-	AssetPathKubeFlannelCfg              = "manifests/kube-flannel-cfg.yaml"
-	AssetPathAPIServerSecret             = "manifests/kube-apiserver-secret.yaml"
-	AssetPathAPIServer                   = "manifests/kube-apiserver.yaml"
-	AssetPathControllerManager           = "manifests/kube-controller-manager.yaml"
-	AssetPathControllerManagerSecret     = "manifests/kube-controller-manager-secret.yaml"
-	AssetPathControllerManagerDisruption = "manifests/kube-controller-manager-disruption.yaml"
-	AssetPathScheduler                   = "manifests/kube-scheduler.yaml"
-	AssetPathSchedulerDisruption         = "manifests/kube-scheduler-disruption.yaml"
-	AssetPathKubeDNSDeployment           = "manifests/kube-dns-deployment.yaml"
-	AssetPathKubeDNSSvc                  = "manifests/kube-dns-svc.yaml"
-	AssetPathSystemNamespace             = "manifests/kube-system-ns.yaml"
-	AssetPathCheckpointer                = "manifests/pod-checkpointer.yaml"
-	AssetPathEtcdOperator                = "manifests/etcd-operator.yaml"
-	AssetPathEtcdSvc                     = "manifests/etcd-service.yaml"
-	AssetPathKenc                        = "manifests/kube-etcd-network-checkpointer.yaml"
-	AssetPathKubeSystemSARoleBinding     = "manifests/kube-system-rbac-role-binding.yaml"
-	AssetPathBootstrapManifests          = "bootstrap-manifests"
-	AssetPathBootstrapAPIServer          = "bootstrap-manifests/bootstrap-apiserver.yaml"
-	AssetPathBootstrapControllerManager  = "bootstrap-manifests/bootstrap-controller-manager.yaml"
-	AssetPathBootstrapScheduler          = "bootstrap-manifests/bootstrap-scheduler.yaml"
-	AssetPathBootstrapEtcd               = "bootstrap-manifests/bootstrap-etcd.yaml"
-	AssetPathBootstrapEtcdService        = "etcd/bootstrap-etcd-service.json"
-	AssetPathMigrateEtcdCluster          = "etcd/migrate-etcd-cluster.json"
+	AssetPathSecrets                        = "tls"
+	AssetPathCAKey                          = "tls/ca.key"
+	AssetPathCACert                         = "tls/ca.crt"
+	AssetPathAPIServerKey                   = "tls/apiserver.key"
+	AssetPathAPIServerCert                  = "tls/apiserver.crt"
+	AssetPathEtcdCA                         = "tls/etcd-ca.crt"
+	AssetPathEtcdClientCert                 = "tls/etcd-client.crt"
+	AssetPathEtcdClientKey                  = "tls/etcd-client.key"
+	AssetPathEtcdPeerCert                   = "tls/etcd-peer.crt"
+	AssetPathEtcdPeerKey                    = "tls/etcd-peer.key"
+	AssetPathSelfHostedOperatorEtcdCA       = "tls/operator/etcd-ca-crt.pem"
+	AssetPathSelfHostedOperatorEtcdCert     = "tls/operator/etcd-crt.pem"
+	AssetPathSelfHostedOperatorEtcdKey      = "tls/operator/etcd-key.pem"
+	AssetPathSelfHostedEtcdMemberClientCA   = "tls/etcdMember/client-ca-crt.pem"
+	AssetPathSelfHostedEtcdMemberClientCert = "tls/etcdMember/client-crt.pem"
+	AssetPathSelfHostedEtcdMemberClientKey  = "tls/etcdMember/client-key.pem"
+	AssetPathSelfHostedEtcdMemberPeerCA     = "tls/etcdMember/peer-ca-crt.pem"
+	AssetPathSelfHostedEtcdMemberPeerCert   = "tls/etcdMember/peer-crt.pem"
+	AssetPathSelfHostedEtcdMemberPeerKey    = "tls/etcdMember/peer-key.pem"
+	AssetPathServiceAccountPrivKey          = "tls/service-account.key"
+	AssetPathServiceAccountPubKey           = "tls/service-account.pub"
+	AssetPathKubeletKey                     = "tls/kubelet.key"
+	AssetPathKubeletCert                    = "tls/kubelet.crt"
+	AssetPathKubeConfig                     = "auth/kubeconfig"
+	AssetPathManifests                      = "manifests"
+	AssetPathKubelet                        = "manifests/kubelet.yaml"
+	AssetPathProxy                          = "manifests/kube-proxy.yaml"
+	AssetPathKubeFlannel                    = "manifests/kube-flannel.yaml"
+	AssetPathKubeFlannelCfg                 = "manifests/kube-flannel-cfg.yaml"
+	AssetPathAPIServerSecret                = "manifests/kube-apiserver-secret.yaml"
+	AssetPathAPIServer                      = "manifests/kube-apiserver.yaml"
+	AssetPathControllerManager              = "manifests/kube-controller-manager.yaml"
+	AssetPathControllerManagerSecret        = "manifests/kube-controller-manager-secret.yaml"
+	AssetPathControllerManagerDisruption    = "manifests/kube-controller-manager-disruption.yaml"
+	AssetPathScheduler                      = "manifests/kube-scheduler.yaml"
+	AssetPathSchedulerDisruption            = "manifests/kube-scheduler-disruption.yaml"
+	AssetPathKubeDNSDeployment              = "manifests/kube-dns-deployment.yaml"
+	AssetPathKubeDNSSvc                     = "manifests/kube-dns-svc.yaml"
+	AssetPathSystemNamespace                = "manifests/kube-system-ns.yaml"
+	AssetPathCheckpointer                   = "manifests/pod-checkpointer.yaml"
+	AssetPathEtcdOperator                   = "manifests/etcd-operator.yaml"
+	AssetPathEtcdSvc                        = "manifests/etcd-service.yaml"
+	AssetPathKenc                           = "manifests/kube-etcd-network-checkpointer.yaml"
+	AssetPathKubeSystemSARoleBinding        = "manifests/kube-system-rbac-role-binding.yaml"
+	AssetPathBootstrapManifests             = "bootstrap-manifests"
+	AssetPathBootstrapAPIServer             = "bootstrap-manifests/bootstrap-apiserver.yaml"
+	AssetPathBootstrapControllerManager     = "bootstrap-manifests/bootstrap-controller-manager.yaml"
+	AssetPathBootstrapScheduler             = "bootstrap-manifests/bootstrap-scheduler.yaml"
+	AssetPathBootstrapEtcd                  = "bootstrap-manifests/bootstrap-etcd.yaml"
+	AssetPathBootstrapEtcdService           = "etcd/bootstrap-etcd-service.json"
+	AssetPathMigrateEtcdCluster             = "etcd/migrate-etcd-cluster.json"
 )
 
 var (
@@ -133,11 +142,19 @@ func NewDefaultAssets(conf Config) (Assets, error) {
 
 	// etcd TLS assets.
 	if conf.EtcdUseTLS {
-		etcdTLSAssets, err := newEtcdTLSAssets(conf.EtcdCACert, conf.EtcdClientCert, conf.EtcdClientKey, conf.CACert, conf.CAPrivKey, conf.EtcdServers)
-		if err != nil {
-			return Assets{}, err
+		if conf.SelfHostedEtcd {
+			tlsAssets, err := newSelfHostedEtcdTLSAssets(conf.EtcdServiceIP.String(), conf.BootEtcdServiceIP.String(), conf.CACert, conf.CAPrivKey)
+			if err != nil {
+				return nil, err
+			}
+			as = append(as, tlsAssets...)
+		} else {
+			etcdTLSAssets, err := newEtcdTLSAssets(conf.EtcdCACert, conf.EtcdClientCert, conf.EtcdClientKey, conf.CACert, conf.CAPrivKey, conf.EtcdServers)
+			if err != nil {
+				return Assets{}, err
+			}
+			as = append(as, etcdTLSAssets...)
 		}
-		as = append(as, etcdTLSAssets...)
 	}
 
 	// K8S kubeconfig
diff --git a/pkg/asset/tls.go b/pkg/asset/tls.go
@@ -148,14 +148,82 @@ func newEtcdTLSAssets(etcdCACert, etcdClientCert *x509.Certificate, etcdClientKe
 	return assets, nil
 }
 
+// newSelfHostedEtcdTLSAssets automatically generates three suites of x509 certificates (CA, key, cert)
+// for self-hosted etcd related components. Two suites are used by etcd members' client and peer ports;
+// one is used via etcd client to talk to etcd by operator, apiserver.
+// Self-hosted etcd doesn't allow user to specify etcd certs.
+func newSelfHostedEtcdTLSAssets(etcdSvcIP, bootEtcdSvcIP string, caCert *x509.Certificate, caPrivKey *rsa.PrivateKey) (Assets, error) {
+	// TODO: This method uses tlsutil.NewSignedCertificate() which will create certs for both client and server auth.
+	//       We can limit on finer granularity.
+
+	var assets []Asset
+
+	key, cert, err := newKeyAndCert(caCert, caPrivKey, "etcd member client", []string{
+		etcdSvcIP,
+		bootEtcdSvcIP,
+		"127.0.0.1",
+		"localhost",
+		"*.kube-etcd.kube-system.svc.cluster.local",
+		"kube-etcd-client.kube-system.svc.cluster.local",
+	})
+	if err != nil {
+		return nil, err
+	}
+	assets = append(assets, []Asset{
+		{Name: AssetPathSelfHostedEtcdMemberClientCA, Data: tlsutil.EncodeCertificatePEM(caCert)},
+		{Name: AssetPathSelfHostedEtcdMemberClientKey, Data: tlsutil.EncodePrivateKeyPEM(key)},
+		{Name: AssetPathSelfHostedEtcdMemberClientCert, Data: tlsutil.EncodeCertificatePEM(cert)},
+	}...)
+
+	key, cert, err = newKeyAndCert(caCert, caPrivKey, "etcd member peer", []string{
+		bootEtcdSvcIP,
+		"*.kube-etcd.kube-system.svc.cluster.local",
+		"kube-etcd-client.kube-system.svc.cluster.local",
+	})
+	if err != nil {
+		return nil, err
+	}
+	assets = append(assets, []Asset{
+		{Name: AssetPathSelfHostedEtcdMemberPeerCA, Data: tlsutil.EncodeCertificatePEM(caCert)},
+		{Name: AssetPathSelfHostedEtcdMemberPeerKey, Data: tlsutil.EncodePrivateKeyPEM(key)},
+		{Name: AssetPathSelfHostedEtcdMemberPeerCert, Data: tlsutil.EncodeCertificatePEM(cert)},
+	}...)
+
+	key, cert, err = newKeyAndCert(caCert, caPrivKey, "operator etcd client", nil)
+	if err != nil {
+		return nil, err
+	}
+	assets = append(assets, []Asset{
+		{Name: AssetPathSelfHostedOperatorEtcdCA, Data: tlsutil.EncodeCertificatePEM(caCert)},
+		{Name: AssetPathSelfHostedOperatorEtcdKey, Data: tlsutil.EncodePrivateKeyPEM(key)},
+		{Name: AssetPathSelfHostedOperatorEtcdCert, Data: tlsutil.EncodeCertificatePEM(cert)},
+	}...)
+	// for APIServer
+	assets = append(assets, []Asset{
+		{Name: AssetPathEtcdCA, Data: tlsutil.EncodeCertificatePEM(caCert)},
+		{Name: AssetPathEtcdClientKey, Data: tlsutil.EncodePrivateKeyPEM(key)},
+		{Name: AssetPathEtcdClientCert, Data: tlsutil.EncodeCertificatePEM(cert)},
+	}...)
+
+	return assets, nil
+}
+
 func newEtcdKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, commonName string, etcdServers []*url.URL) (*rsa.PrivateKey, *x509.Certificate, error) {
+	addrs := make([]string, len(etcdServers))
+	for i := range etcdServers {
+		addrs[i] = etcdServers[i].Host
+	}
+	return newKeyAndCert(caCert, caPrivKey, commonName, addrs)
+}
+
+func newKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, commonName string, addrs []string) (*rsa.PrivateKey, *x509.Certificate, error) {
 	key, err := tlsutil.NewPrivateKey()
 	if err != nil {
 		return nil, nil, err
 	}
 	var altNames tlsutil.AltNames
-	for _, etcdServer := range etcdServers {
-		hostname := stripPort(etcdServer.Host)
+	for _, addr := range addrs {
+		hostname := stripPort(addr)
 		if ip := net.ParseIP(hostname); ip != nil {
 			altNames.IPs = append(altNames.IPs, ip)
 		} else {

Original file line number	Diff line number	Diff line change
`@@ -100,7 +100,8 @@ func validateRenderOpts(cmd *cobra.Command, args []string) error {`
`100`	`100`	`return errors.New("You must specify either all or none of --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path")`
`101`	`101`	`}`
`102`	`102`	`if renderOpts.etcdCertificatePath != "" && renderOpts.selfHostedEtcd {`
`103`		`- return errors.New("Cannot specify --etcd-certificate-path with --experimental-self-hosted-etcd")`
	`103`	`+ return errors.New("Cannot specify --etcd-certificate-path with --experimental-self-hosted-etcd." +`
	`104`	`+ " Self-hosted etcd + TLS will auto-generate certs based on root CA cert.")`
`104`	`105`	`}`
`105`	`106`	`if renderOpts.assetDir == "" {`
`106`	`107`	`return errors.New("Missing required flag: --asset-dir")`
`@@ -211,7 +212,7 @@ func flagsToAssetConfig() (c *asset.Config, err error) {`
`211`	`212`	`}`
`212`	`213`	`}`
`213`	`214`
`214`		`- if etcdUseTLS && etcdCACert == nil {`
	`215`	`+ if etcdUseTLS && etcdCACert == nil && !renderOpts.selfHostedEtcd {`
`215`	`216`	`bootkube.UserOutput("NOTE: --etcd-servers=%s but --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path were not set. Bootkube will create etcd certificates under '%s/tls'. You must configure etcd to use these certificates before invoking 'bootkube run'.\n", renderOpts.etcdServers, renderOpts.assetDir)`
`216`	`217`	`}`
`217`	`218`