Merge pull request #184 from kubernetes-incubator/s

self hosted etcd migration

Author: aaronlevy

cmd/bootkube/render.go

@@ -34,6 +34,7 @@ var (
 		apiServers        string
 		altNames          string
 		selfHostKubelet   bool
+		storageBackend    string
 	}
 )
 
@@ -43,6 +44,7 @@ func init() {
 	cmdRender.Flags().StringVar(&renderOpts.caCertificatePath, "ca-certificate-path", "", "Path to an existing PEM encoded CA. If provided, TLS assets will be generated using this certificate authority.")
 	cmdRender.Flags().StringVar(&renderOpts.caPrivateKeyPath, "ca-private-key-path", "", "Path to an existing Certificate Authority RSA private key. Required if --ca-certificate is set.")
 	cmdRender.Flags().StringVar(&renderOpts.etcdServers, "etcd-servers", "http://127.0.0.1:2379", "List of etcd servers URLs including host:port, comma separated")
+	cmdRender.Flags().StringVar(&renderOpts.storageBackend, "storage-backend", "etcd2", "storage backend for APIServer. Supports: etcd2, etcd3.")
 	cmdRender.Flags().StringVar(&renderOpts.apiServers, "api-servers", "https://127.0.0.1:443", "List of API server URLs including host:port, commma seprated")
 	cmdRender.Flags().StringVar(&renderOpts.altNames, "api-server-alt-names", "", "List of SANs to use in api-server certificate. Example: 'IP=127.0.0.1,IP=127.0.0.2,DNS=localhost'. If empty, SANs will be extracted from the --api-servers flag.")
 	cmdRender.Flags().BoolVar(&renderOpts.selfHostKubelet, "self-host-kubelet", false, "Create a self-hosted kubelet daemonset.")
@@ -114,6 +116,7 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		APIServers:      apiServers,
 		AltNames:        altNames,
 		SelfHostKubelet: renderOpts.selfHostKubelet,
+		StorageBackend:  renderOpts.storageBackend,
 	}, nil
 }
 

cmd/bootkube/start.go

@@ -51,8 +51,9 @@ func runCmdStart(cmd *cobra.Command, args []string) error {
 	}
 
 	bk, err := bootkube.NewBootkube(bootkube.Config{
-		AssetDir:   startOpts.assetDir,
-		EtcdServer: etcdServer,
+		AssetDir:       startOpts.assetDir,
+		EtcdServer:     etcdServer,
+		SelfHostedEtcd: startOpts.selfHostedEtcd,
 	})
 
 	if err != nil {

pkg/asset/asset.go

@@ -34,6 +34,8 @@ const (
 	AssetPathKubeDNSDeployment       = "manifests/kube-dns-deployment.yaml"
 	AssetPathKubeDNSSvc              = "manifests/kube-dns-svc.yaml"
 	AssetPathSystemNamespace         = "manifests/kube-system-ns.yaml"
+	AssetPathEtcdOperator            = "manifests/etcd-operator.yaml"
+	AssetPathEtcdSvc                 = "manifests/etcd-service.yaml"
 )
 
 // AssetConfig holds all configuration needed when generating
@@ -45,6 +47,7 @@ type Config struct {
 	CAPrivKey       *rsa.PrivateKey
 	AltNames        *tlsutil.AltNames
 	SelfHostKubelet bool
+	StorageBackend  string
 }
 
 // NewDefaultAssets returns a list of default assets, optionally

pkg/asset/internal/templates.go

@@ -133,6 +133,7 @@ spec:
         - --insecure-port=8080
         - --advertise-address=$(MY_POD_IP)
         - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}
+        - --storage-backend={{.StorageBackend}}
         - --allow-privileged=true
         - --service-cluster-ip-range=10.3.0.0/24
         - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota
@@ -419,5 +420,43 @@ spec:
   - name: dns-tcp
     port: 53
     protocol: TCP
+`)
+	EtcdOperatorTemplate = []byte(`apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: etcd-operator
+  namespace: kube-system
+  labels:
+    k8s-app: etcd-operator
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: etcd-operator
+    spec:
+      containers:
+      - name: etcd-operator
+        image: quay.io/coreos/etcd-operator
+        env:
+        - name: MY_POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+`)
+	EtcdSvcTemplate = []byte(`apiVersion: v1
+kind: Service
+metadata:
+  name: etcd-service
+  namespace: kube-system
+spec:
+  selector:
+    app: etcd
+    etcd_cluster: etcd-cluster
+  clusterIP: 10.3.0.15
+  ports:
+  - name: client
+    port: 2379
+    protocol: TCP
 `)
 )

pkg/asset/k8s.go

@@ -26,6 +26,8 @@ func newStaticAssets(selfHostKubelet bool) Assets {
 		mustCreateAssetFromTemplate(AssetPathKubeDNSDeployment, internal.DNSDeploymentTemplate, noData),
 		mustCreateAssetFromTemplate(AssetPathKubeDNSSvc, internal.DNSSvcTemplate, noData),
 		mustCreateAssetFromTemplate(AssetPathCheckpointer, internal.CheckpointerTemplate, noData),
+		mustCreateAssetFromTemplate(AssetPathEtcdOperator, internal.EtcdOperatorTemplate, noData),
+		mustCreateAssetFromTemplate(AssetPathEtcdSvc, internal.EtcdSvcTemplate, noData),
 	}
 	if selfHostKubelet {
 		assets = append(assets, mustCreateAssetFromTemplate(AssetPathKubelet, internal.KubeletTemplate, noData))

pkg/bootkube/bootkube.go

@@ -30,35 +30,24 @@ var requiredPods = []string{
 }
 
 type Config struct {
-	AssetDir   string
-	EtcdServer *url.URL
+	AssetDir       string
+	EtcdServer     *url.URL
+	SelfHostedEtcd bool
 }
 
 type bootkube struct {
-	assetDir   string
-	apiServer  *apiserver.APIServer
-	controller *controller.CMServer
-	scheduler  *scheduler.SchedulerServer
+	selfHostedEtcd bool
+	assetDir       string
+	apiServer      *apiserver.APIServer
+	controller     *controller.CMServer
+	scheduler      *scheduler.SchedulerServer
 }
 
 func NewBootkube(config Config) (*bootkube, error) {
 	apiServer := apiserver.NewAPIServer()
 	fs := pflag.NewFlagSet("apiserver", pflag.ExitOnError)
 	apiServer.AddFlags(fs)
-	fs.Parse([]string{
-		"--bind-address=0.0.0.0",
-		"--secure-port=443",
-		"--insecure-port=8080",
-		"--allow-privileged=true",
-		"--tls-private-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerKey),
-		"--tls-cert-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerCert),
-		"--client-ca-file=" + filepath.Join(config.AssetDir, asset.AssetPathCACert),
-		"--etcd-servers=" + config.EtcdServer.String(),
-		"--service-cluster-ip-range=10.3.0.0/24",
-		"--service-account-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathServiceAccountPubKey),
-		"--admission-control=NamespaceLifecycle,ServiceAccount",
-		"--runtime-config=api/all=true",
-	})
+	fs.Parse(makeAPIServerFlags(config))
 
 	cmServer := controller.NewCMServer()
 	fs = pflag.NewFlagSet("controllermanager", pflag.ExitOnError)
@@ -79,13 +68,35 @@ func NewBootkube(config Config) (*bootkube, error) {
 	})
 
 	return &bootkube{
-		apiServer:  apiServer,
-		controller: cmServer,
-		scheduler:  schedServer,
-		assetDir:   config.AssetDir,
+		apiServer:      apiServer,
+		controller:     cmServer,
+		scheduler:      schedServer,
+		assetDir:       config.AssetDir,
+		selfHostedEtcd: config.SelfHostedEtcd,
 	}, nil
 }
 
+func makeAPIServerFlags(config Config) []string {
+	res := []string{
+		"--bind-address=0.0.0.0",
+		"--secure-port=443",
+		"--insecure-port=8080",
+		"--allow-privileged=true",
+		"--tls-private-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerKey),
+		"--tls-cert-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerCert),
+		"--client-ca-file=" + filepath.Join(config.AssetDir, asset.AssetPathCACert),
+		"--etcd-servers=" + config.EtcdServer.String(),
+		"--service-cluster-ip-range=10.3.0.0/24",
+		"--service-account-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathServiceAccountPubKey),
+		"--admission-control=NamespaceLifecycle,ServiceAccount",
+		"--runtime-config=api/all=true",
+	}
+	if config.SelfHostedEtcd {
+		res = append(res, "--storage-backend=etcd3")
+	}
+	return res
+}
+
 func (b *bootkube) Run() error {
 	UserOutput("Running temporary bootstrap control plane...\n")
 
@@ -98,7 +109,7 @@ func (b *bootkube) Run() error {
 			errch <- err
 		}
 	}()
-	go func() { errch <- WaitUntilPodsRunning(requiredPods, assetTimeout) }()
+	go func() { errch <- WaitUntilPodsRunning(requiredPods, assetTimeout, b.selfHostedEtcd) }()
 
 	// If any of the bootkube services exit, it means it is unrecoverable and we should exit.
 	err := <-errch

pkg/bootkube/status.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"github.com/kubernetes-incubator/bootkube/pkg/util/etcdutil"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/cache"
@@ -24,7 +25,7 @@ const (
 	doesNotExist = "DoesNotExist"
 )
 
-func WaitUntilPodsRunning(pods []string, timeout time.Duration) error {
+func WaitUntilPodsRunning(pods []string, timeout time.Duration, selfHostedEtcd bool) error {
 	sc, err := NewStatusController(pods)
 	if err != nil {
 		return err
@@ -35,6 +36,12 @@ func WaitUntilPodsRunning(pods []string, timeout time.Duration) error {
 		return fmt.Errorf("error while checking pod status: %v", err)
 	}
 
+	if selfHostedEtcd {
+		if err := etcdutil.Migrate(); err != nil {
+			return err
+		}
+	}
+
 	UserOutput("All self-hosted control plane components successfully started\n")
 	return nil
 }