Move etcd TPR generation to render phase. (#496)

This requires putting the boot etcd server behind a service so it can
have a pre-determined IP address (before this change the pod IP was
being dynamically detected).

Author: diegs

cmd/bootkube/render.go

@@ -21,6 +21,7 @@ const (
 	apiOffset                    = 1
 	dnsOffset                    = 10
 	etcdOffset                   = 15
+	bootEtcdOffset               = 200
 	defaultServiceBaseIP         = "10.3.0.0"
 	defaultEtcdServers           = "https://127.0.0.1:2379"
 	defaultSelfHostedEtcdServers = "http://127.0.0.1:2379"
@@ -171,6 +172,11 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		return nil, err
 	}
 
+	bootEtcdServiceIP, err := offsetServiceIP(serviceNet, bootEtcdOffset)
+	if err != nil {
+		return nil, err
+	}
+
 	etcdServiceIP, err := offsetServiceIP(serviceNet, etcdOffset)
 	if err != nil {
 		return nil, err
@@ -226,24 +232,25 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 	}
 
 	return &asset.Config{
-		EtcdCACert:      etcdCACert,
-		EtcdClientCert:  etcdClientCert,
-		EtcdClientKey:   etcdClientKey,
-		EtcdServers:     etcdServers,
-		EtcdUseTLS:      etcdUseTLS,
-		CACert:          caCert,
-		CAPrivKey:       caPrivKey,
-		APIServers:      apiServers,
-		AltNames:        altNames,
-		PodCIDR:         podNet,
-		ServiceCIDR:     serviceNet,
-		APIServiceIP:    apiServiceIP,
-		DNSServiceIP:    dnsServiceIP,
-		EtcdServiceIP:   etcdServiceIP,
-		SelfHostKubelet: renderOpts.selfHostKubelet,
-		CloudProvider:   renderOpts.cloudProvider,
-		SelfHostedEtcd:  renderOpts.selfHostedEtcd,
-		Images:          imageVersions,
+		EtcdCACert:        etcdCACert,
+		EtcdClientCert:    etcdClientCert,
+		EtcdClientKey:     etcdClientKey,
+		EtcdServers:       etcdServers,
+		EtcdUseTLS:        etcdUseTLS,
+		CACert:            caCert,
+		CAPrivKey:         caPrivKey,
+		APIServers:        apiServers,
+		AltNames:          altNames,
+		PodCIDR:           podNet,
+		ServiceCIDR:       serviceNet,
+		APIServiceIP:      apiServiceIP,
+		BootEtcdServiceIP: bootEtcdServiceIP,
+		DNSServiceIP:      dnsServiceIP,
+		EtcdServiceIP:     etcdServiceIP,
+		SelfHostKubelet:   renderOpts.selfHostKubelet,
+		CloudProvider:     renderOpts.cloudProvider,
+		SelfHostedEtcd:    renderOpts.selfHostedEtcd,
+		Images:            imageVersions,
 	}, nil
 }
 

pkg/asset/asset.go

@@ -55,6 +55,8 @@ const (
 	AssetPathBootstrapControllerManager  = "bootstrap-manifests/bootstrap-controller-manager.yaml"
 	AssetPathBootstrapScheduler          = "bootstrap-manifests/bootstrap-scheduler.yaml"
 	AssetPathBootstrapEtcd               = "bootstrap-manifests/bootstrap-etcd.yaml"
+	AssetPathBootstrapEtcdService        = "etcd/bootstrap-etcd-service.json"
+	AssetPathMigrateEtcdCluster          = "etcd/migrate-etcd-cluster.json"
 )
 
 var (
@@ -76,6 +78,7 @@ type Config struct {
 	PodCIDR                *net.IPNet
 	ServiceCIDR            *net.IPNet
 	APIServiceIP           net.IP
+	BootEtcdServiceIP      net.IP
 	DNSServiceIP           net.IP
 	EtcdServiceIP          net.IP
 	EtcdServiceName        string

pkg/asset/internal/templates.go

@@ -915,21 +915,71 @@ spec:
     - --name=boot-etcd
     - --listen-client-urls=http://0.0.0.0:12379
     - --listen-peer-urls=http://0.0.0.0:12380
-    - --advertise-client-urls=http://$(MY_POD_IP):12379
-    - --initial-advertise-peer-urls=http://$(MY_POD_IP):12380
-    - --initial-cluster=boot-etcd=http://$(MY_POD_IP):12380
+    - --advertise-client-urls=http://{{ .BootEtcdServiceIP }}:12379
+    - --initial-advertise-peer-urls=http://{{ .BootEtcdServiceIP }}:12380
+    - --initial-cluster=boot-etcd=http://{{ .BootEtcdServiceIP }}:12380
     - --initial-cluster-token=bootkube
     - --initial-cluster-state=new
     - --data-dir=/var/etcd/data
-    env:
-      - name: MY_POD_IP
-        valueFrom:
-          fieldRef:
-            fieldPath: status.podIP
   hostNetwork: true
   restartPolicy: Never
 `)
 
+	BootstrapEtcdSvcTemplate = []byte(`{
+  "apiVersion": "v1",
+  "kind": "Service",
+  "metadata": {
+    "name": "bootstrap-etcd-service",
+    "namespace": "kube-system"
+  },
+  "spec": {
+    "selector": {
+      "k8s-app": "boot-etcd"
+    },
+    "clusterIP": "{{ .BootEtcdServiceIP }}",
+    "ports": [
+      {
+        "name": "client",
+        "port": 12379,
+        "protocol": "TCP"
+      },
+      {
+        "name": "peers",
+        "port": 12380,
+        "protocol": "TCP"
+      }
+    ]
+  }
+}`)
+
+	EtcdTPRTemplate = []byte(`{
+  "apiVersion": "etcd.coreos.com/v1beta1",
+  "kind": "Cluster",
+  "metadata": {
+    "name": "kube-etcd",
+    "namespace": "kube-system"
+  },
+  "spec": {
+    "size": 1,
+    "version": "v3.1.6",
+    "pod": {
+      "nodeSelector": {
+        "node-role.kubernetes.io/master": ""
+      },
+      "tolerations": [
+        {
+          "key": "node-role.kubernetes.io/master",
+          "operator": "Exists",
+          "effect": "NoSchedule"
+        }
+      ]
+    },
+    "selfHosted": {
+      "bootMemberClientEndpoint": "http://{{ .BootEtcdServiceIP }}:12379"
+    }
+  }
+}`)
+
 	KubeFlannelCfgTemplate = []byte(`apiVersion: v1
 kind: ConfigMap
 metadata:

pkg/asset/k8s.go

@@ -59,7 +59,8 @@ func newDynamicAssets(conf Config) Assets {
 			mustCreateAssetFromTemplate(AssetPathEtcdSvc, internal.EtcdSvcTemplate, conf),
 			mustCreateAssetFromTemplate(AssetPathKenc, internal.KencTemplate, conf),
 			mustCreateAssetFromTemplate(AssetPathBootstrapEtcd, internal.BootstrapEtcdTemplate, conf),
-		)
+			mustCreateAssetFromTemplate(AssetPathBootstrapEtcdService, internal.BootstrapEtcdSvcTemplate, conf),
+			mustCreateAssetFromTemplate(AssetPathMigrateEtcdCluster, internal.EtcdTPRTemplate, conf))
 	}
 	return assets
 }

pkg/bootkube/bootkube.go

@@ -87,7 +87,7 @@ func (b *bootkube) Run() error {
 
 	if selfHostedEtcd {
 		UserOutput("Migrating to self-hosted etcd cluster...\n")
-		if err = etcdutil.Migrate(kubeConfig); err != nil {
+		if err = etcdutil.Migrate(kubeConfig, filepath.Join(b.assetDir, asset.AssetPathBootstrapEtcdService), filepath.Join(b.assetDir, asset.AssetPathMigrateEtcdCluster)); err != nil {
 			return err
 		}
 	}