Fix apiserver etcd-servers flag when self-hosting.

This adds the self-hosted etcd cluster to the temporary control plane
config, which should hopefully mitigate situations where the control
plane loses contact with etcd during the boot-etcd -> self-hosted-etcd
pivot and gives up leader status as a result.

I also fixed the hack/single-node config to not start local etcd
when running with self-hosting.

Author: Diego Pontoriero

hack/single-node/bootkube-up

@@ -23,6 +23,9 @@ fi
 if [ ! -d "cluster" ]; then
   ../../_output/bin/${local_os}/bootkube render --asset-dir=cluster --api-servers=https://172.17.4.100:443 ${etcd_render_flags}
   cp user-data.sample cluster/user-data
+	if [ ${SELF_HOST_ETCD} = "false" ]; then
+		cat user-data-etcd.sample >> cluster/user-data
+	fi
 fi
 
 # Start the VM

hack/single-node/user-data-etcd.sample

@@ -0,0 +1,8 @@
+    - name: etcd-member.service
+      enable: true
+      drop-ins:
+        - name: 10-version.conf
+          content: |
+            [Service]
+            Environment="ETCD_IMAGE_TAG=v3.1.0"
+      command: start

hack/single-node/user-data.sample

@@ -2,14 +2,6 @@
 
 coreos:
   units:
-    - name: etcd-member.service
-      enable: true
-      drop-ins:
-        - name: 10-version.conf
-          content: |
-            [Service]
-            Environment="ETCD_IMAGE_TAG=v3.1.0"
-      command: start
     - name: kubelet.service
       enable: true
       command: start

pkg/bootkube/bootkube.go

@@ -85,6 +85,16 @@ func makeAPIServerFlags(config Config) ([]string, error) {
 	if err != nil {
 		return []string{}, err
 	}
+	etcdServers := config.EtcdServer.String()
+	if config.SelfHostedEtcd {
+		// When self-hosting etcd we also point to the (not yet started) permanent cluster since the
+		// bootstrap node will go away at some point.
+		etcdServiceURL, err := detectEtcdServiceURL(config.AssetDir)
+		if err != nil {
+			return nil, err
+		}
+		etcdServers += fmt.Sprintf(",%s", etcdServiceURL)
+	}
 	return []string{
 		"--bind-address=0.0.0.0",
 		"--secure-port=443",
@@ -96,7 +106,7 @@ func makeAPIServerFlags(config Config) ([]string, error) {
 		"--kubelet-client-certificate=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerCert),
 		"--client-ca-file=" + filepath.Join(config.AssetDir, asset.AssetPathCACert),
 		"--authorization-mode=RBAC",
-		"--etcd-servers=" + config.EtcdServer.String(),
+		"--etcd-servers=" + etcdServers,
 		"--service-cluster-ip-range=" + serviceCIDR,
 		"--service-account-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathServiceAccountPubKey),
 		"--admission-control=NamespaceLifecycle,ServiceAccount",

pkg/bootkube/parse.go

@@ -91,6 +91,24 @@ func detectEtcdIP(assetDir string) (string, error) {
 	return service.Spec.ClusterIP, nil
 }
 
+// detectEtcdServiceURL deserializes the etcd-service URL.
+func detectEtcdServiceURL(assetDir string) (string, error) {
+	path := filepath.Join(assetDir, asset.AssetPathEtcdSvc)
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("can't read file %s: %v", path, err)
+	}
+	var service v1.Service
+	err = yaml.Unmarshal(b, &service)
+	if err != nil {
+		return "", fmt.Errorf("can't unmarshal %s: %v", path, err)
+	}
+	if numPorts := len(service.Spec.Ports); numPorts != 1 {
+		return "", fmt.Errorf("expected 1 etcd cluster port, found: %d", numPorts)
+	}
+	return fmt.Sprintf("http://%s:%d", service.Spec.ClusterIP, service.Spec.Ports[0]), nil
+}
+
 func findFlag(flagName string, args []string) string {
 	for _, arg := range args {
 		if strings.HasPrefix(arg, flagName+"=") {