pkg/bootkube: Detect Etcd service IP from manifest

* Etcd Service IP may vary depending on the service
CIDR, so bootkube start with self-hosted etcd should
detect it from manifests

Author: dghubble

cmd/bootkube/render.go

@@ -17,10 +17,10 @@ import (
 )
 
 const (
-	apiOffset           = 1
-	dnsOffset           = 10
-	etcdOffset          = 15
-	defaultDNSServiceIP = "10.3.0.10"
+	apiOffset            = 1
+	dnsOffset            = 10
+	etcdOffset           = 15
+	defaultServiceBaseIP = "10.3.0.0"
 )
 
 var (
@@ -152,7 +152,8 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 		return nil, err
 	}
 
-	if dnsServiceIP.String() != defaultDNSServiceIP {
+	// TODO: Find better option than asking users to make manual changes
+	if serviceNet.IP.String() != defaultServiceBaseIP {
 		fmt.Printf("You have selected a non-default service CIDR %s - be sure your kubelet service file uses --cluster-dns=%s\n", serviceNet.String(), dnsServiceIP.String())
 	}
 

cmd/bootkube/render_test.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"net"
+	"testing"
+)
+
+func TestOffsetIP(t *testing.T) {
+	cases := []struct {
+		input    string
+		offset   int
+		expected string
+	}{
+		{"10.3.0.0/24", 1, "10.3.0.1"},
+		{"10.3.0.0/24", 10, "10.3.0.10"},
+		{"10.3.0.0/24", 15, "10.3.0.15"},
+		{"10.3.0.0/16", 1, "10.3.0.1"},
+		{"10.3.0.0/16", 10, "10.3.0.10"},
+		{"10.3.0.0/16", 15, "10.3.0.15"},
+		{"10.33.1.200/16", 1, "10.33.0.1"},
+		{"10.33.1.200/16", 10, "10.33.0.10"},
+		{"10.33.1.200/16", 15, "10.33.0.15"},
+		{"192.168.1.0/24", 15, "192.168.1.15"},
+	}
+
+	for _, c := range cases {
+		_, cidr, err := net.ParseCIDR(c.input)
+		if err != nil {
+			t.Errorf("unexpected CIDR parse error: %v", err)
+		}
+		ip, err := offsetServiceIP(cidr, c.offset)
+		if ip.String() != c.expected {
+			t.Errorf("expected %s, got %s", c.expected, ip.String())
+		}
+	}
+}

pkg/bootkube/bootkube.go

@@ -15,6 +15,7 @@ import (
 	scheduler "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options"
 
 	"github.com/kubernetes-incubator/bootkube/pkg/asset"
+	"github.com/kubernetes-incubator/bootkube/pkg/util/etcdutil"
 )
 
 const (
@@ -129,11 +130,23 @@ func (b *bootkube) Run() error {
 			errch <- err
 		}
 	}()
-	if b.selfHostedEtcd {
-		requiredPods = append(requiredPods, "etcd-operator")
-	}
-	go func() { errch <- WaitUntilPodsRunning(requiredPods, assetTimeout, b.selfHostedEtcd) }()
-
+	go func() {
+		if b.selfHostedEtcd {
+			requiredPods = append(requiredPods, "etcd-operator")
+			etcdServiceIP, err := detectEtcdIP(b.assetDir)
+			if err != nil {
+				errch <- err
+				return
+			}
+			if err := WaitUntilPodsRunning(requiredPods, assetTimeout); err != nil {
+				errch <- err
+				return
+			}
+			errch <- etcdutil.Migrate(etcdServiceIP)
+		} else {
+			errch <- WaitUntilPodsRunning(requiredPods, assetTimeout)
+		}
+	}()
 	// If any of the bootkube services exit, it means it is unrecoverable and we should exit.
 	err := <-errch
 	if err != nil {

pkg/bootkube/parse.go

@@ -74,6 +74,20 @@ func detectPodCIDR(config Config) (string, error) {
 	return "", fmt.Errorf("can't detect --cluster-cidr flag in %s", asset.AssetPathControllerManager)
 }
 
+// detectEtcdIP deserializes the etcd-service ClusterIP.
+func detectEtcdIP(assetDir string) (string, error) {
+	b, err := ioutil.ReadFile(filepath.Join(assetDir, asset.AssetPathEtcdSvc))
+	if err != nil {
+		return "", fmt.Errorf("can't read file %s: %v", asset.AssetPathEtcdSvc, err)
+	}
+	var service v1.Service
+	err = yaml.Unmarshal(b, &service)
+	if err != nil {
+		return "", fmt.Errorf("can't unmarshal %s: %v", asset.AssetPathEtcdSvc, err)
+	}
+	return service.Spec.ClusterIP, nil
+}
+
 func findFlag(flagName string, args []string) string {
 	for _, arg := range args {
 		if strings.HasPrefix(arg, flagName+"=") {

pkg/bootkube/status.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/golang/glog"
-	"github.com/kubernetes-incubator/bootkube/pkg/util/etcdutil"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/cache"
@@ -25,7 +24,7 @@ const (
 	doesNotExist = "DoesNotExist"
 )
 
-func WaitUntilPodsRunning(pods []string, timeout time.Duration, selfHostedEtcd bool) error {
+func WaitUntilPodsRunning(pods []string, timeout time.Duration) error {
 	sc, err := NewStatusController(pods)
 	if err != nil {
 		return err
@@ -36,12 +35,6 @@ func WaitUntilPodsRunning(pods []string, timeout time.Duration, selfHostedEtcd b
 		return fmt.Errorf("error while checking pod status: %v", err)
 	}
 
-	if selfHostedEtcd {
-		if err := etcdutil.Migrate(); err != nil {
-			return err
-		}
-	}
-
 	UserOutput("All self-hosted control plane components successfully started\n")
 	return nil
 }

pkg/util/etcdutil/migrate.go

@@ -20,9 +20,7 @@ import (
 )
 
 const (
-	apiserverAddr = "http://127.0.0.1:8080"
-	etcdServiceIP = "10.3.0.15"
-
+	apiserverAddr   = "http://127.0.0.1:8080"
 	etcdClusterName = "kube-etcd"
 )
 
@@ -31,7 +29,7 @@ var (
 	waitBootEtcdRemovedTime    = 300 * time.Second
 )
 
-func Migrate() error {
+func Migrate(etcdServiceIP string) error {
 	kubecli, err := clientset.NewForConfig(&restclient.Config{
 		Host: apiserverAddr,
 	})
@@ -62,7 +60,7 @@ func Migrate() error {
 	}
 	glog.Info("etcd cluster for migration is now running")
 
-	if err := waitBootEtcdRemoved(); err != nil {
+	if err := waitBootEtcdRemoved(etcdServiceIP); err != nil {
 		return fmt.Errorf("wait boot etcd deleted failed: %v", err)
 	}
 	glog.Info("the boot etcd is removed from the migration cluster")
@@ -169,7 +167,7 @@ func waitEtcdClusterRunning(restclient restclient.Interface) error {
 	return err
 }
 
-func waitBootEtcdRemoved() error {
+func waitBootEtcdRemoved(etcdServiceIP string) error {
 	err := wait.Poll(10*time.Second, waitBootEtcdRemovedTime, func() (bool, error) {
 		cfg := clientv3.Config{
 			Endpoints:   []string{fmt.Sprintf("http://%s:2379", etcdServiceIP)},