Merge pull request #318 from dghubble/cidr-customize

pkg,cmd: Add Kubernetes pod and service CIDR customziation

Author: aaronlevy

cmd/bootkube/render.go

@@ -16,6 +16,13 @@ import (
 	"github.com/kubernetes-incubator/bootkube/pkg/tlsutil"
 )
 
+const (
+	apiOffset            = 1
+	dnsOffset            = 10
+	etcdOffset           = 15
+	defaultServiceBaseIP = "10.3.0.0"
+)
+
 var (
 	cmdRender = &cobra.Command{
 		Use:          "render",
@@ -33,6 +40,8 @@ var (
 		etcdServers       string
 		apiServers        string
 		altNames          string
+		podCIDR           string
+		serviceCIDR       string
 		selfHostKubelet   bool
 		cloudProvider     string
 		selfHostedEtcd    bool
@@ -47,6 +56,8 @@ func init() {
 	cmdRender.Flags().StringVar(&renderOpts.etcdServers, "etcd-servers", "http://127.0.0.1:2379", "List of etcd servers URLs including host:port, comma separated")
 	cmdRender.Flags().StringVar(&renderOpts.apiServers, "api-servers", "https://127.0.0.1:443", "List of API server URLs including host:port, commma seprated")
 	cmdRender.Flags().StringVar(&renderOpts.altNames, "api-server-alt-names", "", "List of SANs to use in api-server certificate. Example: 'IP=127.0.0.1,IP=127.0.0.2,DNS=localhost'. If empty, SANs will be extracted from the --api-servers flag.")
+	cmdRender.Flags().StringVar(&renderOpts.podCIDR, "pod-cidr", "10.2.0.0/16", "The CIDR range of cluster pods.")
+	cmdRender.Flags().StringVar(&renderOpts.serviceCIDR, "service-cidr", "10.3.0.0/24", "The CIDR range of cluster services.")
 	cmdRender.Flags().BoolVar(&renderOpts.selfHostKubelet, "self-host-kubelet", false, "Create a self-hosted kubelet daemonset.")
 	cmdRender.Flags().StringVar(&renderOpts.cloudProvider, "cloud-provider", "", "The provider for cloud services.  Empty string for no provider")
 	cmdRender.Flags().BoolVar(&renderOpts.selfHostedEtcd, "experimental-self-hosted-etcd", false, "Create self-hosted etcd assets.")
@@ -111,12 +122,52 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
 			return nil, err
 		}
 	}
+
+	_, podNet, err := net.ParseCIDR(renderOpts.podCIDR)
+	if err != nil {
+		return nil, err
+	}
+
+	_, serviceNet, err := net.ParseCIDR(renderOpts.serviceCIDR)
+	if err != nil {
+		return nil, err
+	}
+
+	if podNet.Contains(serviceNet.IP) || serviceNet.Contains(podNet.IP) {
+		return nil, fmt.Errorf("Pod CIDR %s and service CIDR %s must not overlap", podNet.String(), serviceNet.String())
+	}
+
+	apiServiceIP, err := offsetServiceIP(serviceNet, apiOffset)
+	if err != nil {
+		return nil, err
+	}
+
+	dnsServiceIP, err := offsetServiceIP(serviceNet, dnsOffset)
+	if err != nil {
+		return nil, err
+	}
+
+	etcdServiceIP, err := offsetServiceIP(serviceNet, etcdOffset)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: Find better option than asking users to make manual changes
+	if serviceNet.IP.String() != defaultServiceBaseIP {
+		fmt.Printf("You have selected a non-default service CIDR %s - be sure your kubelet service file uses --cluster-dns=%s\n", serviceNet.String(), dnsServiceIP.String())
+	}
+
 	return &asset.Config{
 		EtcdServers:     etcdServers,
 		CACert:          caCert,
 		CAPrivKey:       caPrivKey,
 		APIServers:      apiServers,
 		AltNames:        altNames,
+		PodCIDR:         podNet,
+		ServiceCIDR:     serviceNet,
+		APIServiceIP:    apiServiceIP,
+		DNSServiceIP:    dnsServiceIP,
+		ETCDServiceIP:   etcdServiceIP,
 		SelfHostKubelet: renderOpts.selfHostKubelet,
 		CloudProvider:   renderOpts.cloudProvider,
 		SelfHostedEtcd:  renderOpts.selfHostedEtcd,
@@ -195,3 +246,26 @@ func altNamesFromURLs(urls []*url.URL) *tlsutil.AltNames {
 	}
 	return &an
 }
+
+// offsetServiceIP returns an IP offset by up to 255.
+// TODO: do numeric conversion to generalize this utility.
+func offsetServiceIP(ipnet *net.IPNet, offset int) (net.IP, error) {
+	ip := make(net.IP, len(ipnet.IP))
+	copy(ip, ipnet.IP)
+	for i := 0; i < offset; i++ {
+		incIPv4(ip)
+	}
+	if ipnet.Contains(ip) {
+		return ip, nil
+	}
+	return net.IP([]byte("")), fmt.Errorf("Service IP %v is not in %s", ip, ipnet)
+}
+
+func incIPv4(ip net.IP) {
+	for j := len(ip) - 1; j >= 0; j-- {
+		ip[j]++
+		if ip[j] > 0 {
+			break
+		}
+	}
+}

cmd/bootkube/render_test.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"net"
+	"testing"
+)
+
+func TestOffsetIP(t *testing.T) {
+	cases := []struct {
+		input    string
+		offset   int
+		expected string
+	}{
+		{"10.3.0.0/24", 1, "10.3.0.1"},
+		{"10.3.0.0/24", 10, "10.3.0.10"},
+		{"10.3.0.0/24", 15, "10.3.0.15"},
+		{"10.3.0.0/16", 1, "10.3.0.1"},
+		{"10.3.0.0/16", 10, "10.3.0.10"},
+		{"10.3.0.0/16", 15, "10.3.0.15"},
+		{"10.33.1.200/16", 1, "10.33.0.1"},
+		{"10.33.1.200/16", 10, "10.33.0.10"},
+		{"10.33.1.200/16", 15, "10.33.0.15"},
+		{"192.168.1.0/24", 15, "192.168.1.15"},
+	}
+
+	for _, c := range cases {
+		_, cidr, err := net.ParseCIDR(c.input)
+		if err != nil {
+			t.Errorf("unexpected CIDR parse error: %v", err)
+		}
+		ip, err := offsetServiceIP(cidr, c.offset)
+		if ip.String() != c.expected {
+			t.Errorf("expected %s, got %s", c.expected, ip.String())
+		}
+	}
+}

hack/multi-node/Vagrantfile

@@ -18,7 +18,6 @@ if $worker_vm_memory < 1024
   puts "Workers should have at least 1024 MB of memory"
 end
 
-CONTROLLER_CLUSTER_IP="10.3.0.1"
 CONTROLLER_USER_DATA_PATH = File.expand_path("./cluster/user-data-controller")
 WORKER_USER_DATA_PATH = File.expand_path("./cluster/user-data-worker")
 

pkg/asset/asset.go

@@ -5,6 +5,7 @@ import (
 	"crypto/x509"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -50,6 +51,11 @@ type Config struct {
 	CACert          *x509.Certificate
 	CAPrivKey       *rsa.PrivateKey
 	AltNames        *tlsutil.AltNames
+	PodCIDR         *net.IPNet
+	ServiceCIDR     *net.IPNet
+	APIServiceIP    net.IP
+	DNSServiceIP    net.IP
+	ETCDServiceIP   net.IP
 	SelfHostKubelet bool
 	SelfHostedEtcd  bool
 	CloudProvider   string
@@ -59,9 +65,12 @@ type Config struct {
 // configured via a user provided AssetConfig. Default assets include
 // TLS assets (certs, keys and secrets), and k8s component manifests.
 func NewDefaultAssets(conf Config) (Assets, error) {
-	as := newStaticAssets(conf.SelfHostKubelet, conf.SelfHostedEtcd)
+	as := newStaticAssets()
 	as = append(as, newDynamicAssets(conf)...)
 
+	// Add kube-apiserver service IP
+	conf.AltNames.IPs = append(conf.AltNames.IPs, conf.APIServiceIP)
+
 	// TLS assets
 	tlsAssets, err := newTLSAssets(conf.CACert, conf.CAPrivKey, *conf.AltNames)
 	if err != nil {

pkg/asset/internal/templates.go

@@ -45,7 +45,7 @@ spec:
         - --pod-manifest-path=/etc/kubernetes/manifests
         - --allow-privileged
         - --hostname-override=$(NODE_NAME)
-        - --cluster-dns=10.3.0.10
+        - --cluster-dns={{ .DNSServiceIP }}
         - --cluster-domain=cluster.local
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --require-kubeconfig
@@ -147,7 +147,7 @@ spec:
         - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}
         - --storage-backend=etcd3
         - --allow-privileged=true
-        - --service-cluster-ip-range=10.3.0.0/24
+        - --service-cluster-ip-range={{ .ServiceCIDR }}
         - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota
         - --runtime-config=api/all=true
         - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
@@ -235,7 +235,7 @@ spec:
         - controller-manager
         - --allocate-node-cidrs=true
         - --configure-cloud-routes=false
-        - --cluster-cidr=10.2.0.0/16
+        - --cluster-cidr={{ .PodCIDR }}
         - --root-ca-file=/etc/kubernetes/secrets/ca.crt
         - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
         - --leader-elect=true
@@ -326,7 +326,7 @@ spec:
         - --kubeconfig=/etc/kubernetes/kubeconfig
         - --proxy-mode=iptables
         - --hostname-override=$(NODE_NAME)
-        - --cluster-cidr=10.2.0.0/16
+        - --cluster-cidr={{ .PodCIDR }}
         env:
           - name: NODE_NAME
             valueFrom:
@@ -513,7 +513,7 @@ metadata:
 spec:
   selector:
     k8s-app: kube-dns
-  clusterIP: 10.3.0.10
+  clusterIP: {{ .DNSServiceIP }}
   ports:
   - name: dns
     port: 53
@@ -556,7 +556,7 @@ spec:
   selector:
     app: etcd
     etcd_cluster: kube-etcd
-  clusterIP: 10.3.0.15
+  clusterIP: {{ .ETCDServiceIP }}
   ports:
   - name: client
     port: 2379
@@ -582,7 +582,7 @@ data:
     }
   net-conf.json: |
     {
-      "Network": "10.2.0.0/16",
+      "Network": "{{ .PodCIDR }}",
       "Backend": {
         "Type": "vxlan"
       }

pkg/asset/k8s.go

@@ -17,37 +17,37 @@ const (
 	secretCMName        = "kube-controller-manager"
 )
 
-func newStaticAssets(selfHostKubelet, selfHostedEtcd bool) Assets {
+func newStaticAssets() Assets {
 	var noData interface{}
 	assets := Assets{
 		mustCreateAssetFromTemplate(AssetPathScheduler, internal.SchedulerTemplate, noData),
 		mustCreateAssetFromTemplate(AssetPathSchedulerDisruption, internal.SchedulerDisruptionTemplate, noData),
 		mustCreateAssetFromTemplate(AssetPathControllerManagerDisruption, internal.ControllerManagerDisruptionTemplate, noData),
-		mustCreateAssetFromTemplate(AssetPathProxy, internal.ProxyTemplate, noData),
 		mustCreateAssetFromTemplate(AssetPathKubeDNSDeployment, internal.DNSDeploymentTemplate, noData),
-		mustCreateAssetFromTemplate(AssetPathKubeDNSSvc, internal.DNSSvcTemplate, noData),
 		mustCreateAssetFromTemplate(AssetPathCheckpointer, internal.CheckpointerTemplate, noData),
 		mustCreateAssetFromTemplate(AssetPathKubeFlannel, internal.KubeFlannelTemplate, noData),
-		mustCreateAssetFromTemplate(AssetPathKubeFlannelCfg, internal.KubeFlannelCfgTemplate, noData),
 	}
-	if selfHostKubelet {
-		assets = append(assets, mustCreateAssetFromTemplate(AssetPathKubelet, internal.KubeletTemplate, noData))
-	}
-	if selfHostedEtcd {
-		assets = append(assets,
-			mustCreateAssetFromTemplate(AssetPathEtcdOperator, internal.EtcdOperatorTemplate, noData),
-			mustCreateAssetFromTemplate(AssetPathEtcdSvc, internal.EtcdSvcTemplate, noData),
-		)
-	}
-
 	return assets
 }
 
 func newDynamicAssets(conf Config) Assets {
-	return Assets{
+	assets := Assets{
 		mustCreateAssetFromTemplate(AssetPathControllerManager, internal.ControllerManagerTemplate, conf),
 		mustCreateAssetFromTemplate(AssetPathAPIServer, internal.APIServerTemplate, conf),
+		mustCreateAssetFromTemplate(AssetPathProxy, internal.ProxyTemplate, conf),
+		mustCreateAssetFromTemplate(AssetPathKubeFlannelCfg, internal.KubeFlannelCfgTemplate, conf),
+		mustCreateAssetFromTemplate(AssetPathKubeDNSSvc, internal.DNSSvcTemplate, conf),
 	}
+	if conf.SelfHostKubelet {
+		assets = append(assets, mustCreateAssetFromTemplate(AssetPathKubelet, internal.KubeletTemplate, conf))
+	}
+	if conf.SelfHostedEtcd {
+		assets = append(assets,
+			mustCreateAssetFromTemplate(AssetPathEtcdOperator, internal.EtcdOperatorTemplate, conf),
+			mustCreateAssetFromTemplate(AssetPathEtcdSvc, internal.EtcdSvcTemplate, conf),
+		)
+	}
+	return assets
 }
 
 func newKubeConfigAsset(assets Assets, conf Config) (Asset, error) {

pkg/asset/tls.go

@@ -3,7 +3,6 @@ package asset
 import (
 	"crypto/rsa"
 	"crypto/x509"
-	"net"
 
 	"github.com/kubernetes-incubator/bootkube/pkg/tlsutil"
 )
@@ -78,7 +77,6 @@ func newAPIKeyAndCert(caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, altNa
 	if err != nil {
 		return nil, nil, err
 	}
-	altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1"))
 	altNames.DNSNames = append(altNames.DNSNames, []string{
 		"kubernetes",
 		"kubernetes.default",

pkg/bootkube/bootkube.go

@@ -15,6 +15,7 @@ import (
 	scheduler "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options"
 
 	"github.com/kubernetes-incubator/bootkube/pkg/asset"
+	"github.com/kubernetes-incubator/bootkube/pkg/util/etcdutil"
 )
 
 const (
@@ -129,11 +130,23 @@ func (b *bootkube) Run() error {
 			errch <- err
 		}
 	}()
-	if b.selfHostedEtcd {
-		requiredPods = append(requiredPods, "etcd-operator")
-	}
-	go func() { errch <- WaitUntilPodsRunning(requiredPods, assetTimeout, b.selfHostedEtcd) }()
-
+	go func() {
+		if b.selfHostedEtcd {
+			requiredPods = append(requiredPods, "etcd-operator")
+			etcdServiceIP, err := detectEtcdIP(b.assetDir)
+			if err != nil {
+				errch <- err
+				return
+			}
+			if err := WaitUntilPodsRunning(requiredPods, assetTimeout); err != nil {
+				errch <- err
+				return
+			}
+			errch <- etcdutil.Migrate(etcdServiceIP)
+		} else {
+			errch <- WaitUntilPodsRunning(requiredPods, assetTimeout)
+		}
+	}()
 	// If any of the bootkube services exit, it means it is unrecoverable and we should exit.
 	err := <-errch
 	if err != nil {

pkg/bootkube/parse.go

@@ -74,6 +74,20 @@ func detectPodCIDR(config Config) (string, error) {
 	return "", fmt.Errorf("can't detect --cluster-cidr flag in %s", asset.AssetPathControllerManager)
 }
 
+// detectEtcdIP deserializes the etcd-service ClusterIP.
+func detectEtcdIP(assetDir string) (string, error) {
+	b, err := ioutil.ReadFile(filepath.Join(assetDir, asset.AssetPathEtcdSvc))
+	if err != nil {
+		return "", fmt.Errorf("can't read file %s: %v", asset.AssetPathEtcdSvc, err)
+	}
+	var service v1.Service
+	err = yaml.Unmarshal(b, &service)
+	if err != nil {
+		return "", fmt.Errorf("can't unmarshal %s: %v", asset.AssetPathEtcdSvc, err)
+	}
+	return service.Spec.ClusterIP, nil
+}
+
 func findFlag(flagName string, args []string) string {
 	for _, arg := range args {
 		if strings.HasPrefix(arg, flagName+"=") {