Use static assets for temporary control plane.

This change:
- Starts the bootstrap control plane using static manifests instead of
  launching them in-process.
- Auto-detects whether self-hosted etcd is being used in the run phase,
  rather than requiring synchronized flag usage.
- Simplifies a lot of code based on the above refactorings.

After this change clients can completely customize their bootstrap
environment, including changing the version of kubernetes that is
launched.

Author: Diego Pontoriero

cmd/bootkube/start.go

@@ -3,14 +3,11 @@ package main
 import (
 	"errors"
 	"flag"
-	"fmt"
-	"net/url"
 
 	"github.com/spf13/cobra"
 
 	"github.com/kubernetes-incubator/bootkube/pkg/bootkube"
 	"github.com/kubernetes-incubator/bootkube/pkg/util"
-	"github.com/kubernetes-incubator/bootkube/pkg/util/etcdutil"
 )
 
 var (
@@ -24,36 +21,21 @@ var (
 	}
 
 	startOpts struct {
-		assetDir       string
-		etcdServer     string
-		selfHostedEtcd bool
+		assetDir        string
+		podManifestPath string
 	}
 )
 
 func init() {
 	cmdRoot.AddCommand(cmdStart)
-	cmdStart.Flags().StringVar(&startOpts.etcdServer, "etcd-server", "http://127.0.0.1:2379", "Single etcd node to use during bootkube bootstrap process.")
 	cmdStart.Flags().StringVar(&startOpts.assetDir, "asset-dir", "", "Path to the cluster asset directory. Expected layout genereted by the `bootkube render` command.")
-	cmdStart.Flags().BoolVar(&startOpts.selfHostedEtcd, "experimental-self-hosted-etcd", false, "Self hosted etcd mode. Includes starting the initial etcd member by bootkube.")
+	cmdStart.Flags().StringVar(&startOpts.podManifestPath, "pod-manifest-path", "/etc/kubernetes/manifests", "The location where the kubelet is configured to look for static pod manifests.")
 }
 
 func runCmdStart(cmd *cobra.Command, args []string) error {
-	etcdServer, err := url.Parse(startOpts.etcdServer)
-	if err != nil {
-		return fmt.Errorf("Invalid etcd etcdServer %q: %v", startOpts.etcdServer, err)
-	}
-
-	// TODO: this should likely move into bootkube.Run() eventually.
-	if startOpts.selfHostedEtcd {
-		if err := etcdutil.StartEtcd(startOpts.etcdServer); err != nil {
-			return fmt.Errorf("fail to start etcd: %v", err)
-		}
-	}
-
 	bk, err := bootkube.NewBootkube(bootkube.Config{
-		AssetDir:       startOpts.assetDir,
-		EtcdServer:     etcdServer,
-		SelfHostedEtcd: startOpts.selfHostedEtcd,
+		AssetDir:        startOpts.assetDir,
+		PodManifestPath: startOpts.podManifestPath,
 	})
 
 	if err != nil {
@@ -69,8 +51,8 @@ func runCmdStart(cmd *cobra.Command, args []string) error {
 }
 
 func validateStartOpts(cmd *cobra.Command, args []string) error {
-	if startOpts.etcdServer == "" {
-		return errors.New("missing required flag: --etcd-server")
+	if startOpts.podManifestPath == "" {
+		return errors.New("missing required flag: --pod-manifest-path")
 	}
 	if startOpts.assetDir == "" {
 		return errors.New("missing required flag: --asset-dir")

hack/multi-node/bootkube-up

@@ -12,11 +12,9 @@ fi
 SELF_HOST_ETCD=${SELF_HOST_ETCD:-false}
 if [ ${SELF_HOST_ETCD} = "true" ]; then
     echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
-    etcd_render_flags="--etcd-servers=http://10.3.0.15:2379 --experimental-self-hosted-etcd"
-    etcd_start_flags="--etcd-server=http://172.17.4.101:12379 --experimental-self-hosted-etcd"
+    etcd_render_flags="--experimental-self-hosted-etcd"
 else
     etcd_render_flags="--etcd-servers=http://172.17.4.51:2379"
-    etcd_start_flags="--etcd-server=http://172.17.4.51:2379"
 fi
 
 # Render assets
@@ -36,7 +34,7 @@ scp -q -F ssh_config -r cluster core@c1:/home/core/cluster
 scp -q -F ssh_config ../../_output/bin/linux/bootkube core@c1:/home/core
 
 # Run bootkube
-ssh -q -F ssh_config core@c1 "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/cluster ${etcd_start_flags} 2>> /home/core/bootkube.log"
+ssh -q -F ssh_config core@c1 "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/cluster 2>> /home/core/bootkube.log"
 
 echo
 echo "Bootstrap complete. Access your kubernetes cluster using:"

hack/quickstart/init-master.sh

@@ -36,13 +36,11 @@ function init_master_node() {
     systemctl stop update-engine; systemctl mask update-engine
 
     etcd_render_flags=""
-    etcd_start_flags=""
 
     # Start etcd.
     if [ "$SELF_HOST_ETCD" = true ] ; then
         echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
-        etcd_render_flags="--etcd-servers=http://10.3.0.15:2379 --experimental-self-hosted-etcd"
-        etcd_start_flags="--etcd-server=http://${COREOS_PRIVATE_IPV4}:12379 --experimental-self-hosted-etcd"
+        etcd_render_flags="--experimental-self-hosted-etcd"
     else
         configure_etcd
         systemctl enable etcd-member; sudo systemctl start etcd-member
@@ -61,7 +59,7 @@ function init_master_node() {
     systemctl enable kubelet; sudo systemctl start kubelet
 
     # Start bootkube to launch a self-hosted cluster
-    /home/core/bootkube start --asset-dir=/home/core/assets ${etcd_start_flags}
+    /home/core/bootkube start --asset-dir=/home/core/assets
 }
 
 [ "$#" == 1 ] || usage

hack/single-node/bootkube-up

@@ -13,10 +13,8 @@ SELF_HOST_ETCD=${SELF_HOST_ETCD:-false}
 if [ ${SELF_HOST_ETCD} = "true" ]; then
     echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
     etcd_render_flags="--experimental-self-hosted-etcd"
-    etcd_start_flags="--etcd-server=http://172.17.4.100:12379 --experimental-self-hosted-etcd"
 else
     etcd_render_flags=""
-    etcd_start_flags=""
 fi
 
 # Render assets
@@ -37,7 +35,7 @@ scp -q -F ssh_config -r cluster core@default:/home/core/cluster
 scp -q -F ssh_config ../../_output/bin/linux/bootkube core@default:/home/core
 
 # Run bootkube
-ssh -q -F ssh_config core@default "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/cluster ${etcd_start_flags} 2>> /home/core/bootkube.log"
+ssh -q -F ssh_config core@default "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/cluster 2>> /home/core/bootkube.log"
 
 echo
 echo "Bootstrap complete. Access your kubernetes cluster using:"

pkg/asset/asset.go

@@ -14,6 +14,7 @@ import (
 )
 
 const (
+	AssetPathSecrets                     = "tls"
 	AssetPathCAKey                       = "tls/ca.key"
 	AssetPathCACert                      = "tls/ca.crt"
 	AssetPathAPIServerKey                = "tls/apiserver.key"
@@ -43,30 +44,39 @@ const (
 	AssetPathEtcdSvc                     = "manifests/etcd-service.yaml"
 	AssetPathKenc                        = "manifests/kenc.yaml"
 	AssetPathKubeSystemSARoleBinding     = "manifests/kube-system-rbac-role-binding.yaml"
+	AssetPathBootstrapManifests          = "bootstrap-manifests"
+	AssetPathBootstrapAPIServer          = "bootstrap-manifests/bootstrap-apiserver.yaml"
+	AssetPathBootstrapControllerManager  = "bootstrap-manifests/bootstrap-controller-manager.yaml"
+	AssetPathBootstrapScheduler          = "bootstrap-manifests/bootstrap-scheduler.yaml"
+	AssetPathBootstrapEtcd               = "bootstrap-manifests/bootstrap-etcd.yaml"
+	BootstrapSecretsDir                  = "/tmp/bootkube/secrets"
 )
 
 // AssetConfig holds all configuration needed when generating
 // the default set of assets.
 type Config struct {
-	EtcdServers     []*url.URL
-	APIServers      []*url.URL
-	CACert          *x509.Certificate
-	CAPrivKey       *rsa.PrivateKey
-	AltNames        *tlsutil.AltNames
-	PodCIDR         *net.IPNet
-	ServiceCIDR     *net.IPNet
-	APIServiceIP    net.IP
-	DNSServiceIP    net.IP
-	ETCDServiceIP   net.IP
-	SelfHostKubelet bool
-	SelfHostedEtcd  bool
-	CloudProvider   string
+	EtcdServers         []*url.URL
+	APIServers          []*url.URL
+	CACert              *x509.Certificate
+	CAPrivKey           *rsa.PrivateKey
+	AltNames            *tlsutil.AltNames
+	PodCIDR             *net.IPNet
+	ServiceCIDR         *net.IPNet
+	APIServiceIP        net.IP
+	DNSServiceIP        net.IP
+	ETCDServiceIP       net.IP
+	SelfHostKubelet     bool
+	SelfHostedEtcd      bool
+	CloudProvider       string
+	BootstrapSecretsDir string
 }
 
 // NewDefaultAssets returns a list of default assets, optionally
 // configured via a user provided AssetConfig. Default assets include
 // TLS assets (certs, keys and secrets), and k8s component manifests.
 func NewDefaultAssets(conf Config) (Assets, error) {
+	conf.BootstrapSecretsDir = BootstrapSecretsDir
+
 	as := newStaticAssets()
 	as = append(as, newDynamicAssets(conf)...)
 

pkg/asset/internal/templates.go

@@ -205,6 +205,61 @@ spec:
           path: /var/lock
 `)
 
+	BootstrapAPIServerTemplate = []byte(`apiVersion: v1
+kind: Pod
+metadata:
+  name: bootstrap-kube-apiserver
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-apiserver
+    image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
+    command:
+    - /usr/bin/flock
+    - --exclusive
+    - --timeout=30
+    - /var/lock/api-server.lock
+    - /hyperkube
+    - apiserver
+    - --admission-control=NamespaceLifecycle,ServiceAccount
+    - --allow-privileged=true
+    - --authorization-mode=RBAC
+    - --bind-address=0.0.0.0
+    - --client-ca-file=/etc/kubernetes/secrets/ca.crt
+    - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}{{ if .SelfHostedEtcd }},http://127.0.0.1:12379{{end}}
+    - --insecure-port=8080
+    - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
+    - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
+    - --runtime-config=api/all=true
+    - --secure-port=443
+    - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
+    - --service-cluster-ip-range={{ .ServiceCIDR }}
+    - --storage-backend=etcd3
+    - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
+    - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: /etc/kubernetes/secrets
+      name: secrets
+      readOnly: true
+    - mountPath: /var/lock
+      name: var-lock
+      readOnly: false
+  volumes:
+  - name: secrets
+    hostPath:
+      path: {{ .BootstrapSecretsDir }}
+  - name: ssl-certs-host
+    hostPath:
+      path: /usr/share/ca-certificates
+  - name: var-lock
+    hostPath:
+      path: /var/lock
+`)
+
 	KencTemplate = []byte(`apiVersion: "extensions/v1beta1"
 kind: DaemonSet
 metadata:
@@ -360,6 +415,42 @@ spec:
           path: /usr/share/ca-certificates
       dnsPolicy: Default # Don't use cluster DNS.
 `)
+
+	BootstrapControllerManagerTemplate = []byte(`apiVersion: v1
+kind: Pod
+metadata:
+  name: bootstrap-kube-controller-manager
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-controller-manager
+    image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
+    command:
+    - ./hyperkube
+    - controller-manager
+    - --allocate-node-cidrs=true
+    - --cluster-cidr={{ .PodCIDR }}
+    - --configure-cloud-routes=false
+    - --leader-elect=true
+    - --master=http://127.0.0.1:8080
+    - --root-ca-file=/etc/kubernetes/secrets/ca.crt
+    - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
+    volumeMounts:
+    - name: secrets
+      mountPath: /etc/kubernetes/secrets
+      readOnly: true
+    - name: ssl-host
+      mountPath: /etc/ssl/certs
+      readOnly: true
+  volumes:
+  - name: secrets
+    hostPath:
+      path: {{ .BootstrapSecretsDir }}
+  - name: ssl-host
+    hostPath:
+      path: /usr/share/ca-certificates
+`)
 	ControllerManagerDisruptionTemplate = []byte(`apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
 metadata:
@@ -404,6 +495,23 @@ spec:
           initialDelaySeconds: 15
           timeoutSeconds: 15
 
+`)
+
+	BootstrapSchedulerTemplate = []byte(`apiVersion: v1
+kind: Pod
+metadata:
+  name: bootstrap-kube-scheduler
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-scheduler
+    image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
+    command:
+    - ./hyperkube
+    - scheduler
+    - --leader-elect=true
+    - --master=http://127.0.0.1:8080
 `)
 	SchedulerDisruptionTemplate = []byte(`apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
@@ -684,6 +792,37 @@ spec:
     protocol: TCP
 `)
 
+	BootstrapEtcdTemplate = []byte(`apiVersion: v1
+kind: Pod
+metadata:
+  name: bootstrap-etcd
+  namespace: kube-system
+  labels:
+    k8s-app: boot-etcd
+spec:
+  hostNetwork: true
+  restartPolicy: Never
+  containers:
+  - name: etcd
+    image: quay.io/coreos/etcd:v3.1.0
+    command:
+    - /usr/local/bin/etcd
+    - --name=boot-etcd
+    - --listen-client-urls=http://0.0.0.0:12379
+    - --listen-peer-urls=http://0.0.0.0:12380
+    - --advertise-client-urls=http://$(MY_POD_IP):12379
+    - --initial-advertise-peer-urls=http://$(MY_POD_IP):12380
+    - --initial-cluster=boot-etcd=http://$(MY_POD_IP):12380
+    - --initial-cluster-token=bootkube
+    - --initial-cluster-state=new
+    - --data-dir=/var/etcd/data
+    env:
+      - name: MY_POD_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+`)
+
 	KubeFlannelCfgTemplate = []byte(`apiVersion: v1
 kind: ConfigMap
 metadata:

pkg/asset/k8s.go

@@ -38,6 +38,9 @@ func newDynamicAssets(conf Config) Assets {
 		mustCreateAssetFromTemplate(AssetPathProxy, internal.ProxyTemplate, conf),
 		mustCreateAssetFromTemplate(AssetPathKubeFlannelCfg, internal.KubeFlannelCfgTemplate, conf),
 		mustCreateAssetFromTemplate(AssetPathKubeDNSSvc, internal.DNSSvcTemplate, conf),
+		mustCreateAssetFromTemplate(AssetPathBootstrapAPIServer, internal.BootstrapAPIServerTemplate, conf),
+		mustCreateAssetFromTemplate(AssetPathBootstrapControllerManager, internal.BootstrapControllerManagerTemplate, conf),
+		mustCreateAssetFromTemplate(AssetPathBootstrapScheduler, internal.BootstrapSchedulerTemplate, conf),
 	}
 	if conf.SelfHostKubelet {
 		assets = append(assets, mustCreateAssetFromTemplate(AssetPathKubelet, internal.KubeletTemplate, conf))
@@ -47,6 +50,7 @@ func newDynamicAssets(conf Config) Assets {
 			mustCreateAssetFromTemplate(AssetPathEtcdOperator, internal.EtcdOperatorTemplate, conf),
 			mustCreateAssetFromTemplate(AssetPathEtcdSvc, internal.EtcdSvcTemplate, conf),
 			mustCreateAssetFromTemplate(AssetPathKenc, internal.KencTemplate, conf),
+			mustCreateAssetFromTemplate(AssetPathBootstrapEtcd, internal.BootstrapEtcdTemplate, conf),
 		)
 	}
 	return assets