Disable insecure apiserver access. (#459)

This disables the insecure apiserver port for both the bootstrap
and self-hosted control planes. In so doing I migrated all of bootkube
to use the kubeconfig rather than a static local port.

This assumes that the temporary control plane is also addressable
at the same IP as the the final control plane. If this is a faulty
assumption then this part likely won't work.

While I was at it I changed some dependencies to point at client-go
to help us move off of vendoring all of kubernetes in the future.

Fixes #324.

Author: diegs

cmd/bootkube/start.go

@@ -47,7 +47,11 @@ func runCmdStart(cmd *cobra.Command, args []string) error {
 	util.InitLogs()
 	defer util.FlushLogs()
 
-	return bk.Run()
+	if err := bk.Run(); err != nil {
+		// Always report errors.
+		bootkube.UserOutput("Error: %v\n", err)
+	}
+	return nil
 }
 
 func validateStartOpts(cmd *cobra.Command, args []string) error {

pkg/asset/asset.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 
 	"github.com/kubernetes-incubator/bootkube/pkg/tlsutil"
@@ -60,31 +61,31 @@ const (
 // AssetConfig holds all configuration needed when generating
 // the default set of assets.
 type Config struct {
-	EtcdCACert          *x509.Certificate
-	EtcdClientCert      *x509.Certificate
-	EtcdClientKey       *rsa.PrivateKey
-	EtcdServers         []*url.URL
-	EtcdUseTLS          bool
-	APIServers          []*url.URL
-	CACert              *x509.Certificate
-	CAPrivKey           *rsa.PrivateKey
-	AltNames            *tlsutil.AltNames
-	PodCIDR             *net.IPNet
-	ServiceCIDR         *net.IPNet
-	APIServiceIP        net.IP
-	DNSServiceIP        net.IP
-	EtcdServiceIP       net.IP
-	SelfHostKubelet     bool
-	SelfHostedEtcd      bool
-	CloudProvider       string
-	BootstrapSecretsDir string
+	EtcdCACert             *x509.Certificate
+	EtcdClientCert         *x509.Certificate
+	EtcdClientKey          *rsa.PrivateKey
+	EtcdServers            []*url.URL
+	EtcdUseTLS             bool
+	APIServers             []*url.URL
+	CACert                 *x509.Certificate
+	CAPrivKey              *rsa.PrivateKey
+	AltNames               *tlsutil.AltNames
+	PodCIDR                *net.IPNet
+	ServiceCIDR            *net.IPNet
+	APIServiceIP           net.IP
+	DNSServiceIP           net.IP
+	EtcdServiceIP          net.IP
+	SelfHostKubelet        bool
+	SelfHostedEtcd         bool
+	CloudProvider          string
+	BootstrapSecretsSubdir string
 }
 
 // NewDefaultAssets returns a list of default assets, optionally
 // configured via a user provided AssetConfig. Default assets include
 // TLS assets (certs, keys and secrets), and k8s component manifests.
 func NewDefaultAssets(conf Config) (Assets, error) {
-	conf.BootstrapSecretsDir = BootstrapSecretsDir
+	conf.BootstrapSecretsSubdir = path.Base(BootstrapSecretsDir)
 
 	as := newStaticAssets()
 	as = append(as, newDynamicAssets(conf)...)

pkg/asset/internal/templates.go

@@ -168,20 +168,21 @@ spec:
         - --authorization-mode=RBAC
         - --bind-address=0.0.0.0
         - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-        - --cloud-provider={{ .CloudProvider  }}
+        - --cloud-provider={{ .CloudProvider }}
 {{- if .EtcdUseTLS }}
         - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt
         - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
         - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
 {{- end }}
         - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}
-        - --insecure-port=8080
+        - --insecure-port=0
         - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
         - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
         - --secure-port=443
         - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
         - --service-cluster-ip-range={{ .ServiceCIDR }}
         - --storage-backend=etcd3
+        - --tls-ca-file=/etc/kubernetes/secrets/ca.crt
         - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
         - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
         env:
@@ -247,13 +248,14 @@ spec:
     - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
 {{- end }}
     - --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}{{ if .SelfHostedEtcd }},http://127.0.0.1:12379{{end}}
-    - --insecure-port=8080
+    - --insecure-port=0
     - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
     - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
     - --secure-port=443
     - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
     - --service-cluster-ip-range={{ .ServiceCIDR }}
     - --storage-backend=etcd3
+    - --tls-ca-file=/etc/kubernetes/secrets/ca.crt
     - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
     - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
     volumeMounts:
@@ -270,7 +272,7 @@ spec:
   volumes:
   - name: secrets
     hostPath:
-      path: {{ .BootstrapSecretsDir }}
+      path: /etc/kubernetes/{{ .BootstrapSecretsSubdir }}
   - name: ssl-certs-host
     hostPath:
       path: /usr/share/ca-certificates
@@ -429,7 +431,7 @@ spec:
         - ./hyperkube
         - controller-manager
         - --allocate-node-cidrs=true
-        - --cloud-provider={{ .CloudProvider  }}
+        - --cloud-provider={{ .CloudProvider }}
         - --cluster-cidr={{ .PodCIDR }}
         - --configure-cloud-routes=false
         - --leader-elect=true
@@ -481,22 +483,22 @@ spec:
     - --allocate-node-cidrs=true
     - --cluster-cidr={{ .PodCIDR }}
     - --configure-cloud-routes=false
+    - --kubeconfig=/etc/kubernetes/kubeconfig
     - --leader-elect=true
-    - --master=http://127.0.0.1:8080
-    - --root-ca-file=/etc/kubernetes/secrets/ca.crt
-    - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
+    - --root-ca-file=/etc/kubernetes/{{ .BootstrapSecretsSubdir }}/ca.crt
+    - --service-account-private-key-file=/etc/kubernetes/{{ .BootstrapSecretsSubdir }}/service-account.key
     volumeMounts:
-    - name: secrets
-      mountPath: /etc/kubernetes/secrets
+    - name: kubernetes
+      mountPath: /etc/kubernetes
       readOnly: true
     - name: ssl-host
       mountPath: /etc/ssl/certs
       readOnly: true
   hostNetwork: true
   volumes:
-  - name: secrets
+  - name: kubernetes
     hostPath:
-      path: {{ .BootstrapSecretsDir }}
+      path: /etc/kubernetes
   - name: ssl-host
     hostPath:
       path: /usr/share/ca-certificates
@@ -582,9 +584,17 @@ spec:
     command:
     - ./hyperkube
     - scheduler
+    - --kubeconfig=/etc/kubernetes/kubeconfig
     - --leader-elect=true
-    - --master=http://127.0.0.1:8080
+    volumeMounts:
+    - name: kubernetes
+      mountPath: /etc/kubernetes
+      readOnly: true
   hostNetwork: true
+  volumes:
+  - name: kubernetes
+    hostPath:
+      path: /etc/kubernetes
 `)
 	SchedulerDisruptionTemplate = []byte(`apiVersion: policy/v1beta1
 kind: PodDisruptionBudget

pkg/bootkube/bootkube.go

@@ -5,15 +5,16 @@ import (
 	"path/filepath"
 	"time"
 
+	"k8s.io/client-go/tools/clientcmd"
+
 	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/kubernetes-incubator/bootkube/pkg/asset"
 	"github.com/kubernetes-incubator/bootkube/pkg/util/etcdutil"
 )
 
-const (
-	assetTimeout    = 20 * time.Minute
-	insecureAPIAddr = "http://127.0.0.1:8080"
-)
+const assetTimeout = 20 * time.Minute
+
+var kubeConfig clientcmd.ClientConfig
 
 var requiredPods = []string{
 	"pod-checkpointer",
@@ -49,6 +50,12 @@ func (b *bootkube) Run() error {
 		}
 	}()
 
+	// TODO(diegs): create and share a single client rather than the kubeconfig once all uses of it
+	// are migrated to client-go.
+	kubeConfig = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(b.assetDir, asset.AssetPathKubeConfig)},
+		&clientcmd.ConfigOverrides{})
+
 	var err error
 	defer func() {
 		// Always report errors.
@@ -80,7 +87,7 @@ func (b *bootkube) Run() error {
 		if err != nil {
 			return err
 		}
-		if err = etcdutil.Migrate(insecureAPIAddr, etcdServiceIP); err != nil {
+		if err = etcdutil.Migrate(kubeConfig, etcdServiceIP); err != nil {
 			return err
 		}
 	}

pkg/bootkube/create.go

@@ -9,19 +9,16 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
-	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/api"
 	"k8s.io/client-go/tools/clientcmd"
-	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
-	"k8s.io/kubernetes/pkg/api"
-	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
 	"k8s.io/kubernetes/pkg/kubectl/resource"
 
 	"github.com/kubernetes-incubator/bootkube/pkg/util"
 )
 
 func CreateAssets(manifestDir string, timeout time.Duration) error {
-
 	upFn := func() (bool, error) {
 		if err := apiTest(); err != nil {
 			glog.Warningf("Unable to determine api-server readiness: %v", err)
@@ -65,11 +62,7 @@ func CreateAssets(manifestDir string, timeout time.Duration) error {
 }
 
 func createAssets(manifestDir string) error {
-	config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		&clientcmd.ClientConfigLoadingRules{},
-		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: insecureAPIAddr}},
-	)
-	f := cmdutil.NewFactory(config)
+	f := cmdutil.NewFactory(kubeConfig)
 
 	shouldValidate := true
 	schema, err := f.Validator(shouldValidate, fmt.Sprintf("~/%s/%s", clientcmd.RecommendedHomeDir, clientcmd.RecommendedSchemaName))
@@ -135,7 +128,11 @@ func createAssets(manifestDir string) error {
 }
 
 func apiTest() error {
-	client, err := clientset.NewForConfig(&restclient.Config{Host: insecureAPIAddr})
+	config, err := kubeConfig.ClientConfig()
+	if err != nil {
+		return err
+	}
+	client, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		return err
 	}

pkg/bootkube/parse.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	"github.com/kubernetes-incubator/bootkube/pkg/asset"
-	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/client-go/pkg/api/v1"
 )
 
 // detectEtcdIP deserializes the etcd-service ClusterIP.

pkg/bootkube/status.go

@@ -12,11 +12,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
-	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/api"
+	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/tools/cache"
-	"k8s.io/kubernetes/pkg/api"
-	"k8s.io/kubernetes/pkg/api/v1"
-	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 )
 
 const (
@@ -39,14 +38,18 @@ func WaitUntilPodsRunning(pods []string, timeout time.Duration) error {
 }
 
 type statusController struct {
-	client        clientset.Interface
+	client        kubernetes.Interface
 	podStore      cache.Store
 	watchPods     []string
 	lastPodPhases map[string]v1.PodPhase
 }
 
 func NewStatusController(pods []string) (*statusController, error) {
-	client, err := clientset.NewForConfig(&restclient.Config{Host: insecureAPIAddr})
+	config, err := kubeConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	client, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		return nil, err
 	}

pkg/util/etcdutil/migrate.go

@@ -14,9 +14,10 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/api"
 	restclient "k8s.io/client-go/rest"
-	"k8s.io/kubernetes/pkg/api"
-	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 const (
@@ -28,10 +29,12 @@ var (
 	waitBootEtcdRemovedTime    = 300 * time.Second
 )
 
-func Migrate(apiserverAddr, etcdServiceIP string) error {
-	kubecli, err := clientset.NewForConfig(&restclient.Config{
-		Host: apiserverAddr,
-	})
+func Migrate(kubeConfig clientcmd.ClientConfig, etcdServiceIP string) error {
+	config, err := kubeConfig.ClientConfig()
+	if err != nil {
+		return fmt.Errorf("fail to create kube client: %v", err)
+	}
+	kubecli, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		return fmt.Errorf("fail to create kube client: %v", err)
 	}
@@ -49,7 +52,7 @@ func Migrate(apiserverAddr, etcdServiceIP string) error {
 	}
 	glog.Infof("boot-etcd pod IP is: %s", ip)
 
-	if err := createMigratedEtcdCluster(restClient, apiserverAddr, ip); err != nil {
+	if err := createMigratedEtcdCluster(restClient, ip); err != nil {
 		return fmt.Errorf("fail to create migrated etcd cluster: %v", err)
 	}
 	glog.Infof("etcd cluster for migration is created")
@@ -88,7 +91,7 @@ func waitEtcdTPRReady(restClient restclient.Interface, interval, timeout time.Du
 	return nil
 }
 
-func getBootEtcdPodIP(kubecli clientset.Interface) (string, error) {
+func getBootEtcdPodIP(kubecli kubernetes.Interface) (string, error) {
 	var ip string
 	err := wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) {
 		podList, err := kubecli.CoreV1().Pods(api.NamespaceSystem).List(v1.ListOptions{
@@ -113,7 +116,7 @@ func getBootEtcdPodIP(kubecli clientset.Interface) (string, error) {
 	return ip, err
 }
 
-func createMigratedEtcdCluster(restclient restclient.Interface, host, podIP string) error {
+func createMigratedEtcdCluster(restclient restclient.Interface, podIP string) error {
 	b := []byte(fmt.Sprintf(`{
   "apiVersion": "%s/%s",
   "kind": "%s",