Add apiserver recovery backend. (#500)

Also fix some path munging that I somehow broke in the process of
writing tests :-/

Author: diegs

README.md

@@ -58,20 +58,26 @@ bootkube start --asset-dir=my-cluster
 
 ### Recover a downed cluster
 
-In the case of a partial or total control plane outage (i.e. due to lost master nodes) an experimental `recover` command can extract and write manifests from a backup location. These manifests can then be used by the `start` command to reboot the cluster. Currently recovery from an external running etcd cluster is the only supported method.
+In the case of a partial or total control plane outage (i.e. due to lost master nodes) an experimental `recover` command can extract and write manifests from a backup location. These manifests can then be used by the `start` command to reboot the cluster. Currently recovery from a running apiserver or external running etcd cluster are the only supported methods.
 
 To see available options, run:
 
 ```
 bootkube recover --help
 ```
 
-Example:
+Recover from an external running etcd cluster:
 
 ```
 bootkube recover --asset-dir=recovered --etcd-servers=http://127.0.0.1:2379 --kubeconfig=/etc/kubernetes/kubeconfig
 ```
 
+Recover from a running apiserver (i.e. if the scheduler pods are all down):
+
+```
+bootkube recover --asset-dir=recovered --kubeconfig=/etc/kubernetes/kubeconfig
+```
+
 For a complete recovery example please see the [hack/multi-node/bootkube-test-recovery](hack/multi-node/bootkube-test-recovery) script.
 
 ## Building

cmd/bootkube/recover.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/kubernetes-incubator/bootkube/pkg/bootkube"
 	"github.com/kubernetes-incubator/bootkube/pkg/recovery"
 
 	"github.com/coreos/etcd/clientv3"
@@ -20,8 +21,8 @@ import (
 var (
 	cmdRecover = &cobra.Command{
 		Use:          "recover",
-		Short:        "Recover a control plane from state stored in etcd.",
-		Long:         "",
+		Short:        "Recover a self-hosted control plane",
+		Long:         "This command reads control plane manifests from a running apiserver or etcd and writes them to asset-dir. Users can then use `bootkube start` pointed at this asset-dir to re-the a self-hosted cluster. Please see the project README for more details and examples.",
 		PreRunE:      validateRecoverOpts,
 		RunE:         runCmdRecover,
 		SilenceUsage: true,
@@ -44,7 +45,7 @@ func init() {
 	cmdRecover.Flags().StringVar(&recoverOpts.etcdCAPath, "etcd-ca-path", "", "Path to an existing PEM encoded CA that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-certificate-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
 	cmdRecover.Flags().StringVar(&recoverOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
 	cmdRecover.Flags().StringVar(&recoverOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.")
-	cmdRecover.Flags().StringVar(&recoverOpts.etcdServers, "etcd-servers", "", "List of etcd servers URLs including host:port, comma separated.")
+	cmdRecover.Flags().StringVar(&recoverOpts.etcdServers, "etcd-servers", "", "List of etcd server URLs including host:port, comma separated.")
 	cmdRecover.Flags().StringVar(&recoverOpts.etcdPrefix, "etcd-prefix", "/registry", "Path prefix to Kubernetes cluster data in etcd.")
 	cmdRecover.Flags().StringVar(&recoverOpts.kubeConfigPath, "kubeconfig", "", "Path to kubeconfig for communicating with the cluster.")
 }
@@ -55,11 +56,25 @@ func runCmdRecover(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	etcdClient, err := createEtcdClient()
-	if err != nil {
-		return err
+
+	var backend recovery.Backend
+	switch {
+	case recoverOpts.etcdServers != "":
+		bootkube.UserOutput("Attempting recovery using etcd cluster at %q...\n", recoverOpts.etcdServers)
+		etcdClient, err := createEtcdClient()
+		if err != nil {
+			return err
+		}
+		backend = recovery.NewEtcdBackend(etcdClient, recoverOpts.etcdPrefix)
+	default:
+		bootkube.UserOutput("Attempting recovery using apiserver at %q...\n", recoverOpts.kubeConfigPath)
+		backend, err = recovery.NewAPIServerBackend(recoverOpts.kubeConfigPath)
+		if err != nil {
+			return err
+		}
 	}
-	as, err := recovery.Recover(context.Background(), recovery.NewEtcdBackend(etcdClient, recoverOpts.etcdPrefix), recoverOpts.kubeConfigPath)
+
+	as, err := recovery.Recover(context.Background(), backend, recoverOpts.kubeConfigPath)
 	if err != nil {
 		return err
 	}

pkg/recovery/apiserver.go

@@ -0,0 +1,61 @@
+package recovery
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/api"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+type apiServerBackend struct {
+	client *kubernetes.Clientset
+}
+
+// NewAPIServerBackend constructs a new backend to talk to the API server using the given
+// kubeConfig.
+//
+// TODO(diegs): support using a service account instead of a kubeconfig.
+func NewAPIServerBackend(kubeConfigPath string) (Backend, error) {
+	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeConfigPath},
+		&clientcmd.ConfigOverrides{})
+	config, err := kubeConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return &apiServerBackend{
+		client: client,
+	}, nil
+}
+
+// read implements Backend.read().
+func (b *apiServerBackend) read(context.Context) (*controlPlane, error) {
+	cp := &controlPlane{}
+	configMaps, err := b.client.CoreV1().ConfigMaps(api.NamespaceSystem).List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	cp.configMaps = *configMaps
+	deployments, err := b.client.ExtensionsV1beta1().Deployments(api.NamespaceSystem).List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	cp.deployments = *deployments
+	daemonSets, err := b.client.ExtensionsV1beta1().DaemonSets(api.NamespaceSystem).List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	cp.daemonSets = *daemonSets
+	secrets, err := b.client.CoreV1().Secrets(api.NamespaceSystem).List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	cp.secrets = *secrets
+	return cp, nil
+}

pkg/recovery/recover.go

@@ -13,6 +13,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path"
+	"path/filepath"
 	"reflect"
 
 	"github.com/ghodss/yaml"
@@ -29,7 +30,7 @@ import (
 const (
 	k8sAppLabel           = "k8s-app"   // The label used in versions > v0.4.2
 	componentAppLabel     = "component" // The label used in versions <= v0.4.2
-	kubeletKubeConfigPath = "/etc/kubernetes/kubeconfig"
+	kubeletKubeConfigPath = "/etc/kubernetes"
 )
 
 var (
@@ -190,14 +191,14 @@ func fixUpBootstrapPods(pods []v1.Pod) (requiredConfigMaps, requiredSecrets map[
 		for i := range pod.Spec.Volumes {
 			vol := &pod.Spec.Volumes[i]
 			if vol.Secret != nil {
-				pathPrefix := path.Join(asset.BootstrapSecretsDir, "secrets", vol.Secret.SecretName)
-				requiredSecrets[vol.Secret.SecretName] = pathPrefix
-				vol.HostPath = &v1.HostPathVolumeSource{Path: pathPrefix}
+				pathSuffix := filepath.Join("secrets", vol.Secret.SecretName)
+				requiredSecrets[vol.Secret.SecretName] = filepath.Join(asset.AssetPathSecrets, pathSuffix)
+				vol.HostPath = &v1.HostPathVolumeSource{Path: filepath.Join(asset.BootstrapSecretsDir, pathSuffix)}
 				vol.Secret = nil
 			} else if vol.ConfigMap != nil {
-				pathPrefix := path.Join(asset.BootstrapSecretsDir, "config-maps", vol.ConfigMap.Name)
-				requiredConfigMaps[vol.ConfigMap.Name] = pathPrefix
-				vol.HostPath = &v1.HostPathVolumeSource{Path: pathPrefix}
+				pathSuffix := filepath.Join("config-maps", vol.ConfigMap.Name)
+				requiredConfigMaps[vol.ConfigMap.Name] = filepath.Join(asset.AssetPathSecrets, pathSuffix)
+				vol.HostPath = &v1.HostPathVolumeSource{Path: path.Join(asset.BootstrapSecretsDir, pathSuffix)}
 				vol.ConfigMap = nil
 			}
 		}

pkg/recovery/recover_test.go

@@ -253,7 +253,7 @@ func TestFixUpBootstrapPods(t *testing.T) {
 				VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/bootstrap-secrets/secrets/kube-apiserver"}},
 			}, {
 				Name:         "kubeconfig",
-				VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/kubeconfig"}},
+				VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes"}},
 			}},
 		},
 	}, {
@@ -278,12 +278,12 @@ func TestFixUpBootstrapPods(t *testing.T) {
 			}},
 			Volumes: []v1.Volume{{
 				Name:         "kubeconfig",
-				VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes/kubeconfig"}},
+				VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/etc/kubernetes"}},
 			}},
 		},
 	}}
-	wantConfigMaps := map[string]string{"kube-apiserver": "/etc/kubernetes/bootstrap-secrets/config-maps/kube-apiserver"}
-	wantSecrets := map[string]string{"kube-apiserver": "/etc/kubernetes/bootstrap-secrets/secrets/kube-apiserver"}
+	wantConfigMaps := map[string]string{"kube-apiserver": "tls/config-maps/kube-apiserver"}
+	wantSecrets := map[string]string{"kube-apiserver": "tls/secrets/kube-apiserver"}
 	gotConfigMaps, gotSecrets := fixUpBootstrapPods(pods)
 	if !reflect.DeepEqual(gotSecrets, wantSecrets) || !reflect.DeepEqual(gotConfigMaps, wantConfigMaps) {
 		t.Errorf("fixUpBootstrapPods(%v) = %v, %v, want: %v, %v", pods, gotConfigMaps, gotSecrets, wantConfigMaps, wantSecrets)