Preliminary recovery command for bootkube. (#491)

The new `bootkube recover` command provides a framework for reading
control plane manifests out of a storage backend, serializing them to
disk, and constructing pod manifests for the bootstrap components that
are usable with `bootkube start` to re-bootstrap the control plane.

The command initially provides and etcdclient backend for use with a
running etcd cluster. In the future we can add support other methods,
such as an etcd backup directory or a running apiserver.

There is also documentation and an example script in hack/multi-node
that deletes a master node, recovers the assets from etcd, and then
re-bootstraps the control plane.

Author: diegs

README.md

@@ -18,7 +18,9 @@ If you are interested in the design and details [see the Kubernetes self-hosted
 
 ## Usage
 
-Bootkube has two main commands: `render` and `start`
+Bootkube has two main commands: `render` and `start`.
+
+There is a third, experimental command `recover` which can help reboot a downed cluster (see below).
 
 ### Render assets
 
@@ -54,6 +56,24 @@ Example:
 bootkube start --asset-dir=my-cluster
 ```
 
+### Recover a downed cluster
+
+In the case of a partial or total control plane outage (i.e. due to lost master nodes) an experimental `recover` command can extract and write manifests from a backup location. These manifests can then be used by the `start` command to reboot the cluster. Currently recovery from an external running etcd cluster is the only supported method.
+
+To see available options, run:
+
+```
+bootkube recover --help
+```
+
+Example:
+
+```
+bootkube recover --asset-dir=recovered --etcd-servers=http://127.0.0.1:2379 --kubeconfig=/etc/kubernetes/kubeconfig
+```
+
+For a complete recovery example please see the [hack/multi-node/bootkube-test-recovery](hack/multi-node/bootkube-test-recovery) script.
+
 ## Building
 
 First, clone the repo into the proper location in your $GOPATH:

cmd/bootkube/recover.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/kubernetes-incubator/bootkube/pkg/recovery"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/spf13/cobra"
+)
+
+var (
+	cmdRecover = &cobra.Command{
+		Use:          "recover",
+		Short:        "Recover a control plane from state stored in etcd.",
+		Long:         "",
+		PreRunE:      validateRecoverOpts,
+		RunE:         runCmdRecover,
+		SilenceUsage: true,
+	}
+
+	recoverOpts struct {
+		assetDir            string
+		etcdCAPath          string
+		etcdCertificatePath string
+		etcdPrivateKeyPath  string
+		etcdServers         string
+		etcdPrefix          string
+		kubeConfigPath      string
+	}
+)
+
+func init() {
+	cmdRoot.AddCommand(cmdRecover)
+	cmdRecover.Flags().StringVar(&recoverOpts.assetDir, "asset-dir", "", "Output path for writing recovered cluster assets.")
+	cmdRecover.Flags().StringVar(&recoverOpts.etcdCAPath, "etcd-ca-path", "", "Path to an existing PEM encoded CA that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-certificate-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRecover.Flags().StringVar(&recoverOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRecover.Flags().StringVar(&recoverOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.")
+	cmdRecover.Flags().StringVar(&recoverOpts.etcdServers, "etcd-servers", "", "List of etcd servers URLs including host:port, comma separated.")
+	cmdRecover.Flags().StringVar(&recoverOpts.etcdPrefix, "etcd-prefix", "/registry", "Path prefix to Kubernetes cluster data in etcd.")
+	cmdRecover.Flags().StringVar(&recoverOpts.kubeConfigPath, "kubeconfig", "", "Path to kubeconfig for communicating with the cluster.")
+}
+
+func runCmdRecover(cmd *cobra.Command, args []string) error {
+	var err error
+	recoverOpts.kubeConfigPath, err = filepath.Abs(recoverOpts.kubeConfigPath)
+	if err != nil {
+		return err
+	}
+	etcdClient, err := createEtcdClient()
+	if err != nil {
+		return err
+	}
+	as, err := recovery.Recover(context.Background(), recovery.NewEtcdBackend(etcdClient, recoverOpts.etcdPrefix), recoverOpts.kubeConfigPath)
+	if err != nil {
+		return err
+	}
+	return as.WriteFiles(recoverOpts.assetDir)
+}
+
+func validateRecoverOpts(cmd *cobra.Command, args []string) error {
+	if recoverOpts.assetDir == "" {
+		return errors.New("missing required flag: --asset-dir")
+	}
+	if (recoverOpts.etcdCAPath != "" || recoverOpts.etcdCertificatePath != "" || recoverOpts.etcdPrivateKeyPath != "") && (recoverOpts.etcdCAPath == "" || recoverOpts.etcdCertificatePath == "" || recoverOpts.etcdPrivateKeyPath == "") {
+		return errors.New("you must specify either all or none of --etcd-ca-path, --etcd-certificate-path, and --etcd-private-key-path")
+	}
+	if recoverOpts.etcdPrefix == "" {
+		return errors.New("missing required flag: --etcd-prefix")
+	}
+	if recoverOpts.kubeConfigPath == "" {
+		return errors.New("missing required flag: --kubeconfig")
+	}
+	return nil
+}
+
+func createEtcdClient() (*clientv3.Client, error) {
+	cfg := clientv3.Config{
+		Endpoints:   strings.Split(recoverOpts.etcdServers, ","),
+		DialTimeout: 5 * time.Second,
+	}
+	if recoverOpts.etcdCAPath != "" {
+		clientCert, err := tls.LoadX509KeyPair(recoverOpts.etcdCertificatePath, recoverOpts.etcdPrivateKeyPath)
+		if err != nil {
+			return nil, err
+		}
+		roots := x509.NewCertPool()
+		etcdCA, err := ioutil.ReadFile(recoverOpts.etcdCAPath)
+		if err != nil {
+			return nil, err
+		}
+		if ok := roots.AppendCertsFromPEM(etcdCA); !ok {
+			return nil, fmt.Errorf("error processing --etcd-ca-file %s", recoverOpts.etcdCAPath)
+		}
+		cfg.TLS = &tls.Config{
+			Certificates: []tls.Certificate{clientCert},
+			RootCAs:      roots,
+		}
+	}
+	return clientv3.New(cfg)
+}

hack/multi-node/bootkube-test-recovery

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+GLOG_v=${GLOG_v:-1}
+HOST=${HOST:-c1}
+SELF_HOST_ETCD=${SELF_HOST_ETCD:-false}
+
+if [ ! -d "cluster" ]; then
+  echo "Need some cluster assets to perform recovery; try running bootkube-up."
+fi
+
+if [ -f cluster/bootstrap-manifests/bootstrap-etcd.yaml ]; then
+  echo "ERROR: $(basename $0) does not currently support self-hosted etcd."
+  exit 1
+fi
+
+echo
+echo "Destroying and re-creating the master node..."
+echo
+
+vagrant destroy -f $HOST
+vagrant up $HOST
+
+echo
+echo "As you can see, the cluster is now dead:"
+echo
+
+set -x
+! kubectl --kubeconfig=cluster/auth/kubeconfig get nodes
+{ set +x; } 2>/dev/null
+
+echo
+echo "Recovering the control plane from etcd..."
+echo
+
+scp -q -F ssh_config ../../_output/bin/linux/bootkube cluster/auth/kubeconfig cluster/tls/etcd-* core@$HOST:/home/core
+ssh -q -F ssh_config core@$HOST "GLOG_v=${GLOG_v} /home/core/bootkube recover \
+  --asset-dir=/home/core/recovered \
+  --etcd-ca-path=/home/core/etcd-ca.crt \
+  --etcd-certificate-path=/home/core/etcd-client.crt \
+  --etcd-private-key-path=/home/core/etcd-client.key \
+  --etcd-servers=https://172.17.4.51:2379 \
+  --kubeconfig=/home/core/kubeconfig 2>> /home/core/recovery.log"
+
+echo
+echo "Running bootkube start..."
+echo
+
+ssh -q -F ssh_config core@$HOST "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/recovered 2>> /home/core/recovery.log"
+
+echo
+echo "The cluster should now be recovered. You should be able to access the cluster again using:"
+echo "kubectl --kubeconfig=cluster/auth/kubeconfig get nodes"
+echo

pkg/bootkube/bootkube.go

@@ -41,19 +41,21 @@ func NewBootkube(config Config) (*bootkube, error) {
 }
 
 func (b *bootkube) Run() error {
-	defer func() {
-		// Always clean up the bootstrap control plane and secrets.
-		if err := CleanupBootstrapControlPlane(b.assetDir, b.podManifestPath); err != nil {
-			UserOutput("Error cleaning up temporary bootstrap control plane: %v\n", err)
-		}
-	}()
-
 	// TODO(diegs): create and share a single client rather than the kubeconfig once all uses of it
 	// are migrated to client-go.
 	kubeConfig = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
 		&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(b.assetDir, asset.AssetPathKubeConfig)},
 		&clientcmd.ConfigOverrides{})
 
+	bcp := NewBootstrapControlPlane(b.assetDir, b.podManifestPath)
+
+	defer func() {
+		// Always tear down the bootstrap control plane and clean up manifests and secrets.
+		if err := bcp.Teardown(); err != nil {
+			UserOutput("Error tearing down temporary bootstrap control plane: %v\n", err)
+		}
+	}()
+
 	var err error
 	defer func() {
 		// Always report errors.
@@ -62,7 +64,7 @@ func (b *bootkube) Run() error {
 		}
 	}()
 
-	if err = CreateBootstrapControlPlane(b.assetDir, b.podManifestPath); err != nil {
+	if err = bcp.Start(); err != nil {
 		return err
 	}
 

pkg/bootkube/bootstrap.go

@@ -5,68 +5,62 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/kubernetes-incubator/bootkube/pkg/asset"
 )
 
-// CreateBootstrapControlPlane seeds static manifests to the kubelet to launch the bootstrap control
-// plane.
-func CreateBootstrapControlPlane(assetDir string, podManifestPath string) error {
-	UserOutput("Running temporary bootstrap control plane...\n")
+type bootstrapControlPlane struct {
+	assetDir        string
+	podManifestPath string
+	ownedManifests  []string
+}
+
+// NewBootstrapControlPlane constructs a new bootstrap control plane object.
+func NewBootstrapControlPlane(assetDir, podManifestPath string) *bootstrapControlPlane {
+	return &bootstrapControlPlane{
+		assetDir:        assetDir,
+		podManifestPath: podManifestPath,
+	}
+}
 
+// Start seeds static manifests to the kubelet to launch the bootstrap control plane.
+// Users should always ensure that Cleanup() is called even in the case of errors.
+func (b *bootstrapControlPlane) Start() error {
+	UserOutput("Starting temporary bootstrap control plane...\n")
 	// Make secrets temporarily available to bootstrap cluster.
 	if err := os.RemoveAll(asset.BootstrapSecretsDir); err != nil {
 		return err
 	}
-	if err := os.Mkdir(asset.BootstrapSecretsDir, os.FileMode(0700)); err != nil {
+	secretsDir := filepath.Join(b.assetDir, asset.AssetPathSecrets)
+	if _, err := copyDirectory(secretsDir, asset.BootstrapSecretsDir, true /* overwrite */); err != nil {
 		return err
 	}
-	secretsDir := filepath.Join(assetDir, asset.AssetPathSecrets)
-	secrets, err := ioutil.ReadDir(secretsDir)
-	if err != nil {
-		return err
-	}
-	for _, secret := range secrets {
-		if err := copyFile(filepath.Join(secretsDir, secret.Name()), filepath.Join(asset.BootstrapSecretsDir, secret.Name()), true); err != nil {
-			return err
-		}
-	}
-
 	// Copy the static manifests to the kubelet's pod manifest path.
-	manifestsDir := filepath.Join(assetDir, asset.AssetPathBootstrapManifests)
-	manifests, err := ioutil.ReadDir(manifestsDir)
-	if err != nil {
-		return err
-	}
-	for _, manifest := range manifests {
-		if err := copyFile(filepath.Join(manifestsDir, manifest.Name()), filepath.Join(podManifestPath, manifest.Name()), false); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	manifestsDir := filepath.Join(b.assetDir, asset.AssetPathBootstrapManifests)
+	ownedManifests, err := copyDirectory(manifestsDir, b.podManifestPath, false /* overwrite */)
+	b.ownedManifests = ownedManifests // always copy in case of partial failure.
+	return err
 }
 
-// CleanupBootstrapControlPlane brings down the bootstrap control plane and cleans up the temporary
+// Teardown brings down the bootstrap control plane and cleans up the temporary manifests and
 // secrets. This function is idempotent.
-func CleanupBootstrapControlPlane(assetDir string, podManifestPath string) error {
-	UserOutput("Cleaning up temporary bootstrap control plane...\n")
-
+func (b *bootstrapControlPlane) Teardown() error {
+	UserOutput("Tearing down temporary bootstrap control plane...\n")
 	if err := os.RemoveAll(asset.BootstrapSecretsDir); err != nil {
 		return err
 	}
-	manifests, err := ioutil.ReadDir(filepath.Join(assetDir, asset.AssetPathBootstrapManifests))
-	if err != nil {
-		return err
-	}
-	for _, manifest := range manifests {
-		if err := os.Remove(filepath.Join(podManifestPath, manifest.Name())); err != nil && !os.IsNotExist(err) {
+	for _, manifest := range b.ownedManifests {
+		if err := os.Remove(manifest); err != nil && !os.IsNotExist(err) {
 			return err
 		}
 	}
+	b.ownedManifests = nil
 	return nil
 }
 
+// copyFile copies a single file from src to dst. Returns an error if overwrite is true and dst
+// exists, or if any I/O error occurs during copying.
 func copyFile(src, dst string, overwrite bool) error {
 	if !overwrite {
 		fi, err := os.Stat(dst)
@@ -83,3 +77,27 @@ func copyFile(src, dst string, overwrite bool) error {
 	}
 	return ioutil.WriteFile(dst, data, os.FileMode(0600))
 }
+
+// copyDirectory copies srcDir to dstDir recursively. It returns the paths of files (not
+// directories) that were copied.
+func copyDirectory(srcDir, dstDir string, overwrite bool) ([]string, error) {
+	var copied []string
+	return copied, filepath.Walk(srcDir, func(src string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		dst := filepath.Join(dstDir, strings.TrimPrefix(src, srcDir))
+		if info.IsDir() {
+			err = os.Mkdir(dst, os.FileMode(0700))
+			if os.IsExist(err) {
+				err = nil
+			}
+			return err
+		}
+		if err := copyFile(src, dst, overwrite); err != nil {
+			return err
+		}
+		copied = append(copied, dst)
+		return nil
+	})
+}

pkg/bootkube/create.go

@@ -2,6 +2,7 @@ package bootkube
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -19,6 +20,11 @@ import (
 )
 
 func CreateAssets(manifestDir string, timeout time.Duration) error {
+	if _, err := os.Stat(manifestDir); os.IsNotExist(err) {
+		UserOutput(fmt.Sprintf("WARNING: %v does not exist, not creating any self-hosted assets.\n", manifestDir))
+		return nil
+	}
+
 	upFn := func() (bool, error) {
 		if err := apiTest(); err != nil {
 			glog.Warningf("Unable to determine api-server readiness: %v", err)