*: add backup file option to recover subcommand

Author: xiang90

cmd/bootkube/recover.go

@@ -36,6 +36,8 @@ var (
 		etcdServers         string
 		etcdPrefix          string
 		kubeConfigPath      string
+		podManifestPath     string
+		etcdBackupFile      string
 	}
 )
 
@@ -46,8 +48,10 @@ func init() {
 	cmdRecover.Flags().StringVar(&recoverOpts.etcdCertificatePath, "etcd-certificate-path", "", "Path to an existing certificate that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-private-key-path, and must have etcd configured to use TLS with matching secrets.")
 	cmdRecover.Flags().StringVar(&recoverOpts.etcdPrivateKeyPath, "etcd-private-key-path", "", "Path to an existing private key that will be used for TLS-enabled communication between the apiserver and etcd. Must be used in conjunction with --etcd-ca-path and --etcd-certificate-path, and must have etcd configured to use TLS with matching secrets.")
 	cmdRecover.Flags().StringVar(&recoverOpts.etcdServers, "etcd-servers", "", "List of etcd server URLs including host:port, comma separated.")
+	cmdRecover.Flags().StringVar(&recoverOpts.etcdBackupFile, "etcd-backup-file", "", "Path to the etcd backup file.")
 	cmdRecover.Flags().StringVar(&recoverOpts.etcdPrefix, "etcd-prefix", "/registry", "Path prefix to Kubernetes cluster data in etcd.")
 	cmdRecover.Flags().StringVar(&recoverOpts.kubeConfigPath, "kubeconfig", "", "Path to kubeconfig for communicating with the cluster.")
+	cmdRecover.Flags().StringVar(&recoverOpts.podManifestPath, "pod-manifest-path", "/etc/kubernetes/manifests", "The location where the kubelet is configured to look for static pod manifests. (Only need to be set when recovering from a etcd backup file)")
 }
 
 func runCmdRecover(cmd *cobra.Command, args []string) error {
@@ -66,6 +70,31 @@ func runCmdRecover(cmd *cobra.Command, args []string) error {
 			return err
 		}
 		backend = recovery.NewEtcdBackend(etcdClient, recoverOpts.etcdPrefix)
+	case recoverOpts.etcdBackupFile != "":
+		bootkube.UserOutput("Attempting recovery using etcd backup file at %q...\n", recoverOpts.etcdBackupFile)
+
+		err := recovery.StartRecoveryEtcdForBackup(recoverOpts.podManifestPath, recoverOpts.etcdBackupFile)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			err := recovery.CleanRecoveryEtcd(recoverOpts.podManifestPath)
+			if err != nil {
+				bootkube.UserOutput("Failed to cleanup recovery etcd from the podManifestPath %v\n", recoverOpts.podManifestPath)
+			}
+		}()
+
+		bootkube.UserOutput("Waiting for etcd server to start...\n")
+		// TODO: better way to detect the startup...
+		time.Sleep(60 * time.Second)
+
+		recoverOpts.etcdServers = "http://localhost:32379"
+		etcdClient, err := createEtcdClient()
+		if err != nil {
+			return err
+		}
+		backend = recovery.NewEtcdBackend(etcdClient, recoverOpts.etcdPrefix)
+
 	default:
 		bootkube.UserOutput("Attempting recovery using apiserver at %q...\n", recoverOpts.kubeConfigPath)
 		backend, err = recovery.NewAPIServerBackend(recoverOpts.kubeConfigPath)

pkg/recovery/etcd.go

@@ -6,9 +6,12 @@ package recovery
 import (
 	"context"
 	"fmt"
+	"os"
 	"path"
 	"strings"
 
+	"github.com/kubernetes-incubator/bootkube/pkg/asset"
+
 	"github.com/coreos/etcd/clientv3"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -96,3 +99,28 @@ func (s *etcdBackend) list(ctx context.Context, key string, listObj runtime.Obje
 	}
 	return decodeList(elems, listPtr, s.decoder)
 }
+
+const assetPathRecoveryEtcd = "recovery-etcd.yaml"
+
+func StartRecoveryEtcdForBackup(p, backupPath string) error {
+	d, f := path.Split(backupPath)
+
+	config := struct {
+		Image      string
+		BackupFile string
+		BackupDir  string
+	}{
+		// TODO: this already exists in bootkube/cmd.
+		// do not duplicate this!
+		Image:      "quay.io/coreos/etcd:v3.1.6",
+		BackupFile: f,
+		BackupDir:  d,
+	}
+
+	as := asset.MustCreateAssetFromTemplate(assetPathRecoveryEtcd, RecoveryEtcdTemplate, config)
+	return as.WriteFile(p)
+}
+
+func CleanRecoveryEtcd(p string) error {
+	return os.Remove(path.Join(p, assetPathRecoveryEtcd))
+}

pkg/recovery/etcd_template.go

@@ -0,0 +1,57 @@
+package recovery
+
+var RecoveryEtcdTemplate = []byte(`apiVersion: v1
+kind: Pod
+metadata:
+  name: recovery-etcd
+  namespace: kube-system
+  labels:
+    k8s-app: recovery-etcd
+spec:
+  initContainers:
+  - name: recovery
+    image: {{ .Image }}
+    command: 
+    - /usr/local/bin/etcdctl
+    - snapshot
+    - restore
+    - --data-dir=/var/etcd/recovery
+    - --name=recovery-etcd
+    - --initial-cluster=recovery-etcd=http://localhost:32380
+    - --initial-cluster-token=bootkube-recovery
+    - --initial-advertise-peer-urls=http://localhost:32380
+    - --skip-hash-check=true
+    - /var/etcd-backupdir/{{ .BackupFile }}
+    env:
+    - name: ETCDCTL_API
+      value: "3"
+    volumeMounts:
+      - mountPath: /var/etcd
+        name: etcd
+        readOnly: false
+      - mountPath: /var/etcd-backupdir
+        name: etcdbackup
+        readOnly: false
+  containers:
+  - name: etcd
+    image: {{ .Image }}
+    command:
+    - /usr/local/bin/etcd
+    - --name=recovery-etcd
+    - --listen-client-urls=http://0.0.0.0:32379
+    - --listen-peer-urls=http://0.0.0.0:32380
+    - --advertise-client-urls=http://localhost:32379
+    - --data-dir=/var/etcd/recovery
+    volumeMounts:
+      - mountPath: /var/etcd
+        name: etcd
+        readOnly: false
+  hostNetwork: true
+  restartPolicy: Never
+  volumes:
+    - name: etcd
+      emptyDir: {}
+    - name: etcdbackup
+      hostPath:
+        path: {{ .BackupDir }}
+`)