multi: add Vault support

Author: guggero

cmd_init_wallet.go

@@ -52,6 +52,15 @@ type secretSourceK8s struct {
 	Base64                bool   `long:"base64" description:"Encode as base64 when storing and decode as base64 when reading"`
 }
 
+type secretSourceVault struct {
+	AuthTokenPath           string `long:"auth-token-path" description:"The full path to the token file that should be used to authenticate against HashiCorp Vault"`
+	AuthRole                string `long:"auth-role" description:"The role to acquire when logging into HashiCorp Vault"`
+	SecretName              string `long:"secret-name" description:"The name of the Kubernetes secret"`
+	SeedKeyName             string `long:"seed-key-name" description:"The name of the entry within the secret that contains the seed"`
+	SeedPassphraseEntryName string `long:"seed-passphrase-entry-name" description:"The name of the entry within the secret that contains the seed passphrase"`
+	WalletPasswordEntryName string `long:"wallet-password-entry-name" description:"The name of the entry within the secret that contains the wallet password"`
+}
+
 type initTypeFile struct {
 	OutputWalletDir  string `long:"output-wallet-dir" description:"The directory in which the wallet.db file should be initialized"`
 	ValidatePassword bool   `long:"validate-password" description:"If a wallet file already exists in the output wallet directory, validate that it can be unlocked with the given password; this will try to decrypt the wallet and will take several seconds to complete"`
@@ -65,13 +74,14 @@ type initTypeRpc struct {
 }
 
 type initWalletCommand struct {
-	Network      string            `long:"network" description:"The Bitcoin network to initialize the wallet for, required for wallet internals" choice:"mainnet" choice:"testnet" choice:"testnet3" choice:"regtest" choice:"simnet"`
-	SecretSource string            `long:"secret-source" description:"Where to read the secrets from to initialize the wallet with" choice:"file" choice:"k8s"`
-	File         *secretSourceFile `group:"Flags for reading the secrets from files (use when --secret-source=file)" namespace:"file"`
-	K8s          *secretSourceK8s  `group:"Flags for reading the secrets from Kubernetes (use when --secret-source=k8s)" namespace:"k8s"`
-	InitType     string            `long:"init-type" description:"How to initialize the wallet" choice:"file" choice:"rpc"`
-	InitFile     *initTypeFile     `group:"Flags for initializing the wallet as a file (use when --init-type=file)" namespace:"init-file"`
-	InitRpc      *initTypeRpc      `group:"Flags for initializing the wallet through RPC (use when --init-type=rpc)" namespace:"init-rpc"`
+	Network      string             `long:"network" description:"The Bitcoin network to initialize the wallet for, required for wallet internals" choice:"mainnet" choice:"testnet" choice:"testnet3" choice:"regtest" choice:"simnet"`
+	SecretSource string             `long:"secret-source" description:"Where to read the secrets from to initialize the wallet with" choice:"file" choice:"k8s"`
+	File         *secretSourceFile  `group:"Flags for reading the secrets from files (use when --secret-source=file)" namespace:"file"`
+	K8s          *secretSourceK8s   `group:"Flags for reading the secrets from Kubernetes (use when --secret-source=k8s)" namespace:"k8s"`
+	Vault        *secretSourceVault `group:"Flags for reading the secrets from HashiCorp Vault (use when --secret-source=vault)" namespace:"vault"`
+	InitType     string             `long:"init-type" description:"How to initialize the wallet" choice:"file" choice:"rpc"`
+	InitFile     *initTypeFile      `group:"Flags for initializing the wallet as a file (use when --init-type=file)" namespace:"init-file"`
+	InitRpc      *initTypeRpc       `group:"Flags for initializing the wallet through RPC (use when --init-type=rpc)" namespace:"init-rpc"`
 }
 
 func newInitWalletCommand() *initWalletCommand {
@@ -82,6 +92,9 @@ func newInitWalletCommand() *initWalletCommand {
 		K8s: &secretSourceK8s{
 			Namespace: defaultK8sNamespace,
 		},
+		Vault: &secretSourceVault{
+			AuthTokenPath: defaultK8sServiceAccountTokenPath,
+		},
 		InitType: typeFile,
 		InitFile: &initTypeFile{},
 		InitRpc: &initTypeRpc{
@@ -248,8 +261,9 @@ func (x *initWalletCommand) readInput(requireSeed bool) (string, string, string,
 		}
 
 		if requireSeed {
-			logger.Infof("Reading seed from k8s secret %s (namespace %s)",
-				x.K8s.SecretName, x.K8s.Namespace)
+			logger.Infof("Reading seed from k8s secret %s "+
+				"(namespace %s)", x.K8s.SecretName,
+				x.K8s.Namespace)
 			seed, _, err = readK8s(k8sSecret)
 			if err != nil {
 				return "", "", "", err
@@ -258,8 +272,8 @@ func (x *initWalletCommand) readInput(requireSeed bool) (string, string, string,
 
 		// The seed passphrase is optional.
 		if x.K8s.SeedPassphraseKeyName != "" {
-			logger.Infof("Reading seed passphrase from k8s secret %s "+
-				"(namespace %s)", x.K8s.SecretName,
+			logger.Infof("Reading seed passphrase from k8s secret "+
+				"%s (namespace %s)", x.K8s.SecretName,
 				x.K8s.Namespace)
 			k8sSecret.KeyName = x.K8s.SeedPassphraseKeyName
 			seedPassPhrase, _, err = readK8s(k8sSecret)
@@ -268,13 +282,49 @@ func (x *initWalletCommand) readInput(requireSeed bool) (string, string, string,
 			}
 		}
 
-		logger.Infof("Reading wallet password from k8s secret %s (namespace %s)",
-			x.K8s.SecretName, x.K8s.Namespace)
+		logger.Infof("Reading wallet password from k8s secret %s "+
+			"(namespace %s)", x.K8s.SecretName, x.K8s.Namespace)
 		k8sSecret.KeyName = x.K8s.WalletPasswordKeyName
 		walletPassword, _, err = readK8s(k8sSecret)
 		if err != nil {
 			return "", "", "", err
 		}
+
+	// Read passphrase from HashiCorp Vault secret.
+	case storageVault:
+		vaultSecret := &vaultSecretOptions{
+			AuthTokenPath: x.Vault.AuthTokenPath,
+			AuthRole:      x.Vault.AuthRole,
+			SecretName:    x.Vault.SecretName,
+		}
+		vaultSecret.SecretKeyName = x.Vault.SeedKeyName
+
+		logger.Infof("Reading seed from vault secret %s",
+			x.Vault.SecretName)
+		seed, _, err = readVault(vaultSecret)
+		if err != nil {
+			return "", "", "", err
+		}
+
+		// The seed passphrase is optional.
+		if x.Vault.SeedPassphraseEntryName != "" {
+			logger.Infof("Reading seed passphrase from vault "+
+				"secret %s", x.Vault.SecretName)
+			vaultSecret.SecretKeyName =
+				x.Vault.SeedPassphraseEntryName
+			seedPassPhrase, _, err = readVault(vaultSecret)
+			if err != nil {
+				return "", "", "", err
+			}
+		}
+
+		logger.Infof("Reading wallet password from vault secret %s",
+			x.Vault.SecretName)
+		vaultSecret.SecretKeyName = x.Vault.WalletPasswordEntryName
+		walletPassword, _, err = readVault(vaultSecret)
+		if err != nil {
+			return "", "", "", err
+		}
 	}
 
 	// The seed, its passphrase and the wallet password should all never

cmd_load_secret.go

@@ -7,9 +7,10 @@ import (
 )
 
 type loadSecretCommand struct {
-	Source string            `long:"source" short:"s" description:"Secret storage source" choice:"k8s"`
-	K8s    *k8sSecretOptions `group:"Flags for looking up the secret as a value inside a Kubernetes Secret (use when --source=k8s)" namespace:"k8s"`
-	Output string            `long:"output" short:"o" description:"Output format" choice:"raw" choice:"json"`
+	Source string              `long:"source" short:"s" description:"Secret storage source" choice:"k8s"`
+	K8s    *k8sSecretOptions   `group:"Flags for looking up the secret as a value inside a Kubernetes Secret (use when --source=k8s)" namespace:"k8s"`
+	Vault  *vaultSecretOptions `group:"Flags for looking up the secret as a value inside a HashiCorp Vault (use when --source=vault)" namespace:"vault"`
+	Output string              `long:"output" short:"o" description:"Output format" choice:"raw" choice:"json"`
 }
 
 func newLoadSecretCommand() *loadSecretCommand {
@@ -18,6 +19,9 @@ func newLoadSecretCommand() *loadSecretCommand {
 		K8s: &k8sSecretOptions{
 			Namespace: defaultK8sNamespace,
 		},
+		Vault: &vaultSecretOptions{
+			AuthTokenPath: defaultK8sServiceAccountTokenPath,
+		},
 		Output: outputFormatRaw,
 	}
 }
@@ -70,6 +74,27 @@ func (x *loadSecretCommand) Execute(_ []string) error {
 
 		return nil
 
+	case storageVault:
+		content, secret, err := readVault(x.Vault)
+		if err != nil {
+			return fmt.Errorf("error reading secret %s from "+
+				"vault: %v", x.Vault.SecretName, err)
+		}
+
+		if x.Output == outputFormatJSON {
+			content, err = asJSON(&struct {
+				*jsonVaultObject `json:",inline"`
+				Value            string `json:"value"`
+			}{
+				jsonVaultObject: secret,
+				Value:           content,
+			})
+		}
+
+		fmt.Printf("%s\n", content)
+
+		return nil
+
 	default:
 		return fmt.Errorf("invalid secret storage source %s", x.Source)
 	}

cmd_store_secret.go

@@ -22,10 +22,11 @@ type entry struct {
 }
 
 type storeSecretCommand struct {
-	Batch     bool             `long:"batch" description:"Instead of reading one secret from stdin, read all files of the argument list and store them as entries in the secret"`
-	Overwrite bool             `long:"overwrite" description:"Overwrite existing secret entries instead of aborting"`
-	Target    string           `long:"target" short:"t" description:"Secret storage target" choice:"k8s"`
-	K8s       *targetK8sSecret `group:"Flags for storing the secret as a value inside a Kubernetes Secret (use when --target=k8s)" namespace:"k8s"`
+	Batch     bool                `long:"batch" description:"Instead of reading one secret from stdin, read all files of the argument list and store them as entries in the secret"`
+	Overwrite bool                `long:"overwrite" description:"Overwrite existing secret entries instead of aborting"`
+	Target    string              `long:"target" short:"t" description:"Secret storage target" choice:"k8s"`
+	K8s       *targetK8sSecret    `group:"Flags for storing the secret as a value inside a Kubernetes Secret (use when --target=k8s)" namespace:"k8s"`
+	Vault     *vaultSecretOptions `group:"Flags for storing the secret as a value inside a HashiCorp Vault (use when --target=vault)" namespace:"vault"`
 }
 
 func newStoreSecretCommand() *storeSecretCommand {
@@ -39,6 +40,9 @@ func newStoreSecretCommand() *storeSecretCommand {
 				ResourcePolicy: defaultK8sResourcePolicy,
 			},
 		},
+		Vault: &vaultSecretOptions{
+			AuthTokenPath: defaultK8sServiceAccountTokenPath,
+		},
 	}
 }
 
@@ -101,6 +105,15 @@ func (x *storeSecretCommand) Execute(args []string) error {
 
 		return storeSecretsK8s(entries, x.K8s, x.Overwrite)
 
+	case storageVault:
+		// Take the actual entry name from the options if we aren't in
+		// batch mode.
+		if len(entries) == 1 && entries[0].key == "" {
+			entries[0].key = x.Vault.SecretKeyName
+		}
+
+		return storeSecretsVault(entries, x.Vault, x.Overwrite)
+
 	default:
 		return fmt.Errorf("invalid secret storage target %s", x.Target)
 	}
@@ -138,3 +151,34 @@ func storeSecretsK8s(entries []*entry, opts *targetK8sSecret,
 
 	return nil
 }
+
+func storeSecretsVault(entries []*entry, opts *vaultSecretOptions,
+	overwrite bool) error {
+
+	if opts.SecretName == "" {
+		return fmt.Errorf("secret name is required")
+	}
+
+	for _, entry := range entries {
+		if entry.key == "" {
+			return fmt.Errorf("secret entry key is required")
+		}
+
+		entryOpts := &vaultSecretOptions{
+			AuthTokenPath: opts.AuthTokenPath,
+			AuthRole:      opts.AuthRole,
+			SecretName:    opts.SecretName,
+			SecretKeyName: entry.key,
+		}
+
+		logger.Infof("Storing entry with name %s to secret %s in vault",
+			entryOpts.SecretKeyName, entryOpts.SecretName)
+		err := saveVault(entry.value, entryOpts, overwrite)
+		if err != nil {
+			return fmt.Errorf("error storing secret %s entry %s: "+
+				"%v", opts.SecretName, entry.key, err)
+		}
+	}
+
+	return nil
+}

example-init-wallet-vault.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -e
+
+VAULT_AUTH_ROLE=${VAULT_AUTH_ROLE:-lnd-role}
+WALLET_SECRET_NAME=${WALLET_SECRET_NAME:-lnd-wallet-secret}
+WALLET_DIR=${WALLET_DIR:-~/.lnd/data/chain/bitcoin/mainnet}
+WALLET_PASSWORD_FILE=${WALLET_PASSWORD_FILE:-/tmp/wallet-password}
+CERT_DIR=${CERT_DIR:-~/.lnd}
+UPLOAD_RPC_SECRETS=${UPLOAD_RPC_SECRETS:-0}
+RPC_SECRETS_NAME=${RPC_SECRETS_NAME:-lnd-rpc-secret}
+
+echo "[STARTUP] Asserting wallet password exists in secret ${WALLET_SECRET_NAME}"
+lndinit gen-password \
+  | lndinit -v store-secret \
+  --target=vault \
+  --vault.auth-role="${VAULT_AUTH_ROLE}" \
+  --vault.secret-name="${WALLET_SECRET_NAME}" \
+  --vault.secret-key-name=wallet-password
+
+echo ""
+echo "[STARTUP] Asserting seed exists in secret ${WALLET_SECRET_NAME}"
+lndinit gen-seed \
+  | lndinit -v store-secret \
+  --target=vault \
+  --vault.auth-role="${VAULT_AUTH_ROLE}" \
+  --vault.secret-name="${WALLET_SECRET_NAME}" \
+  --vault.secret-key-name=seed
+
+echo ""
+echo "[STARTUP] Asserting wallet is created with values from secret ${WALLET_SECRET_NAME}"
+lndinit -v init-wallet \
+  --secret-source=vault \
+  --vault.auth-role="${VAULT_AUTH_ROLE}" \
+  --vault.secret-name="${WALLET_SECRET_NAME}" \
+  --vault.seed-key-name=seed \
+  --vault.wallet-password-entry-name=wallet-password \
+  --output-wallet-dir="${WALLET_DIR}" \
+  --validate-password
+
+echo ""
+echo "[STARTUP] Preparing lnd auto unlock file"
+
+# To make sure the password can be read exactly once (by lnd itself), we create
+# a named pipe. Because we can only write to such a pipe if there's a reader on
+# the other end, we need to run this in a sub process in the background.
+mkfifo "${WALLET_PASSWORD_FILE}"
+lndinit -v load-secret \
+  --source=vault \
+  --vault.auth-role="${VAULT_AUTH_ROLE}" \
+  --vault.secret-name="${WALLET_SECRET_NAME}" \
+  --vault.secret-key-name=wallet-password > "${WALLET_PASSWORD_FILE}" &
+
+# In case we want to upload the TLS certificate and macaroons to k8s secrets as
+# well once lnd is ready, we can use the wait-ready and store-secret commands in
+# combination to wait until lnd is ready and then batch upload the files to k8s.
+if [[ "${UPLOAD_RPC_SECRETS}" == "1" ]]; then
+  echo ""
+  echo "[STARTUP] Starting RPC secret uploader process in background"
+  lndinit -v wait-ready \
+    && lndinit -v store-secret \
+    --batch \
+    --overwrite \
+    --target=vault \
+    --vault.auth-role="${VAULT_AUTH_ROLE}" \
+    --vault.secret-name="${RPC_SECRETS_NAME}" \
+    "${CERT_DIR}/tls.cert" \
+    "${WALLET_DIR}"/*.macaroon &
+fi
+
+# And finally start lnd. We need to use "exec" here to make sure all signals are
+# forwarded correctly.
+echo ""
+echo "[STARTUP] Starting lnd with flags: $@"
+exec lnd "$@"