Merge #16546: External signer support - Wallet Box edition

f75e0c1 doc: add external-signer.md (Sjors Provoost)
d4b0107 rpc: send: support external signer (Sjors Provoost)
245b445 rpc: signerdisplayaddress (Sjors Provoost)
7ebc7c0 wallet: ExternalSigner: add GetDescriptors method (Sjors Provoost)
fc5da52 wallet: add GetExternalSigner() (Sjors Provoost)
259f52c test: external_signer wallet flag is immutable (Sjors Provoost)
2655197 rpc: add external_signer option to createwallet (Sjors Provoost)
2700f09 rpc: signer: add enumeratesigners to list external signers (Sjors Provoost)
07b7c94 rpc: add external signer RPC files (Sjors Provoost)
8ce7767 wallet: add ExternalSignerScriptPubKeyMan (Sjors Provoost)
157ea7c wallet: add external_signer flag (Sjors Provoost)
f3e6ce7 test: add external signer test (Sjors Provoost)
8cf543f wallet: add -signer argument for external signer command (Sjors Provoost)
f7eb7ec test: framework: add skip_if_no_external_signer (Sjors Provoost)
87a9794 configure: add --enable-external-signer (Sjors Provoost)

Pull request description:

  Big picture overview in [this gist](https://gist.github.com/Sjors/29d06728c685e6182828c1ce9b74483d).

  This PR lets `bitcoind` call an arbitrary command `-signer=<cmd>`, e.g. a hardware wallet driver,  where it can fetch public keys, ask to display an address, and sign a transaction (using PSBT under the hood).

  It's design to work with https://github.com/bitcoin-core/HWI, which supports multiple hardware wallets. Any command with the same arguments and return values will work. It simplifies the manual procedure described [here](https://github.com/bitcoin-core/HWI/blob/master/docs/bitcoin-core-usage.md).

  Usage is documented in [doc/external-signer.md](
  https://github.com/Sjors/bitcoin/blob/2019/08/hww-box2/doc/external-signer.md), which also describes what protocol a different signer binary should conform to.

  Use `--enable-external-signer` to opt in, requires Boost::Process:

  ```
  Options used to compile and link:
    with wallet     = yes
    with gui / qt   = no
    external signer = yes
  ```

  It adds the following RPC methods:
  * `enumeratesigners`: asks <cmd> for a list of signers (e.g. devices) and their master key fingerprint
  * `signerdisplayaddress <address>`:  asks <cmd> to display an address

  It enhances the following RPC methods:
  * `createwallet`: takes an additional `external_signer` argument and fetches keys from device
  * `send`: automatically sends transaction to device and waits

  Usage TL&DR:
  * clone HWI repo somewhere and launch `bitcoind -signer=../HWI/hwi.py`
  * check if you can see your hardware device: `bitcoin-cli enumeratesigners`
  * create wallet and auto import keys `bitcoin-cli createwallet "hww" true true "" true true true`
  * display address on device: `bitcoin-cli signerdisplayaddress ...`
  * to spend, use `send` RPC and approve transaction on device

  Prerequisites:
  - [x] #21127 load wallet flags before everything else
  - [x] #21182 remove mostly pointless BOOST_PROCESS macro

  Potentially useful followups:
  - GUI support: bitcoin-core#4
  - bumpfee support
  - (automatically) verify (a subset of) keys on the device after import, through message signing

ACKs for top commit:
  laanwj:
    re-ACK f75e0c1

Tree-SHA512: 7db8afd54762295c1424c3f01d8c587ec256a72f34bd5256e04b21832dabd5dc212be8ab975ae3b67de75259fd569a561491945750492f417111dc7b6641e77f

Author: laanwj

build_msvc/bitcoin_config.h

@@ -50,8 +50,8 @@
 /* define if the Boost::Filesystem library is available */
 #define HAVE_BOOST_FILESYSTEM /**/
 
-/* define if the Boost::Process library is available */
-#define HAVE_BOOST_PROCESS /**/
+/* define if external signer support is enabled (requires Boost::Process) */
+#define ENABLE_EXTERNAL_SIGNER /**/
 
 /* define if the Boost::System library is available */
 #define HAVE_BOOST_SYSTEM /**/

configure.ac

@@ -338,10 +338,10 @@ AC_ARG_ENABLE([werror],
     [enable_werror=$enableval],
     [enable_werror=no])
 
-AC_ARG_WITH([boost-process],
-    [AS_HELP_STRING([--with-boost-process],[Opt in to using Boost Process (default is no)])],
-    [boost_process=$withval],
-    [boost_process=no])
+AC_ARG_ENABLE([external-signer],
+    [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is no, requires Boost::Process)])],
+    [use_external_signer=$enableval],
+    [use_external_signer=no])
 
 AC_LANG_PUSH([C++])
 
@@ -1253,6 +1253,7 @@ if test "x$enable_fuzz" = "xyes"; then
   bitcoin_enable_qt_dbus=no
   enable_wallet=no
   use_bench=no
+  use_external_signer=no
   use_upnp=no
   use_natpmp=no
   use_zmq=no
@@ -1390,16 +1391,20 @@ fi
 AX_BOOST_SYSTEM
 AX_BOOST_FILESYSTEM
 
-dnl Opt-in to Boost Process
-if test "x$boost_process" != xno; then
+dnl Opt-in to Boost Process if external signer support is requested
+if test "x$use_external_signer" != xno; then
 AC_MSG_CHECKING(for Boost Process)
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <boost/process.hpp>]],
  [[ boost::process::child* child = new boost::process::child; delete child; ]])],
- [ AC_MSG_RESULT(yes); AC_DEFINE([HAVE_BOOST_PROCESS],,[define if Boost::Process is available])],
- [ AC_MSG_ERROR([Boost::Process is not available!])]
+ [ AC_MSG_RESULT(yes)
+ AC_DEFINE([ENABLE_EXTERNAL_SIGNER],,[define if external signer support is enabled])
+ ],
+ [ AC_MSG_ERROR([Boost::Process is required for external signer support, but not available!])]
 )
 fi
 
+AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "x$use_external_signer" = "xyes"])
+
 if test x$suppress_external_warnings != xno; then
     BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS)
 fi
@@ -1810,6 +1815,7 @@ AC_SUBST(ARM_CRC_CXXFLAGS)
 AC_SUBST(LIBTOOL_APP_LDFLAGS)
 AC_SUBST(USE_SQLITE)
 AC_SUBST(USE_BDB)
+AC_SUBST(ENABLE_EXTERNAL_SIGNER)
 AC_SUBST(USE_UPNP)
 AC_SUBST(USE_QRCODE)
 AC_SUBST(BOOST_LIBS)
@@ -1885,43 +1891,43 @@ esac
 
 echo
 echo "Options used to compile and link:"
-echo "  boost process = $with_boost_process"
-echo "  multiprocess  = $build_multiprocess"
-echo "  with libs     = $build_bitcoin_libs"
-echo "  with wallet   = $enable_wallet"
+echo "  external signer = $use_external_signer"
+echo "  multiprocess    = $build_multiprocess"
+echo "  with libs       = $build_bitcoin_libs"
+echo "  with wallet     = $enable_wallet"
 if test "x$enable_wallet" != "xno"; then
-    echo "    with sqlite = $use_sqlite"
-    echo "    with bdb    = $use_bdb"
+    echo "    with sqlite   = $use_sqlite"
+    echo "    with bdb      = $use_bdb"
 fi
-echo "  with gui / qt = $bitcoin_enable_qt"
+echo "    with gui / qt = $bitcoin_enable_qt"
 if test x$bitcoin_enable_qt != xno; then
-    echo "    with qr     = $use_qr"
+    echo "  with qr         = $use_qr"
 fi
-echo "  with zmq      = $use_zmq"
+echo "  with zmq        = $use_zmq"
 if test x$enable_fuzz == xno; then
-    echo "  with test     = $use_tests"
+    echo "  with test       = $use_tests"
 else
-    echo "  with test     = not building test_bitcoin because fuzzing is enabled"
-    echo "    with fuzz   = $enable_fuzz"
+    echo "  with test       = not building test_bitcoin because fuzzing is enabled"
+    echo "    with fuzz     = $enable_fuzz"
 fi
-echo "  with bench    = $use_bench"
-echo "  with upnp     = $use_upnp"
-echo "  with natpmp   = $use_natpmp"
-echo "  use asm       = $use_asm"
-echo "  ebpf tracing  = $have_sdt"
-echo "  sanitizers    = $use_sanitizers"
-echo "  debug enabled = $enable_debug"
-echo "  gprof enabled = $enable_gprof"
-echo "  werror        = $enable_werror"
+echo "  with bench      = $use_bench"
+echo "  with upnp       = $use_upnp"
+echo "  with natpmp     = $use_natpmp"
+echo "  use asm         = $use_asm"
+echo "  ebpf tracing    = $have_sdt"
+echo "  sanitizers      = $use_sanitizers"
+echo "  debug enabled   = $enable_debug"
+echo "  gprof enabled   = $enable_gprof"
+echo "  werror          = $enable_werror"
 echo
-echo "  target os     = $TARGET_OS"
-echo "  build os      = $build_os"
+echo "  target os       = $TARGET_OS"
+echo "  build os        = $build_os"
 echo
-echo "  CC            = $CC"
-echo "  CFLAGS        = $PTHREAD_CFLAGS $CFLAGS"
-echo "  CPPFLAGS      = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS"
-echo "  CXX           = $CXX"
-echo "  CXXFLAGS      = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS"
-echo "  LDFLAGS       = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS"
-echo "  ARFLAGS       = $ARFLAGS"
+echo "  CC              = $CC"
+echo "  CFLAGS          = $PTHREAD_CFLAGS $CFLAGS"
+echo "  CPPFLAGS        = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS"
+echo "  CXX             = $CXX"
+echo "  CXXFLAGS        = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS"
+echo "  LDFLAGS         = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS"
+echo "  ARFLAGS         = $ARFLAGS"
 echo

doc/Doxyfile.in

@@ -2073,7 +2073,7 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             = HAVE_BOOST_PROCESS
+PREDEFINED             = ENABLE_EXTERNAL_SIGNER
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The

doc/external-signer.md

@@ -0,0 +1,171 @@
+# Support for signing transactions outside of Bitcoin Core
+
+Bitcoin Core can be launched with `-signer=<cmd>` where `<cmd>` is an external tool which can sign transactions and perform other functions. For example, it can be used to communicate with a hardware wallet.
+
+## Example usage
+
+The following example is based on the [HWI](https://github.com/bitcoin-core/HWI) tool. Although this tool is hosted under the Bitcoin Core GitHub organization and maintained by Bitcoin Core developers, it should be used with caution. It is considered experimental and has far less review than Bitcoin Core itself. Be particularly careful when running tools such as these on a computer with private keys on it.
+
+When using a hardware wallet, consult the manufacturer website for (alternative) software they recommend. As long as their software conforms to the standard below, it should be able to work with Bitcoin Core.
+
+Start Bitcoin Core:
+
+```sh
+$ bitcoind -signer=../HWI/hwi.py
+```
+
+### Device setup
+
+Follow the hardware manufacturers instructions for the initial device setup, as well as their instructions for creating a backup. Alternatively, for some devices, you can use the `setup`, `restore` and `backup` commands provided by [HWI](https://github.com/bitcoin-core/HWI).
+
+### Create wallet and import keys
+
+Get a list of signing devices / services:
+
+```
+$ bitcoin-cli enumeratesigners
+{
+  "signers": [
+    {
+      "fingerprint": "c8df832a"
+    }
+]
+```
+
+The master key fingerprint is used to identify a device.
+
+Create a wallet, this automatically imports the public keys:
+
+```sh
+$ bitcoin-cli createwallet "hww" true true "" true true true
+```
+
+### Verify an address
+
+Display an address on the device:
+
+```sh
+$ bitcoin-cli -rpcwallet=<wallet> getnewaddress
+$ bitcoin-cli -rpcwallet=<wallet> signerdisplayaddress <address>
+```
+
+Replace `<address>` with the result of `getnewaddress`.
+
+### Spending
+
+Under the hood this uses a [Partially Signed Bitcoin Transaction](psbt.md).
+
+```sh
+$ bitcoin-cli -rpcwallet=<wallet> sendtoaddress <address> <amount>
+```
+
+This prompts your hardware wallet to sign, and fail if it's not connected. If successful
+it automatically broadcasts the transaction.
+
+```sh
+{"complete": true, "txid": <txid>}
+```
+
+## Signer API
+
+In order to be compatible with Bitcoin Core any signer command should conform to the specification below. This specification is subject to change. Ideally a BIP should propose a standard so that other wallets can also make use of it.
+
+Prerequisite knowledge:
+* [Output Descriptors](descriptors.md)
+* Partially Signed Bitcoin Transaction ([PSBT](psbt.md))
+
+### `enumerate` (required)
+
+Usage:
+```
+$ <cmd> enumerate
+[
+    {
+        "fingerprint": "00000000"
+    }
+]
+```
+
+The command MUST return an (empty) array with at least a `fingerprint` field.
+
+A future extension could add an optional return field with device capabilities. Perhaps a descriptor with wildcards. For example: `["pkh("44'/0'/$'/{0,1}/*"), sh(wpkh("49'/0'/$'/{0,1}/*")), wpkh("84'/0'/$'/{0,1}/*")]`. This would indicate the device supports legacy, wrapped SegWit and native SegWit. In addition it restricts the derivation paths that can used for those, to maintain compatibility with other wallet software. It also indicates the device, or the driver, doesn't support multisig.
+
+A future extension could add an optional return field `reachable`, in case `<cmd>` knows a signer exists but can't currently reach it.
+
+### `signtransaction` (required)
+
+Usage:
+```
+$ <cmd> --fingerprint=<fingerprint> (--testnet) signtransaction <psbt>
+base64_encode_signed_psbt
+```
+
+The command returns a psbt with any signatures.
+
+The `psbt` SHOULD include bip32 derivations. The command SHOULD fail if none of the bip32 derivations match a key owned by the device.
+
+The command SHOULD fail if the user cancels.
+
+The command MAY complain if `--testnet` is set, but any of the BIP32 derivation paths contain a coin type other than `1h` (and vice versa).
+
+### `getdescriptors` (optional)
+
+Usage:
+
+```
+$ <cmd> --fingerprint=<fingerprint> (--testnet) getdescriptors <account>
+<xpub>
+```
+
+Returns descriptors supported by the device. Example:
+
+```
+$ <cmd> --fingerprint=00000000 --testnet getdescriptors
+{
+  "receive": [
+    "pkh([00000000/44h/0h/0h]xpub6C.../0/*)#fn95jwmg",
+    "sh(wpkh([00000000/49h/0h/0h]xpub6B..../0/*))#j4r9hntt",
+    "wpkh([00000000/84h/0h/0h]xpub6C.../0/*)#qw72dxa9"
+  ],
+  "internal": [
+    "pkh([00000000/44h/0h/0h]xpub6C.../1/*)#c8q40mts",
+    "sh(wpkh([00000000/49h/0h/0h]xpub6B..../1/*))#85dn0v75",
+    "wpkh([00000000/84h/0h/0h]xpub6C..../1/*)#36mtsnda"
+  ]
+}
+```
+
+### `displayaddress` (optional)
+
+Usage:
+```
+<cmd> --fingerprint=<fingerprint> (--testnet) displayaddress --desc descriptor
+```
+
+Example, display the first native SegWit receive address on Testnet:
+
+```
+<cmd> --fingerprint=00000000 --testnet displayaddress --desc "wpkh([00000000/84h/1h/0h]tpubDDUZ..../0/0)"
+```
+
+The command MUST be able to figure out the address type from the descriptor.
+
+If <descriptor> contains a master key fingerprint, the command MUST fail if it does not match the fingerprint known by the device.
+
+If <descriptor> contains an xpub, the command MUST fail if it does not match the xpub known by the device.
+
+The command MAY complain if `--testnet` is set, but the BIP32 coin type is not `1h` (and vice versa).
+
+## How Bitcoin Core uses the Signer API
+
+The `enumeratesigners` RPC simply calls `<cmd> enumerate`.
+
+The `createwallet` RPC calls:
+
+* `<cmd> --fingerprint=00000000 getdescriptors 0`
+
+It then imports descriptors for all support address types, in a BIP44/49/84 compatible manner.
+
+The `displayaddress` RPC reuses some code from `getaddressinfo` on the provided address and obtains the inferred descriptor. It then calls `<cmd> --fingerprint=00000000 displayaddress --desc=<descriptor>`.
+
+`sendtoaddress` and `sendmany` check `inputs->bip32_derivs` to see if any inputs have the same `master_fingerprint` as the signer. If so, it calls `<cmd> --fingerprint=00000000 signtransaction <psbt>`. It waits for the device to return a (partially) signed psbt, tries to finalize it and broadcasts the transation.

src/Makefile.am

@@ -265,10 +265,13 @@ BITCOIN_CORE_H = \
   wallet/crypter.h \
   wallet/db.h \
   wallet/dump.h \
+  wallet/external_signer.h \
+  wallet/external_signer_scriptpubkeyman.h \
   wallet/feebumper.h \
   wallet/fees.h \
   wallet/ismine.h \
   wallet/load.h \
+  wallet/rpcsigner.h \
   wallet/rpcwallet.h \
   wallet/salvage.h \
   wallet/scriptpubkeyman.h \
@@ -379,11 +382,14 @@ libbitcoin_wallet_a_SOURCES = \
   wallet/crypter.cpp \
   wallet/db.cpp \
   wallet/dump.cpp \
+  wallet/external_signer_scriptpubkeyman.cpp \
+  wallet/external_signer.cpp \
   wallet/feebumper.cpp \
   wallet/fees.cpp \
   wallet/interfaces.cpp \
   wallet/load.cpp \
   wallet/rpcdump.cpp \
+  wallet/rpcsigner.cpp \
   wallet/rpcwallet.cpp \
   wallet/scriptpubkeyman.cpp \
   wallet/wallet.cpp \

src/dummywallet.cpp

@@ -38,6 +38,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const
         "-paytxfee=<amt>",
         "-rescan",
         "-salvagewallet",
+        "-signer=<cmd>",
         "-spendzeroconfchange",
         "-txconfirmtarget=<n>",
         "-wallet=<path>",

src/rpc/client.cpp

@@ -183,6 +183,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "createwallet", 4, "avoid_reuse"},
     { "createwallet", 5, "descriptors"},
     { "createwallet", 6, "load_on_startup"},
+    { "createwallet", 7, "external_signer"},
     { "loadwallet", 1, "load_on_startup"},
     { "unloadwallet", 1, "load_on_startup"},
     { "getnodeaddresses", 0, "count"},