Merge #14021: Import key origin data through descriptors in importmulti

cb3511b Add release notes for importing key origin info change (Andrew Chow)
4c75a69 Test importing descriptors with key origin information (Andrew Chow)
02d6586 Import KeyOriginData when importing descriptors (Andrew Chow)
3d235df Implement a function to add KeyOriginInfo to a wallet (Andrew Chow)
eab63bc Store key origin info in key metadata (Andrew Chow)
345bff6 Remove hdmasterkeyid (Andrew Chow)
bac8c67 Add a method to CWallet to write just CKeyMetadata (Andrew Chow)
e7652d3 Add WriteHDKeypath function and move *HDKeypath to util/bip32.{h,cpp} (Andrew Chow)
c45415f Refactor keymetadata writing to a separate method (Andrew Chow)

Pull request description:

  This PR allows for key origin data as defined by the descriptors document to be imported to the wallet when importing a descriptor using `importmulti`. This allows the `walletprocesspsbt` to include the BIP 32 derivation paths for keys that it is watching that are from a different HD wallet.

  In order to make this easier to use, a new field `hdmasterkeyfingerprint` has been added to `getaddressinfo`. Additionally I have removed `hdmasterkeyid` as was planned. I think that this API change is fine since it was going to be removed in 0.18 anyways. `CKeyMetadata` has also been extended to store key origin info to facilitate this.

Tree-SHA512: 9c7794f3c793da57e23c5abbdc3d58779ee9dea3d53168bb86c0643a4ad5a11a446264961e2f772f35eea645048cb60954ed58050002caee4e43cd9f51215097

Author: meshcollider

doc/release-notes-14021.md

@@ -0,0 +1,11 @@
+Miscellaneous RPC Changes
+-------------------------
+- Descriptors with key origin information imported through `importmulti` will have their key origin information stored in the wallet for use with creating PSBTs.
+- If `bip32derivs` of both `walletprocesspsbt` and `walletcreatefundedpsbt` is set to true but the key metadata for a public key has not been updated yet, then that key will have a derivation path as if it were just an independent key (i.e. no derivation path and its master fingerprint is itself)
+
+Miscellaneous Wallet changes
+----------------------------
+
+- The key metadata will need to be upgraded the first time that the HD seed is available.
+For unencrypted wallets this will occur on wallet loading.
+For encrypted wallets this will occur the first time the wallet is unlocked.

src/Makefile.am

@@ -196,6 +196,7 @@ BITCOIN_CORE_H = \
   txmempool.h \
   ui_interface.h \
   undo.h \
+  util/bip32.h \
   util/bytevectorhash.h \
   util/system.h \
   util/memory.h \
@@ -456,6 +457,7 @@ libbitcoin_util_a_SOURCES = \
   support/cleanse.cpp \
   sync.cpp \
   threadinterrupt.cpp \
+  util/bip32.cpp \
   util/bytevectorhash.cpp \
   util/system.cpp \
   util/moneystr.cpp \

src/rpc/rawtransaction.cpp

@@ -26,6 +26,7 @@
 #include <script/sign.h>
 #include <script/standard.h>
 #include <uint256.h>
+#include <util/bip32.h>
 #include <util/strencodings.h>
 #include <validation.h>
 #include <validationinterface.h>

src/script/descriptor.cpp

@@ -10,6 +10,7 @@
 #include <script/standard.h>
 
 #include <span.h>
+#include <util/bip32.h>
 #include <util/system.h>
 #include <util/strencodings.h>
 
@@ -25,16 +26,6 @@ namespace {
 
 typedef std::vector<uint32_t> KeyPath;
 
-std::string FormatKeyPath(const KeyPath& path)
-{
-    std::string ret;
-    for (auto i : path) {
-        ret += strprintf("/%i", (i << 1) >> 1);
-        if (i >> 31) ret += '\'';
-    }
-    return ret;
-}
-
 /** Interface for public key objects in descriptors. */
 struct PubkeyProvider
 {
@@ -63,7 +54,7 @@ class OriginPubkeyProvider final : public PubkeyProvider
 
     std::string OriginString() const
     {
-        return HexStr(std::begin(m_origin.fingerprint), std::end(m_origin.fingerprint)) + FormatKeyPath(m_origin.path);
+        return HexStr(std::begin(m_origin.fingerprint), std::end(m_origin.fingerprint)) + FormatHDKeypath(m_origin.path);
     }
 
 public:
@@ -184,7 +175,7 @@ class BIP32PubkeyProvider final : public PubkeyProvider
     }
     std::string ToString() const override
     {
-        std::string ret = EncodeExtPubKey(m_extkey) + FormatKeyPath(m_path);
+        std::string ret = EncodeExtPubKey(m_extkey) + FormatHDKeypath(m_path);
         if (IsRange()) {
             ret += "/*";
             if (m_derive == DeriveType::HARDENED) ret += '\'';
@@ -195,7 +186,7 @@ class BIP32PubkeyProvider final : public PubkeyProvider
     {
         CExtKey key;
         if (!GetExtKey(arg, key)) return false;
-        out = EncodeExtKey(key) + FormatKeyPath(m_path);
+        out = EncodeExtKey(key) + FormatHDKeypath(m_path);
         if (IsRange()) {
             out += "/*";
             if (m_derive == DeriveType::HARDENED) out += '\'';

src/script/sign.h

@@ -22,13 +22,27 @@ struct CMutableTransaction;
 
 struct KeyOriginInfo
 {
-    unsigned char fingerprint[4];
+    unsigned char fingerprint[4]; //!< First 32 bits of the Hash160 of the public key at the root of the path
     std::vector<uint32_t> path;
 
     friend bool operator==(const KeyOriginInfo& a, const KeyOriginInfo& b)
     {
         return std::equal(std::begin(a.fingerprint), std::end(a.fingerprint), std::begin(b.fingerprint)) && a.path == b.path;
     }
+
+    ADD_SERIALIZE_METHODS;
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action)
+    {
+        READWRITE(fingerprint);
+        READWRITE(path);
+    }
+
+    void clear()
+    {
+        memset(fingerprint, 0, 4);
+        path.clear();
+    }
 };
 
 /** An interface to be implemented by keystores that support signing. */

src/util/bip32.cpp

@@ -0,0 +1,66 @@
+// Copyright (c) 2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <sstream>
+#include <stdio.h>
+#include <tinyformat.h>
+#include <util/bip32.h>
+#include <util/strencodings.h>
+
+
+bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath)
+{
+    std::stringstream ss(keypath_str);
+    std::string item;
+    bool first = true;
+    while (std::getline(ss, item, '/')) {
+        if (item.compare("m") == 0) {
+            if (first) {
+                first = false;
+                continue;
+            }
+            return false;
+        }
+        // Finds whether it is hardened
+        uint32_t path = 0;
+        size_t pos = item.find("'");
+        if (pos != std::string::npos) {
+            // The hardened tick can only be in the last index of the string
+            if (pos != item.size() - 1) {
+                return false;
+            }
+            path |= 0x80000000;
+            item = item.substr(0, item.size() - 1); // Drop the last character which is the hardened tick
+        }
+
+        // Ensure this is only numbers
+        if (item.find_first_not_of( "0123456789" ) != std::string::npos) {
+            return false;
+        }
+        uint32_t number;
+        if (!ParseUInt32(item, &number)) {
+            return false;
+        }
+        path |= number;
+
+        keypath.push_back(path);
+        first = false;
+    }
+    return true;
+}
+
+std::string FormatHDKeypath(const std::vector<uint32_t>& path)
+{
+    std::string ret;
+    for (auto i : path) {
+        ret += strprintf("/%i", (i << 1) >> 1);
+        if (i >> 31) ret += '\'';
+    }
+    return ret;
+}
+
+std::string WriteHDKeypath(const std::vector<uint32_t>& keypath)
+{
+    return "m" + FormatHDKeypath(keypath);
+}

src/util/bip32.h

@@ -0,0 +1,19 @@
+// Copyright (c) 2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_UTIL_BIP32_H
+#define BITCOIN_UTIL_BIP32_H
+
+#include <attributes.h>
+#include <string>
+#include <vector>
+
+/** Parse an HD keypaths like "m/7/0'/2000". */
+NODISCARD bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath);
+
+/** Write HD keypaths as strings */
+std::string WriteHDKeypath(const std::vector<uint32_t>& keypath);
+std::string FormatHDKeypath(const std::vector<uint32_t>& path);
+
+#endif // BITCOIN_UTIL_BIP32_H

src/util/strencodings.cpp

@@ -546,47 +546,6 @@ bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out)
     return true;
 }
 
-bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath)
-{
-    std::stringstream ss(keypath_str);
-    std::string item;
-    bool first = true;
-    while (std::getline(ss, item, '/')) {
-        if (item.compare("m") == 0) {
-            if (first) {
-                first = false;
-                continue;
-            }
-            return false;
-        }
-        // Finds whether it is hardened
-        uint32_t path = 0;
-        size_t pos = item.find("'");
-        if (pos != std::string::npos) {
-            // The hardened tick can only be in the last index of the string
-            if (pos != item.size() - 1) {
-                return false;
-            }
-            path |= 0x80000000;
-            item = item.substr(0, item.size() - 1); // Drop the last character which is the hardened tick
-        }
-
-        // Ensure this is only numbers
-        if (item.find_first_not_of( "0123456789" ) != std::string::npos) {
-            return false;
-        }
-        uint32_t number;
-        if (!ParseUInt32(item, &number)) {
-            return false;
-        }
-        path |= number;
-
-        keypath.push_back(path);
-        first = false;
-    }
-    return true;
-}
-
 void Downcase(std::string& str)
 {
     std::transform(str.begin(), str.end(), str.begin(), [](char c){return ToLower(c);});

src/util/strencodings.h

@@ -197,9 +197,6 @@ bool ConvertBits(const O& outfn, I it, I end) {
     return true;
 }
 
-/** Parse an HD keypaths like "m/7/0'/2000". */
-NODISCARD bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath);
-
 /**
  * Converts the given character to its lowercase equivalent.
  * This function is locale independent. It only converts uppercase

src/wallet/rpcdump.cpp

@@ -13,6 +13,7 @@
 #include <script/script.h>
 #include <script/standard.h>
 #include <sync.h>
+#include <util/bip32.h>
 #include <util/system.h>
 #include <util/time.h>
 #include <validation.h>
@@ -850,7 +851,7 @@ UniValue dumpwallet(const JSONRPCRequest& request)
             } else {
                 file << "change=1";
             }
-            file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].hdKeypath.size() > 0 ? " hdkeypath="+pwallet->mapKeyMetadata[keyid].hdKeypath : ""));
+            file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(pwallet->mapKeyMetadata[keyid].key_origin.path) : ""));
         }
     }
     file << "\n";
@@ -887,6 +888,7 @@ struct ImportData
     // Output data
     std::set<CScript> import_scripts;
     std::map<CKeyID, bool> used_keys; //!< Import these private keys if available (the value indicates whether if the key is required for solvability)
+    std::map<CKeyID, KeyOriginInfo> key_origins;
 };
 
 enum class ScriptContext
@@ -1157,7 +1159,7 @@ static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID
     }
 
     std::copy(out_keys.pubkeys.begin(), out_keys.pubkeys.end(), std::inserter(pubkey_map, pubkey_map.end()));
-
+    import_data.key_origins.insert(out_keys.origins.begin(), out_keys.origins.end());
     for (size_t i = 0; i < priv_keys.size(); ++i) {
         const auto& str = priv_keys[i].get_str();
         CKey key = DecodeSecret(str);
@@ -1260,6 +1262,11 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con
             if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) {
                 throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet");
             }
+            const auto& key_orig_it = import_data.key_origins.find(id);
+            if (key_orig_it != import_data.key_origins.end()) {
+                pwallet->AddKeyOrigin(pubkey, key_orig_it->second);
+            }
+            pwallet->mapKeyMetadata[id].nCreateTime = timestamp;
         }
 
         for (const CScript& script : script_pub_keys) {