Merge bitcoin#23866: test: use MiniWallet for rpc_scantxoutset.py

e844115 test: use MiniWallet for rpc_scantxoutset.py (Sebastian Falbesoner)
983ca04 test: introduce `address_to_scriptpubkey` helper (Sebastian Falbesoner)
e704d4d test: introduce `getnewdestination` helper for generating various address types (Sebastian Falbesoner)

Pull request description:

  This PR enables one more of the non-wallet functional tests (rpc_scantxoutset.py) to be run even with the Bitcoin Core wallet disabled by using the MiniWallet instead, as proposed in bitcoin#20078 and bitcoin#23858 (comment) more recently.

  Reviewer's guide:
  * [commit 1/3] For replacing the getnewaddress/getaddressinfo RPC calls a helper `getnewdestination` is introduced which allows to create addresses with the common address format types ('legacy', 'p2sh-segwit' and 'bech32'), but also additionally returns the corresponding pubkey and output script.
  * [commit 2/3] In order to send to addresses with MiniWallet, a helper `address_to_scriptpubkey` is introduced. It only supports legacy addresses (Base58Check) so far, which is sufficient for the scantxoutset test.
  * [commit 3/3] With those helpers, the use of MiniWallet in the test is quite straight-forward. To avoid repeatedly specifying parameters like `from_node` to MiniWallet's `send_to` method, another test-internal helper `sendtodestination` is introduced which supports specifying the destination both as outputscript or as address.

ACKs for top commit:
  w0xlt:
    reACK [e844115](bitcoin@e844115)

Tree-SHA512: e4823dc507019b2b8e479602963c5dddc4c78211e1d934218ee0f0e32c068ab7e44e9793307f549127946364f57d684c4ea1d70f25865ea70d30d4f3855836d0

Author: MarcoFalke

test/functional/p2p_filter.py

@@ -31,7 +31,7 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.wallet import (
     MiniWallet,
-    random_p2wpkh,
+    getnewdestination,
 )
 
 
@@ -169,14 +169,14 @@ def test_filter(self, filter_peer):
 
         self.log.info('Check that we only receive a merkleblock if the filter does not match a tx in a block')
         filter_peer.tx_received = False
-        block_hash = self.generatetoscriptpubkey(random_p2wpkh())
+        block_hash = self.generatetoscriptpubkey(getnewdestination()[1])
         filter_peer.wait_for_merkleblock(block_hash)
         assert not filter_peer.tx_received
 
         self.log.info('Check that we not receive a tx if the filter does not match a mempool tx')
         filter_peer.merkleblock_received = False
         filter_peer.tx_received = False
-        self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=random_p2wpkh(), amount=7 * COIN)
+        self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=7 * COIN)
         filter_peer.sync_send_with_ping()
         assert not filter_peer.merkleblock_received
         assert not filter_peer.tx_received
@@ -190,14 +190,14 @@ def test_filter(self, filter_peer):
         self.log.info('Check that after deleting filter all txs get relayed again')
         filter_peer.send_and_ping(msg_filterclear())
         for _ in range(5):
-            txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=random_p2wpkh(), amount=7 * COIN)
+            txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=7 * COIN)
             filter_peer.wait_for_tx(txid)
 
         self.log.info('Check that request for filtered blocks is ignored if no filter is set')
         filter_peer.merkleblock_received = False
         filter_peer.tx_received = False
         with self.nodes[0].assert_debug_log(expected_msgs=['received getdata']):
-            block_hash = self.generatetoscriptpubkey(random_p2wpkh())
+            block_hash = self.generatetoscriptpubkey(getnewdestination()[1])
             filter_peer.wait_for_inv([CInv(MSG_BLOCK, int(block_hash, 16))])
             filter_peer.sync_with_ping()
             assert not filter_peer.merkleblock_received

test/functional/rpc_scantxoutset.py

@@ -3,12 +3,16 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the scantxoutset rpc call."""
+from test_framework.messages import COIN
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, assert_raises_rpc_error
+from test_framework.wallet import (
+    MiniWallet,
+    address_to_scriptpubkey,
+    getnewdestination,
+)
 
 from decimal import Decimal
-import shutil
-import os
 
 
 def descriptors(out):
@@ -18,66 +22,55 @@ def descriptors(out):
 class ScantxoutsetTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
-        self.setup_clean_chain = True
 
-    def skip_test_if_missing_module(self):
-        self.skip_if_no_wallet()
+    def sendtodestination(self, destination, amount):
+        # interpret strings as addresses, assume scriptPubKey otherwise
+        if isinstance(destination, str):
+            destination = address_to_scriptpubkey(destination)
+        self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=destination, amount=int(COIN * amount))
 
     def run_test(self):
-        self.log.info("Mining blocks...")
-        self.generate(self.nodes[0], 110)
-
-        addr_P2SH_SEGWIT = self.nodes[0].getnewaddress("", "p2sh-segwit")
-        pubk1 = self.nodes[0].getaddressinfo(addr_P2SH_SEGWIT)['pubkey']
-        addr_LEGACY = self.nodes[0].getnewaddress("", "legacy")
-        pubk2 = self.nodes[0].getaddressinfo(addr_LEGACY)['pubkey']
-        addr_BECH32 = self.nodes[0].getnewaddress("", "bech32")
-        pubk3 = self.nodes[0].getaddressinfo(addr_BECH32)['pubkey']
-        txid = self.nodes[0].sendtoaddress(addr_P2SH_SEGWIT, 0.001)
-        self.nodes[0].lockunspent(unlock=False, transactions=[{"txid": txid, "vout": 0}, {"txid": txid, "vout": 1}])
-        txid = self.nodes[0].sendtoaddress(addr_LEGACY, 0.002)
-        self.nodes[0].lockunspent(unlock=False, transactions=[{"txid": txid, "vout": 0}, {"txid": txid, "vout": 1}])
-        txid = self.nodes[0].sendtoaddress(addr_BECH32, 0.004)
-        self.nodes[0].lockunspent(unlock=False, transactions=[{"txid": txid, "vout": 0}, {"txid": txid, "vout": 1}])
+        self.wallet = MiniWallet(self.nodes[0])
+        self.wallet.rescan_utxos()
+
+        self.log.info("Create UTXOs...")
+        pubk1, spk_P2SH_SEGWIT, addr_P2SH_SEGWIT = getnewdestination("p2sh-segwit")
+        pubk2, spk_LEGACY, addr_LEGACY = getnewdestination("legacy")
+        pubk3, spk_BECH32, addr_BECH32 = getnewdestination("bech32")
+        self.sendtodestination(spk_P2SH_SEGWIT, 0.001)
+        self.sendtodestination(spk_LEGACY, 0.002)
+        self.sendtodestination(spk_BECH32, 0.004)
 
         #send to child keys of tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK
-        self.nodes[0].sendtoaddress("mkHV1C6JLheLoUSSZYk7x3FH5tnx9bu7yc", 0.008)  # (m/0'/0'/0')
-        self.nodes[0].sendtoaddress("mipUSRmJAj2KrjSvsPQtnP8ynUon7FhpCR", 0.016)  # (m/0'/0'/1')
-        self.nodes[0].sendtoaddress("n37dAGe6Mq1HGM9t4b6rFEEsDGq7Fcgfqg", 0.032)  # (m/0'/0'/1500')
-        self.nodes[0].sendtoaddress("mqS9Rpg8nNLAzxFExsgFLCnzHBsoQ3PRM6", 0.064)  # (m/0'/0'/0)
-        self.nodes[0].sendtoaddress("mnTg5gVWr3rbhHaKjJv7EEEc76ZqHgSj4S", 0.128)  # (m/0'/0'/1)
-        self.nodes[0].sendtoaddress("mketCd6B9U9Uee1iCsppDJJBHfvi6U6ukC", 0.256)  # (m/0'/0'/1500)
-        self.nodes[0].sendtoaddress("mj8zFzrbBcdaWXowCQ1oPZ4qioBVzLzAp7", 0.512)  # (m/1/1/0')
-        self.nodes[0].sendtoaddress("mfnKpKQEftniaoE1iXuMMePQU3PUpcNisA", 1.024)  # (m/1/1/1')
-        self.nodes[0].sendtoaddress("mou6cB1kaP1nNJM1sryW6YRwnd4shTbXYQ", 2.048)  # (m/1/1/1500')
-        self.nodes[0].sendtoaddress("mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", 4.096)  # (m/1/1/0)
-        self.nodes[0].sendtoaddress("mxp7w7j8S1Aq6L8StS2PqVvtt4HGxXEvdy", 8.192)  # (m/1/1/1)
-        self.nodes[0].sendtoaddress("mpQ8rokAhp1TAtJQR6F6TaUmjAWkAWYYBq", 16.384)  # (m/1/1/1500)
+        self.sendtodestination("mkHV1C6JLheLoUSSZYk7x3FH5tnx9bu7yc", 0.008)  # (m/0'/0'/0')
+        self.sendtodestination("mipUSRmJAj2KrjSvsPQtnP8ynUon7FhpCR", 0.016)  # (m/0'/0'/1')
+        self.sendtodestination("n37dAGe6Mq1HGM9t4b6rFEEsDGq7Fcgfqg", 0.032)  # (m/0'/0'/1500')
+        self.sendtodestination("mqS9Rpg8nNLAzxFExsgFLCnzHBsoQ3PRM6", 0.064)  # (m/0'/0'/0)
+        self.sendtodestination("mnTg5gVWr3rbhHaKjJv7EEEc76ZqHgSj4S", 0.128)  # (m/0'/0'/1)
+        self.sendtodestination("mketCd6B9U9Uee1iCsppDJJBHfvi6U6ukC", 0.256)  # (m/0'/0'/1500)
+        self.sendtodestination("mj8zFzrbBcdaWXowCQ1oPZ4qioBVzLzAp7", 0.512)  # (m/1/1/0')
+        self.sendtodestination("mfnKpKQEftniaoE1iXuMMePQU3PUpcNisA", 1.024)  # (m/1/1/1')
+        self.sendtodestination("mou6cB1kaP1nNJM1sryW6YRwnd4shTbXYQ", 2.048)  # (m/1/1/1500')
+        self.sendtodestination("mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", 4.096)  # (m/1/1/0)
+        self.sendtodestination("mxp7w7j8S1Aq6L8StS2PqVvtt4HGxXEvdy", 8.192)  # (m/1/1/1)
+        self.sendtodestination("mpQ8rokAhp1TAtJQR6F6TaUmjAWkAWYYBq", 16.384)  # (m/1/1/1500)
 
         self.generate(self.nodes[0], 1)
 
-        self.log.info("Stop node, remove wallet, mine again some blocks...")
-        self.stop_node(0)
-        shutil.rmtree(os.path.join(self.nodes[0].datadir, self.chain, 'wallets'))
-        self.start_node(0, ['-nowallet'])
-        self.import_deterministic_coinbase_privkeys()
-        self.generate(self.nodes[0], 110)
-
         scan = self.nodes[0].scantxoutset("start", [])
         info = self.nodes[0].gettxoutsetinfo()
         assert_equal(scan['success'], True)
         assert_equal(scan['height'], info['height'])
         assert_equal(scan['txouts'], info['txouts'])
         assert_equal(scan['bestblock'], info['bestblock'])
 
-        self.restart_node(0, ['-nowallet'])
         self.log.info("Test if we have found the non HD unspent outputs.")
-        assert_equal(self.nodes[0].scantxoutset("start", ["pkh(" + pubk1 + ")", "pkh(" + pubk2 + ")", "pkh(" + pubk3 + ")"])['total_amount'], Decimal("0.002"))
-        assert_equal(self.nodes[0].scantxoutset("start", ["wpkh(" + pubk1 + ")", "wpkh(" + pubk2 + ")", "wpkh(" + pubk3 + ")"])['total_amount'], Decimal("0.004"))
-        assert_equal(self.nodes[0].scantxoutset("start", ["sh(wpkh(" + pubk1 + "))", "sh(wpkh(" + pubk2 + "))", "sh(wpkh(" + pubk3 + "))"])['total_amount'], Decimal("0.001"))
-        assert_equal(self.nodes[0].scantxoutset("start", ["combo(" + pubk1 + ")", "combo(" + pubk2 + ")", "combo(" + pubk3 + ")"])['total_amount'], Decimal("0.007"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["pkh(" + pubk1.hex() + ")", "pkh(" + pubk2.hex() + ")", "pkh(" + pubk3.hex() + ")"])['total_amount'], Decimal("0.002"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["wpkh(" + pubk1.hex() + ")", "wpkh(" + pubk2.hex() + ")", "wpkh(" + pubk3.hex() + ")"])['total_amount'], Decimal("0.004"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["sh(wpkh(" + pubk1.hex() + "))", "sh(wpkh(" + pubk2.hex() + "))", "sh(wpkh(" + pubk3.hex() + "))"])['total_amount'], Decimal("0.001"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["combo(" + pubk1.hex() + ")", "combo(" + pubk2.hex() + ")", "combo(" + pubk3.hex() + ")"])['total_amount'], Decimal("0.007"))
         assert_equal(self.nodes[0].scantxoutset("start", ["addr(" + addr_P2SH_SEGWIT + ")", "addr(" + addr_LEGACY + ")", "addr(" + addr_BECH32 + ")"])['total_amount'], Decimal("0.007"))
-        assert_equal(self.nodes[0].scantxoutset("start", ["addr(" + addr_P2SH_SEGWIT + ")", "addr(" + addr_LEGACY + ")", "combo(" + pubk3 + ")"])['total_amount'], Decimal("0.007"))
+        assert_equal(self.nodes[0].scantxoutset("start", ["addr(" + addr_P2SH_SEGWIT + ")", "addr(" + addr_LEGACY + ")", "combo(" + pubk3.hex() + ")"])['total_amount'], Decimal("0.007"))
 
         self.log.info("Test range validation.")
         assert_raises_rpc_error(-8, "End of range is too high", self.nodes[0].scantxoutset, "start", [{"desc": "desc", "range": -1}])

test/functional/test_framework/wallet.py

@@ -9,7 +9,13 @@
 from enum import Enum
 from random import choice
 from typing import Optional
-from test_framework.address import create_deterministic_address_bcrt1_p2tr_op_true
+from test_framework.address import (
+    base58_to_byte,
+    create_deterministic_address_bcrt1_p2tr_op_true,
+    key_to_p2pkh,
+    key_to_p2sh_p2wpkh,
+    key_to_p2wpkh,
+)
 from test_framework.descriptors import descsum_create
 from test_framework.key import ECKey
 from test_framework.messages import (
@@ -31,7 +37,11 @@
 )
 from test_framework.script_util import (
     key_to_p2pk_script,
+    key_to_p2pkh_script,
+    key_to_p2sh_p2wpkh_script,
     key_to_p2wpkh_script,
+    keyhash_to_p2pkh_script,
+    scripthash_to_p2sh_script,
 )
 from test_framework.util import (
     assert_equal,
@@ -209,12 +219,40 @@ def sendrawtransaction(self, *, from_node, tx_hex):
         return txid
 
 
-def random_p2wpkh():
-    """Generate a random P2WPKH scriptPubKey. Can be used when a random destination is needed,
-    but no compiled wallet is available (e.g. as replacement to the getnewaddress RPC)."""
+def getnewdestination(address_type='bech32'):
+    """Generate a random destination of the specified type and return the
+       corresponding public key, scriptPubKey and address. Supported types are
+       'legacy', 'p2sh-segwit' and 'bech32'. Can be used when a random
+       destination is needed, but no compiled wallet is available (e.g. as
+       replacement to the getnewaddress/getaddressinfo RPCs)."""
     key = ECKey()
     key.generate()
-    return key_to_p2wpkh_script(key.get_pubkey().get_bytes())
+    pubkey = key.get_pubkey().get_bytes()
+    if address_type == 'legacy':
+        scriptpubkey = key_to_p2pkh_script(pubkey)
+        address = key_to_p2pkh(pubkey)
+    elif address_type == 'p2sh-segwit':
+        scriptpubkey = key_to_p2sh_p2wpkh_script(pubkey)
+        address = key_to_p2sh_p2wpkh(pubkey)
+    elif address_type == 'bech32':
+        scriptpubkey = key_to_p2wpkh_script(pubkey)
+        address = key_to_p2wpkh(pubkey)
+    # TODO: also support bech32m (need to generate x-only-pubkey)
+    else:
+        assert False
+    return pubkey, scriptpubkey, address
+
+
+def address_to_scriptpubkey(address):
+    """Converts a given address to the corresponding output script (scriptPubKey)."""
+    payload, version = base58_to_byte(address)
+    if version == 111:  # testnet pubkey hash
+        return keyhash_to_p2pkh_script(payload)
+    elif version == 196:  # testnet script hash
+        return scripthash_to_p2sh_script(payload)
+    # TODO: also support other address formats
+    else:
+        assert False
 
 
 def make_chain(node, address, privkeys, parent_txid, parent_value, n=0, parent_locking_script=None, fee=DEFAULT_FEE):