partial Merge bitcoin#20267: Disable and fix tests for when BDB is not compiled

Backport notice: changes in feature_notification.py are missing due to bitcoin#18878 is not done yet

49797c3 tests: Disable bdb dump test when no bdb (Andrew Chow)
1194cf9 Fix wallet_send.py wallet setup to work with descriptors (Andrew Chow)
fbaea7b Require legacy wallet for wallet_upgradewallet.py (Andrew Chow)
b1b679e Explicitly mark legacy wallet tests as such (Andrew Chow)
09514e1 Setup wallets for interface_zmq.py (Andrew Chow)
4d03ef9 Use MiniWallet in rpc_net.py (Andrew Chow)
4de2382 Setup wallets for interface_bitcoin_cli.py (Andrew Chow)
7c71c62 Setup wallets with descriptors for feature_notifications (Andrew Chow)
1f1bef8 Have feature_filelock.py test both bdb and sqlite, depending on compiled (Andrew Chow)
c77975a Disable upgrades tests that require BDB if BDB is not compiled (Andrew Chow)
1f20cac Disable wallet_descriptor.py bdb format check if BDB is not compiled (Andrew Chow)
3641597 tests: Don't make any wallets unless wallet is required (Andrew Chow)
b9b88f5 Skip legacy wallet reliant tests if BDB is not compiled (Andrew Chow)
6f36242 tests: Set descriptors default based on compilation (Andrew Chow)

Pull request description:

  This PR fixes tests for when BDB is not compiled. Tests which rely on or test legacy wallet behavior are disabled and skipped when BDB is not compiled. For the components of some tests that are for legacy wallet things, those parts of the tests are skipped.

  For the majority of tests, changes are made so that they can be run with either legacy wallets or descriptor wallets without materially effecting the test. Most tests only need the wallet for balance and transactions, so the type of wallet is not an important part of those tests. Additionally, some tests are wallet agnostic and modified to instead use the test framework's MiniWallet.

ACKs for top commit:
  laanwj:
    ACK 49797c3
  ryanofsky:
    Code review ACK 49797c3. Only change since last review is dropping last commit. Previous review w/ suggestions for future followup is bitcoin#20267 (review)

Tree-SHA512: 69659f8a81fb437ecbca962f4082c12835282dbf1fba7d9952f727a49e01981d749af9b09feda1c8ca737516c7d7a08ef17e782795df3fa69892d5021b41c1ed

Author: laanwj

test/functional/feature_backwards_compatibility.py

@@ -350,51 +350,53 @@ def run_test(self):
         #v17_hdkeypath = v17_info["hdkeypath"]
         v17_pubkey = v17_info["pubkey"]
 
-        # Copy the 0.17 wallet to the last Bitcoin Core version and open it:
-        node_v17.unloadwallet("u1_v17")
-        shutil.copytree(
-            os.path.join(node_v17_wallets_dir, "u1_v17"),
-            os.path.join(node_master_wallets_dir, "u1_v17")
-        )
-        node_master.loadwallet("u1_v17")
-        wallet = node_master.get_wallet_rpc("u1_v17")
-        info = wallet.getaddressinfo(address)
-        # TODO enable back when HD wallets are created by default
-        #descriptor = "pkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v17_pubkey + ")"
-        #assert_equal(info["desc"], descsum_create(descriptor))
-        assert_equal(info["pubkey"], v17_pubkey)
-
-        # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it
-        node_master.unloadwallet("u1_v17")
-        shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17"))
-        shutil.copytree(
-            os.path.join(node_master_wallets_dir, "u1_v17"),
-            os.path.join(node_v17_wallets_dir, "u1_v17")
-        )
-        node_v17.loadwallet("u1_v17")
-        wallet = node_v17.get_wallet_rpc("u1_v17")
-        info = wallet.getaddressinfo(address)
-        assert_equal(info, v17_info)
-
-        # Copy the 0.19 wallet to the last Bitcoin Core version and open it:
-        shutil.copytree(
-            os.path.join(node_v19_wallets_dir, "w1_v19"),
-            os.path.join(node_master_wallets_dir, "w1_v19")
-        )
-        node_master.loadwallet("w1_v19")
-        wallet = node_master.get_wallet_rpc("w1_v19")
-        assert wallet.getaddressinfo(address_18075)["solvable"]
+        if self.is_bdb_compiled():
+            # Old wallets are BDB and will only work if BDB is compiled
+            # Copy the 0.16 wallet to the last Bitcoin Core version and open it:
+            node_v17.unloadwallet("u1_v17")
+            shutil.copytree(
+                os.path.join(node_v17_wallets_dir, "u1_v17"),
+                os.path.join(node_master_wallets_dir, "u1_v17")
+            )
+            node_master.loadwallet("u1_v17")
+            wallet = node_master.get_wallet_rpc("u1_v17")
+            info = wallet.getaddressinfo(address)
+            # TODO enable back when HD wallets are created by default
+            #descriptor = "pkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v17_pubkey + ")"
+            #assert_equal(info["desc"], descsum_create(descriptor))
+            assert_equal(info["pubkey"], v17_pubkey)
 
-        # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it
-        node_master.unloadwallet("w1_v19")
-        shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19"))
-        shutil.copytree(
-            os.path.join(node_master_wallets_dir, "w1_v19"),
-            os.path.join(node_v19_wallets_dir, "w1_v19")
-        )
-        node_v19.loadwallet("w1_v19")
-        wallet = node_v19.get_wallet_rpc("w1_v19")
-        assert wallet.getaddressinfo(address_18075)["solvable"]
+            # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it
+            node_master.unloadwallet("u1_v17")
+            shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17"))
+            shutil.copytree(
+                os.path.join(node_master_wallets_dir, "u1_v17"),
+                os.path.join(node_v17_wallets_dir, "u1_v17")
+            )
+            node_v17.loadwallet("u1_v17")
+            wallet = node_v17.get_wallet_rpc("u1_v17")
+            info = wallet.getaddressinfo(address)
+            assert_equal(info, v17_info)
+
+            # Copy the 0.19 wallet to the last Bitcoin Core version and open it:
+            shutil.copytree(
+                os.path.join(node_v19_wallets_dir, "w1_v19"),
+                os.path.join(node_master_wallets_dir, "w1_v19")
+            )
+            node_master.loadwallet("w1_v19")
+            wallet = node_master.get_wallet_rpc("w1_v19")
+            assert wallet.getaddressinfo(address_18075)["solvable"]
+
+            # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it
+            node_master.unloadwallet("w1_v19")
+            shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19"))
+            shutil.copytree(
+                os.path.join(node_master_wallets_dir, "w1_v19"),
+                os.path.join(node_v19_wallets_dir, "w1_v19")
+            )
+            node_v19.loadwallet("w1_v19")
+            wallet = node_v19.get_wallet_rpc("w1_v19")
+            assert wallet.getaddressinfo(address_18075)["solvable"]
 
 if __name__ == '__main__':
     BackwardsCompatibilityTest().main()

test/functional/feature_filelock.py

@@ -33,14 +33,21 @@ def run_test(self):
         self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg)
 
         if self.is_wallet_compiled():
-            wallet_name = ''.join([random.choice(string.ascii_lowercase) for _ in range(6)])
-            self.nodes[0].createwallet(wallet_name=wallet_name)
-            wallet_dir = os.path.join(datadir, 'wallets')
-            self.log.info("Check that we can't start a second dashd instance using the same wallet")
-            expected_msg = "Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another dashd?"
+            def check_wallet_filelock(descriptors):
+                wallet_name = ''.join([random.choice(string.ascii_lowercase) for _ in range(6)])
+                self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=descriptors)
+                wallet_dir = os.path.join(datadir, 'wallets')
+                self.log.info("Check that we can't start a second dashd instance using the same wallet")
+                if descriptors:
+                    expected_msg = "Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another dashd?"
+                else:
+                    expected_msg = "Error: Error initializing wallet database environment"
+                self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX)
+
             if self.is_bdb_compiled():
-                expected_msg = "Error: Error initializing wallet database environment"
-            self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX)
+                check_wallet_filelock(False)
+            if self.is_sqlite_compiled():
+                check_wallet_filelock(True)
 
 if __name__ == '__main__':
     FilelockTest().main()

test/functional/interface_zmq.py

@@ -42,6 +42,8 @@ def receive(self):
 class ZMQTest (BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 2
+        if self.is_wallet_compiled():
+            self.requires_wallet = True
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_py3_zmq()

test/functional/rpc_net.py

@@ -23,6 +23,7 @@
     assert_raises_rpc_error,
     p2p_port,
 )
+from test_framework.wallet import MiniWallet
 
 
 def assert_net_servicesnames(servicesflag, servicenames):
@@ -44,6 +45,9 @@ def set_test_params(self):
         self.supports_cli = False
 
     def run_test(self):
+        # We need miniwallet to make a transaction
+        self.wallet = MiniWallet(self.nodes[0])
+        self.wallet.generate(1)
         # Get out of IBD for the getpeerinfo tests.
         self.nodes[0].generate(101)
         # Wait for one ping/pong to finish so that we can be sure that there is no chatter between nodes for some time
@@ -163,8 +167,7 @@ def test_getaddednodeinfo(self):
     def test_getpeerinfo(self):
         self.log.info("Test getpeerinfo")
         # Create a few getpeerinfo last_block/last_transaction values.
-        if self.is_wallet_compiled():
-            self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1)
+        self.wallet.send_self_transfer(from_node=self.nodes[0]) # Make a transaction so we can see it in the getpeerinfo results
         self.nodes[1].generate(1)
         self.sync_all()
         time_now = self.mocktime

test/functional/test_framework/test_framework.py

@@ -222,9 +222,9 @@ def parse_args(self):
         parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts')
 
         group = parser.add_mutually_exclusive_group()
-        group.add_argument("--descriptors", default=False, action="store_true",
+        group.add_argument("--descriptors", action='store_const', const=True,
                             help="Run test using a descriptor wallet", dest='descriptors')
-        group.add_argument("--legacy-wallet", default=False, action="store_false",
+        group.add_argument("--legacy-wallet", action='store_const', const=False,
                             help="Run test using legacy wallets", dest='descriptors')
 
         self.add_options(parser)
@@ -240,6 +240,17 @@ def parse_args(self):
         # source: https://stackoverflow.com/questions/48796169/how-to-fix-ipykernel-launcher-py-error-unrecognized-arguments-in-jupyter/56349168#56349168
         parser.add_argument("-f", "--fff", help="a dummy argument to fool ipython", default="1")
 
+        if self.options.descriptors is None:
+            # Prefer BDB unless it isn't available
+            if self.is_bdb_compiled():
+                self.options.descriptors = False
+            elif self.is_sqlite_compiled():
+                self.options.descriptors = True
+            else:
+                # If neither are compiled, tests requiring a wallet will be skipped and the value of self.options.descriptors won't matter
+                # It still needs to exist and be None in order for tests to work however.
+                self.options.descriptors = None
+
         PortSeed.n = self.options.port_seed
 
     def setup(self):