Merge #15686: [tests] make pruning test faster

03d6d23 [tests] make pruning test faster (John Newbery)
1c29ac4 [tests] style fixes in feature_pruning.py (John Newbery)

Pull request description:

  This commit makes the pruning.py much faster.

  Key insights to do this:

  - pruning.py doesn't care what kind of transactions make up the big
  blocks that are pruned in the test. Instead of making blocks with
  several large, expensive to construct and validate transactions,
  instead make the large blocks contain a single coinbase transaction with
  a huge OP_RETURN txout.
  - avoid stop-starting nodes where possible.

ACKs for commit 03d6d2:
  MarcoFalke:
    utACK 03d6d23

Tree-SHA512: 511642ce0fa294319dce3486fe06d75970d8ab66deda7f692be081d3056b4ce5b4cf91a7b5762eefbba224ba6c848750016454ff1e5d564acc507b1c41213628

Author: MarcoFalke

test/functional/feature_pruning.py

@@ -8,11 +8,13 @@
 This test uses 4GB of disk space.
 This test takes 30 mins or more (up to 2 hours)
 """
+import os
 
+from test_framework.blocktools import create_coinbase
+from test_framework.messages import CBlock, ToHex
+from test_framework.script import CScript, OP_RETURN, OP_NOP
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, mine_large_block, sync_blocks, wait_until
-
-import os
+from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, disconnect_nodes, sync_blocks, wait_until
 
 MIN_BLOCKS_TO_KEEP = 288
 
@@ -21,19 +23,59 @@
 # compatible with pruning based on key creation time.
 TIMESTAMP_WINDOW = 2 * 60 * 60
 
+def mine_large_blocks(node, n):
+    # Make a large scriptPubKey for the coinbase transaction. This is OP_RETURN
+    # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase
+    # transaction but is consensus valid.
+
+    # Get the block parameters for the first block
+    big_script = CScript([OP_RETURN] + [OP_NOP] * 950000)
+    best_block = node.getblock(node.getbestblockhash())
+    height = int(best_block["height"]) + 1
+    try:
+        # Static variable ensures that time is monotonicly increasing and is therefore
+        # different for each block created => blockhash is unique.
+        mine_large_blocks.nTime = min(mine_large_blocks.nTime, int(best_block["time"])) + 1
+    except AttributeError:
+        mine_large_blocks.nTime = int(best_block["time"]) + 1
+    previousblockhash = int(best_block["hash"], 16)
+
+    for _ in range(n):
+        # Build the coinbase transaction (with large scriptPubKey)
+        coinbase_tx = create_coinbase(height)
+        coinbase_tx.vin[0].nSequence = 2 ** 32 - 1
+        coinbase_tx.vout[0].scriptPubKey = big_script
+        coinbase_tx.rehash()
+
+        # Build the block
+        block = CBlock()
+        block.nVersion = best_block["version"]
+        block.hashPrevBlock = previousblockhash
+        block.nTime = mine_large_blocks.nTime
+        block.nBits = int('207fffff', 16)
+        block.nNonce = 0
+        block.vtx = [coinbase_tx]
+        block.hashMerkleRoot = block.calc_merkle_root()
+        block.solve()
+
+        # Submit to the node
+        node.submitblock(ToHex(block))
+
+        previousblockhash = block.sha256
+        height += 1
+        mine_large_blocks.nTime += 1
 
 def calc_usage(blockdir):
-    return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.)
+    return sum(os.path.getsize(blockdir + f) for f in os.listdir(blockdir) if os.path.isfile(os.path.join(blockdir, f))) / (1024. * 1024.)
 
 class PruneTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
         self.num_nodes = 6
-        self.rpc_timeout = 900
 
         # Create nodes 0 and 1 to mine.
         # Create node 2 to test pruning.
-        self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000"]
+        self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"]
         # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
         # Create nodes 5 to test wallet in prune mode, but do not connect
         self.extra_args = [
@@ -55,7 +97,7 @@ def setup_network(self):
 
         connect_nodes(self.nodes[0], 1)
         connect_nodes(self.nodes[1], 2)
-        connect_nodes(self.nodes[2], 0)
+        connect_nodes(self.nodes[0], 2)
         connect_nodes(self.nodes[0], 3)
         connect_nodes(self.nodes[0], 4)
         sync_blocks(self.nodes[0:5])
@@ -71,30 +113,27 @@ def create_big_chain(self):
         self.nodes[1].generate(200)
         sync_blocks(self.nodes[0:2])
         self.nodes[0].generate(150)
+
         # Then mine enough full blocks to create more than 550MiB of data
-        for i in range(645):
-            mine_large_block(self.nodes[0], self.utxo_cache_0)
+        mine_large_blocks(self.nodes[0], 645)
 
         sync_blocks(self.nodes[0:5])
 
     def test_height_min(self):
-        if not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")):
-            raise AssertionError("blk00000.dat is missing, pruning too early")
+        assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early"
         self.log.info("Success")
         self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir))
         self.log.info("Mining 25 more blocks should cause the first block file to be pruned")
         # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
-        for i in range(25):
-            mine_large_block(self.nodes[0], self.utxo_cache_0)
+        mine_large_blocks(self.nodes[0], 25)
 
         # Wait for blk00000.dat to be pruned
         wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30)
 
         self.log.info("Success")
         usage = calc_usage(self.prunedir)
         self.log.info("Usage should be below target: %d" % usage)
-        if (usage > 550):
-            raise AssertionError("Pruning target not being met")
+        assert_greater_than(550, usage)
 
     def create_chain_with_staleblocks(self):
         # Create stale blocks in manageable sized chunks
@@ -103,90 +142,66 @@ def create_chain_with_staleblocks(self):
         for j in range(12):
             # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
             # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
-            # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine
-            self.stop_node(0)
-            self.start_node(0, extra_args=self.full_node_default_args)
+            disconnect_nodes(self.nodes[0], 1)
+            disconnect_nodes(self.nodes[0], 2)
             # Mine 24 blocks in node 1
-            for i in range(24):
-                if j == 0:
-                    mine_large_block(self.nodes[1], self.utxo_cache_1)
-                else:
-                    # Add node1's wallet transactions back to the mempool, to
-                    # avoid the mined blocks from being too small.
-                    self.nodes[1].resendwallettransactions()
-                    self.nodes[1].generate(1) #tx's already in mempool from previous disconnects
+            mine_large_blocks(self.nodes[1], 24)
 
             # Reorg back with 25 block chain from node 0
-            for i in range(25):
-                mine_large_block(self.nodes[0], self.utxo_cache_0)
+            mine_large_blocks(self.nodes[0], 25)
 
             # Create connections in the order so both nodes can see the reorg at the same time
-            connect_nodes(self.nodes[1], 0)
-            connect_nodes(self.nodes[2], 0)
+            connect_nodes(self.nodes[0], 1)
+            connect_nodes(self.nodes[0], 2)
             sync_blocks(self.nodes[0:3])
 
         self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir))
 
     def reorg_test(self):
         # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
         # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
-        # Reboot node 1 to clear its mempool (hopefully make the invalidate faster)
-        # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks)
-        self.stop_node(1)
-        self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"])
 
         height = self.nodes[1].getblockcount()
         self.log.info("Current block height: %d" % height)
 
-        invalidheight = height-287
-        badhash = self.nodes[1].getblockhash(invalidheight)
-        self.log.info("Invalidating block %s at height %d" % (badhash,invalidheight))
-        self.nodes[1].invalidateblock(badhash)
+        self.forkheight = height - 287
+        self.forkhash = self.nodes[1].getblockhash(self.forkheight)
+        self.log.info("Invalidating block %s at height %d" % (self.forkhash, self.forkheight))
+        self.nodes[1].invalidateblock(self.forkhash)
 
         # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want
         # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
-        mainchainhash = self.nodes[0].getblockhash(invalidheight - 1)
-        curhash = self.nodes[1].getblockhash(invalidheight - 1)
+        mainchainhash = self.nodes[0].getblockhash(self.forkheight - 1)
+        curhash = self.nodes[1].getblockhash(self.forkheight - 1)
         while curhash != mainchainhash:
             self.nodes[1].invalidateblock(curhash)
-            curhash = self.nodes[1].getblockhash(invalidheight - 1)
+            curhash = self.nodes[1].getblockhash(self.forkheight - 1)
 
-        assert self.nodes[1].getblockcount() == invalidheight - 1
+        assert self.nodes[1].getblockcount() == self.forkheight - 1
         self.log.info("New best height: %d" % self.nodes[1].getblockcount())
 
-        # Reboot node1 to clear those giant tx's from mempool
-        self.stop_node(1)
-        self.start_node(1, extra_args=["-maxreceivebuffer=20000","-blockmaxweight=20000", "-checkblocks=5"])
+        # Disconnect node1 and generate the new chain
+        disconnect_nodes(self.nodes[0], 1)
+        disconnect_nodes(self.nodes[1], 2)
 
         self.log.info("Generating new longer chain of 300 more blocks")
         self.nodes[1].generate(300)
 
         self.log.info("Reconnect nodes")
         connect_nodes(self.nodes[0], 1)
-        connect_nodes(self.nodes[2], 1)
+        connect_nodes(self.nodes[1], 2)
         sync_blocks(self.nodes[0:3], timeout=120)
 
         self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount())
-        self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir))
-
-        self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)")
+        self.log.info("Usage possibly still high because of stale blocks in block files: %d" % calc_usage(self.prunedir))
 
-        # Get node0's wallet transactions back in its mempool, to avoid the
-        # mined blocks from being too small.
-        self.nodes[0].resendwallettransactions()
+        self.log.info("Mine 220 more large blocks so we have requisite history")
 
-        for i in range(22):
-            # This can be slow, so do this in multiple RPC calls to avoid
-            # RPC timeouts.
-            self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects
-        sync_blocks(self.nodes[0:3], timeout=300)
+        mine_large_blocks(self.nodes[0], 220)
 
         usage = calc_usage(self.prunedir)
         self.log.info("Usage should be below target: %d" % usage)
-        if (usage > 550):
-            raise AssertionError("Pruning target not being met")
-
-        return invalidheight,badhash
+        assert_greater_than(550, usage)
 
     def reorg_back(self):
         # Verify that a block on the old main chain fork has been pruned away
@@ -219,17 +234,17 @@ def reorg_back(self):
             blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
             self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine)
             self.nodes[0].invalidateblock(curchainhash)
-            assert self.nodes[0].getblockcount() == self.mainchainheight
-            assert self.nodes[0].getbestblockhash() == self.mainchainhash2
+            assert_equal(self.nodes[0].getblockcount(), self.mainchainheight)
+            assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2)
             goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1]
             goalbestheight = first_reorg_height + 1
 
         self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload")
         # Wait for Node 2 to reorg to proper height
         wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900)
-        assert self.nodes[2].getbestblockhash() == goalbesthash
+        assert_equal(self.nodes[2].getbestblockhash(), goalbesthash)
         # Verify we can now have the data for a block previously pruned
-        assert self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight
+        assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight)
 
     def manual_test(self, node_number, use_timestamp):
         # at this point, node has 995 blocks and has not yet run in prune mode
@@ -287,38 +302,30 @@ def has_block(index):
 
         # height=100 too low to prune first block file so this is a no-op
         prune(100)
-        if not has_block(0):
-            raise AssertionError("blk00000.dat is missing when should still be there")
+        assert has_block(0), "blk00000.dat is missing when should still be there"
 
         # Does nothing
         node.pruneblockchain(height(0))
-        if not has_block(0):
-            raise AssertionError("blk00000.dat is missing when should still be there")
+        assert has_block(0), "blk00000.dat is missing when should still be there"
 
         # height=500 should prune first file
         prune(500)
-        if has_block(0):
-            raise AssertionError("blk00000.dat is still there, should be pruned by now")
-        if not has_block(1):
-            raise AssertionError("blk00001.dat is missing when should still be there")
+        assert not has_block(0), "blk00000.dat is still there, should be pruned by now"
+        assert has_block(1), "blk00001.dat is missing when should still be there"
 
         # height=650 should prune second file
         prune(650)
-        if has_block(1):
-            raise AssertionError("blk00001.dat is still there, should be pruned by now")
+        assert not has_block(1), "blk00001.dat is still there, should be pruned by now"
 
         # height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
         prune(1000, 1001 - MIN_BLOCKS_TO_KEEP)
-        if not has_block(2):
-            raise AssertionError("blk00002.dat is still there, should be pruned by now")
+        assert has_block(2), "blk00002.dat is still there, should be pruned by now"
 
         # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
         node.generate(288)
         prune(1000)
-        if has_block(2):
-            raise AssertionError("blk00002.dat is still there, should be pruned by now")
-        if has_block(3):
-            raise AssertionError("blk00003.dat is still there, should be pruned by now")
+        assert not has_block(2), "blk00002.dat is still there, should be pruned by now"
+        assert not has_block(3), "blk00003.dat is still there, should be pruned by now"
 
         # stop node, start back up with auto-prune at 550 MiB, make sure still runs
         self.stop_node(node_number)
@@ -339,21 +346,14 @@ def wallet_test(self):
         connect_nodes(self.nodes[0], 5)
         nds = [self.nodes[0], self.nodes[5]]
         sync_blocks(nds, wait=5, timeout=300)
-        self.stop_node(5) #stop and start to trigger rescan
+        self.stop_node(5)  # stop and start to trigger rescan
         self.start_node(5, extra_args=["-prune=550"])
         self.log.info("Success")
 
     def run_test(self):
-        self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
-        self.log.info("Mining a big blockchain of 995 blocks")
-
-        # Determine default relay fee
-        self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]
-
-        # Cache for utxos, as the listunspent may take a long time later in the test
-        self.utxo_cache_0 = []
-        self.utxo_cache_1 = []
+        self.log.info("Warning! This test requires 4GB of disk space")
 
+        self.log.info("Mining a big blockchain of 995 blocks")
         self.create_big_chain()
         # Chain diagram key:
         # *   blocks on main chain
@@ -394,11 +394,11 @@ def run_test(self):
         #                    +...+(1044)  &..                    $...$(1319)
 
         # Save some current chain state for later use
-        self.mainchainheight = self.nodes[2].getblockcount()   #1320
+        self.mainchainheight = self.nodes[2].getblockcount()  # 1320
         self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
 
         self.log.info("Check that we can survive a 288 block reorg still")
-        (self.forkheight,self.forkhash) = self.reorg_test() #(1033, )
+        self.reorg_test()  # (1033, )
         # Now create a 288 block reorg by mining a longer chain on N1
         # First disconnect N1
         # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain