Merge #10704: [tests] nits in dbcrash.py

27c63dc [tests] nits in dbcrash.py (John Newbery)

Tree-SHA512: 2a75feeb65e6147e3337200cde982248bea8977a9585d5ee284d62bbc25f6d7c368754da0083aec37338c8f66cf698ee25bbd9e192df14a9fb976b8f75afa986

Author: MarcoFalke

test/functional/dbcrash.py

@@ -2,21 +2,7 @@
 # Copyright (c) 2017 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""Test recovery from a crash during chainstate writing."""
-
-from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import *
-from test_framework.script import *
-from test_framework.mininode import *
-import random
-try:
-    import http.client as httplib
-except ImportError:
-    import httplib
-import errno
-
-'''
-Test structure:
+"""Test recovery from a crash during chainstate writing.
 
 - 4 nodes
   * node0, node1, and node2 will have different dbcrash ratios, and different
@@ -37,11 +23,26 @@
        * submit block to node
        * if node crashed on/after submitting:
          - restart until recovery succeeds
-         - check that utxo matches node3 using gettxoutsetinfo
-'''
+         - check that utxo matches node3 using gettxoutsetinfo"""
 
-class ChainstateWriteCrashTest(BitcoinTestFramework):
+import errno
+import http.client
+import random
+import sys
+import time
+
+from test_framework.mininode import *
+from test_framework.script import *
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+
+HTTP_DISCONNECT_ERRORS = [http.client.CannotSendRequest]
+try:
+    HTTP_DISCONNECT_ERRORS.append(http.client.RemoteDisconnected)
+except AttributeError:
+    pass
 
+class ChainstateWriteCrashTest(BitcoinTestFramework):
     def __init__(self):
         super().__init__()
         self.num_nodes = 4
@@ -50,32 +51,28 @@ def __init__(self):
         # Set -maxmempool=0 to turn off mempool memory sharing with dbcache
         # Set -rpcservertimeout=900 to reduce socket disconnects in this
         # long-running test
-        self.base_args = ["-limitdescendantsize=0", "-maxmempool=0", "-rpcservertimeout=900"]
+        self.base_args = ["-limitdescendantsize=0", "-maxmempool=0", "-rpcservertimeout=900", "-dbbatchsize=200000"]
 
         # Set different crash ratios and cache sizes.  Note that not all of
         # -dbcache goes to pcoinsTip.
-        self.node0_args = ["-dbcrashratio=8", "-dbcache=4", "-dbbatchsize=200000"] + self.base_args
-        self.node1_args = ["-dbcrashratio=16", "-dbcache=8", "-dbbatchsize=200000"] + self.base_args
-        self.node2_args = ["-dbcrashratio=24", "-dbcache=16", "-dbbatchsize=200000"] + self.base_args
+        self.node0_args = ["-dbcrashratio=8", "-dbcache=4"] + self.base_args
+        self.node1_args = ["-dbcrashratio=16", "-dbcache=8"] + self.base_args
+        self.node2_args = ["-dbcrashratio=24", "-dbcache=16"] + self.base_args
 
         # Node3 is a normal node with default args, except will mine full blocks
         self.node3_args = ["-blockmaxweight=4000000"]
         self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args]
 
-        # We'll track some test coverage statistics
-        self.restart_counts = [0, 0, 0] # Track the restarts for nodes 0-2
-        self.crashed_on_restart = 0     # Track count of crashes during recovery
-
     def setup_network(self):
         self.setup_nodes()
         # Leave them unconnected, we'll use submitblock directly in this test
 
-    # Starts up a given node id, waits for the tip to reach the given block
-    # hash, and calculates the utxo hash.  Exceptions on startup should
-    # indicate node crash (due to -dbcrashratio), in which case we try again.
-    # Give up after 60 seconds.
-    # Returns the utxo hash of the given node.
     def restart_node(self, node_index, expected_tip):
+        """Start up a given node id, wait for the tip to reach the given block hash, and calculate the utxo hash.
+
+        Exceptions on startup should indicate node crash (due to -dbcrashratio), in which case we try again. Give up
+        after 60 seconds. Returns the utxo hash of the given node."""
+
         time_start = time.time()
         while time.time() - time_start < 60:
             try:
@@ -99,14 +96,23 @@ def restart_node(self, node_index, expected_tip):
         # and make sure that recovery happens.
         raise AssertionError("Unable to successfully restart node %d in allotted time", node_index)
 
-    # Try submitting a block to the given node.
-    # Catch any exceptions that indicate the node has crashed.
-    # Returns true if the block was submitted successfully; false otherwise.
     def submit_block_catch_error(self, node_index, block):
+        """Try submitting a block to the given node.
+
+        Catch any exceptions that indicate the node has crashed.
+        Returns true if the block was submitted successfully; false otherwise."""
+
         try:
             self.nodes[node_index].submitblock(block)
             return True
-        except (httplib.CannotSendRequest, httplib.RemoteDisconnected) as e:
+        except http.client.BadStatusLine as e:
+            # Prior to 3.5 BadStatusLine('') was raised for a remote disconnect error.
+            if sys.version_info[0] == 3 and sys.version_info[1] < 5 and e.line == "''":
+                self.log.debug("node %d submitblock raised exception: %s", node_index, e)
+                return False
+            else:
+                raise
+        except tuple(HTTP_DISCONNECT_ERRORS) as e:
             self.log.debug("node %d submitblock raised exception: %s", node_index, e)
             return False
         except OSError as e:
@@ -118,11 +124,13 @@ def submit_block_catch_error(self, node_index, block):
                 # Unexpected exception, raise
                 raise
 
-    # Use submitblock to sync node3's chain with the other nodes
-    # If submitblock fails, restart the node and get the new utxo hash.
     def sync_node3blocks(self, block_hashes):
-        # If any nodes crash while updating, we'll compare utxo hashes to
-        # ensure recovery was successful.
+        """Use submitblock to sync node3's chain with the other nodes
+
+        If submitblock fails, restart the node and get the new utxo hash.
+        If any nodes crash while updating, we'll compare utxo hashes to
+        ensure recovery was successful."""
+
         node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2']
 
         # Retrieve all the blocks from node3
@@ -161,9 +169,10 @@ def sync_node3blocks(self, block_hashes):
                 self.log.debug("Checking txoutsetinfo matches for node %d", i)
                 assert_equal(nodei_utxo_hash, node3_utxo_hash)
 
-    # Verify that the utxo hash of each node matches node3.
-    # Restart any nodes that crash while querying.
     def verify_utxo_hash(self):
+        """Verify that the utxo hash of each node matches node3.
+
+        Restart any nodes that crash while querying."""
         node3_utxo_hash = self.nodes[3].gettxoutsetinfo()['hash_serialized_2']
         self.log.info("Verifying utxo hash matches for all nodes")
 
@@ -175,9 +184,8 @@ def verify_utxo_hash(self):
                 nodei_utxo_hash = self.restart_node(i, self.nodes[3].getbestblockhash())
             assert_equal(nodei_utxo_hash, node3_utxo_hash)
 
-
     def generate_small_transactions(self, node, count, utxo_list):
-        FEE = 1000 # TODO: replace this with node relay fee based calculation
+        FEE = 1000  # TODO: replace this with node relay fee based calculation
         num_transactions = 0
         random.shuffle(utxo_list)
         while len(utxo_list) >= 2 and num_transactions < count:
@@ -186,8 +194,8 @@ def generate_small_transactions(self, node, count, utxo_list):
             for i in range(2):
                 utxo = utxo_list.pop()
                 tx.vin.append(CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout'])))
-                input_amount += int(utxo['amount']*COIN)
-            output_amount = (input_amount - FEE)//3
+                input_amount += int(utxo['amount'] * COIN)
+            output_amount = (input_amount - FEE) // 3
 
             if output_amount <= 0:
                 # Sanity check -- if we chose inputs that are too small, skip
@@ -202,6 +210,9 @@ def generate_small_transactions(self, node, count, utxo_list):
             num_transactions += 1
 
     def run_test(self):
+        # Track test coverage statistics
+        self.restart_counts = [0, 0, 0]  # Track the restarts for nodes 0-2
+        self.crashed_on_restart = 0      # Track count of crashes during recovery
 
         # Start by creating a lot of utxos on node3
         initial_height = self.nodes[3].getblockcount()
@@ -210,7 +221,7 @@ def run_test(self):
 
         # Sync these blocks with the other nodes
         block_hashes_to_sync = []
-        for height in range(initial_height+1, self.nodes[3].getblockcount()+1):
+        for height in range(initial_height + 1, self.nodes[3].getblockcount() + 1):
             block_hashes_to_sync.append(self.nodes[3].getblockhash(height))
 
         self.log.debug("Syncing %d blocks with other nodes", len(block_hashes_to_sync))
@@ -233,13 +244,15 @@ def run_test(self):
             if random_height > starting_tip_height:
                 # Randomly reorg from this point with some probability (1/4 for
                 # tip, 1/5 for tip-1, ...)
-                if random.random() < 1.0/(current_height + 4 - random_height):
+                if random.random() < 1.0 / (current_height + 4 - random_height):
                     self.log.debug("Invalidating block at height %d", random_height)
                     self.nodes[3].invalidateblock(self.nodes[3].getblockhash(random_height))
 
             # Now generate new blocks until we pass the old tip height
             self.log.debug("Mining longer tip")
-            block_hashes = self.nodes[3].generate(current_height+1-self.nodes[3].getblockcount())
+            block_hashes = []
+            while current_height + 1 > self.nodes[3].getblockcount():
+                block_hashes.extend(self.nodes[3].generate(min(10, current_height + 1 - self.nodes[3].getblockcount())))
             self.log.debug("Syncing %d new blocks...", len(block_hashes))
             self.sync_node3blocks(block_hashes)
             utxo_list = self.nodes[3].listunspent()

test/functional/test_framework/util.py

@@ -412,7 +412,10 @@ def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants):
 # Helper to create at least "count" utxos
 # Pass in a fee that is sufficient for relay and mining new transactions.
 def create_confirmed_utxos(fee, node, count):
-    node.generate(int(0.5 * count) + 101)
+    to_generate = int(0.5 * count) + 101
+    while to_generate > 0:
+        node.generate(min(25, to_generate))
+        to_generate -= 25
     utxos = node.listunspent()
     iterations = count - len(utxos)
     addr1 = node.getnewaddress()