Add comparison tool test runner, built on mininode

comptool.py creates a tool for running a test suite on top of the mininode p2p
framework.  It supports two types of tests: those for which we expect certain
behavior (acceptance or rejection of a block or transaction) and those for
which we are just comparing that the behavior of 2 or more nodes is the same.

blockstore.py defines BlockStore and TxStore, which provide db-backed maps
between block/tx hashes and the corresponding block or tx.

blocktools.py defines utility functions for creating and manipulating blocks
and transactions.

invalidblockrequest.py is an example test in the comptool framework, which
tests the behavior of a single node when sent two different types of invalid
blocks (a block with a duplicated transaction and a block with a bad coinbase
value).

Author: sdaftuar

qa/pull-tester/rpc-tests.sh

@@ -31,6 +31,7 @@ testScripts=(
     'merkle_blocks.py'
 #    'forknotify.py'
     'maxblocksinflight.py'
+    'invalidblockrequest.py'
 );
 if [ "x${ENABLE_BITCOIND}${ENABLE_UTILS}${ENABLE_WALLET}" = "x111" ]; then
     for (( i = 0; i < ${#testScripts[@]}; i++ ))

qa/rpc-tests/blockstore.py

@@ -0,0 +1,127 @@
+# BlockStore: a helper class that keeps a map of blocks and implements
+#             helper functions for responding to getheaders and getdata,
+#             and for constructing a getheaders message
+#
+
+from mininode import *
+import dbm
+
+class BlockStore(object):
+    def __init__(self, datadir):
+        self.blockDB = dbm.open(datadir + "/blocks", 'c')
+        self.currentBlock = 0L
+    
+    def close(self):
+        self.blockDB.close()
+
+    def get(self, blockhash):
+        serialized_block = None
+        try:
+            serialized_block = self.blockDB[repr(blockhash)]
+        except KeyError:
+            return None
+        f = cStringIO.StringIO(serialized_block)
+        ret = CBlock()
+        ret.deserialize(f)
+        ret.calc_sha256()
+        return ret
+
+    # Note: this pulls full blocks out of the database just to retrieve
+    # the headers -- perhaps we could keep a separate data structure
+    # to avoid this overhead.
+    def headers_for(self, locator, hash_stop, current_tip=None):
+        if current_tip is None:
+            current_tip = self.currentBlock
+        current_block = self.get(current_tip)
+        if current_block is None:
+            return None
+
+        response = msg_headers()
+        headersList = [ CBlockHeader(current_block) ]
+        maxheaders = 2000
+        while (headersList[0].sha256 not in locator.vHave):
+            prevBlockHash = headersList[0].hashPrevBlock
+            prevBlock = self.get(prevBlockHash)
+            if prevBlock is not None:
+                headersList.insert(0, CBlockHeader(prevBlock))
+            else:
+                break
+        headersList = headersList[:maxheaders] # truncate if we have too many
+        hashList = [x.sha256 for x in headersList]
+        index = len(headersList)
+        if (hash_stop in hashList):
+            index = hashList.index(hash_stop)+1
+        response.headers = headersList[:index]
+        return response
+
+    def add_block(self, block):
+        block.calc_sha256()
+        try:
+            self.blockDB[repr(block.sha256)] = bytes(block.serialize())
+        except TypeError as e:
+            print "Unexpected error: ", sys.exc_info()[0], e.args
+        self.currentBlock = block.sha256
+
+    def get_blocks(self, inv):
+        responses = []
+        for i in inv:
+            if (i.type == 2): # MSG_BLOCK
+                block = self.get(i.hash)
+                if block is not None:
+                    responses.append(msg_block(block))
+        return responses
+
+    def get_locator(self, current_tip=None):
+        if current_tip is None:
+            current_tip = self.currentBlock
+        r = []
+        counter = 0
+        step = 1
+        lastBlock = self.get(current_tip)
+        while lastBlock is not None:
+            r.append(lastBlock.hashPrevBlock)
+            for i in range(step):
+                lastBlock = self.get(lastBlock.hashPrevBlock)
+                if lastBlock is None:
+                    break
+            counter += 1
+            if counter > 10:
+                step *= 2
+        locator = CBlockLocator()
+        locator.vHave = r
+        return locator
+
+class TxStore(object):
+    def __init__(self, datadir):
+        self.txDB = dbm.open(datadir + "/transactions", 'c')
+
+    def close(self):
+        self.txDB.close()
+
+    def get(self, txhash):
+        serialized_tx = None
+        try:
+            serialized_tx = self.txDB[repr(txhash)]
+        except KeyError:
+            return None
+        f = cStringIO.StringIO(serialized_tx)
+        ret = CTransaction()
+        ret.deserialize(f)
+        ret.calc_sha256()
+        return ret
+
+    def add_transaction(self, tx):
+        tx.calc_sha256()
+        try:
+            self.txDB[repr(tx.sha256)] = bytes(tx.serialize())
+        except TypeError as e:
+            print "Unexpected error: ", sys.exc_info()[0], e.args
+
+    def get_transactions(self, inv):
+        responses = []
+        for i in inv:
+            if (i.type == 1): # MSG_TX
+                tx = self.get(i.hash)
+                if tx is not None:
+                    responses.append(msg_tx(tx))
+        return responses

qa/rpc-tests/blocktools.py

@@ -0,0 +1,65 @@
+# blocktools.py - utilities for manipulating blocks and transactions
+#
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+
+from mininode import *
+from script import CScript, CScriptOp
+
+# Create a block (with regtest difficulty)
+def create_block(hashprev, coinbase, nTime=None):
+    block = CBlock()
+    if nTime is None:
+        import time
+        block.nTime = int(time.time()+600)
+    else:
+        block.nTime = nTime
+    block.hashPrevBlock = hashprev
+    block.nBits = 0x207fffff # Will break after a difficulty adjustment...
+    block.vtx.append(coinbase)
+    block.hashMerkleRoot = block.calc_merkle_root()
+    block.calc_sha256()
+    return block
+
+def serialize_script_num(value):
+    r = bytearray(0)
+    if value == 0:
+        return r
+    neg = value < 0
+    absvalue = -value if neg else value
+    while (absvalue):
+        r.append(chr(absvalue & 0xff))
+        absvalue >>= 8
+    if r[-1] & 0x80:
+        r.append(0x80 if neg else 0)
+    elif neg:
+        r[-1] |= 0x80
+    return r
+
+counter=1
+# Create an anyone-can-spend coinbase transaction, assuming no miner fees
+def create_coinbase(heightAdjust = 0):
+    global counter
+    coinbase = CTransaction()
+    coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), 
+                ser_string(serialize_script_num(counter+heightAdjust)), 0xffffffff))
+    counter += 1
+    coinbaseoutput = CTxOut()
+    coinbaseoutput.nValue = 50*100000000
+    halvings = int((counter+heightAdjust)/150) # regtest
+    coinbaseoutput.nValue >>= halvings
+    coinbaseoutput.scriptPubKey = ""
+    coinbase.vout = [ coinbaseoutput ]
+    coinbase.calc_sha256()
+    return coinbase
+
+# Create a transaction with an anyone-can-spend output, that spends the
+# nth output of prevtx.
+def create_transaction(prevtx, n, sig, value):
+    tx = CTransaction()
+    assert(n < len(prevtx.vout))
+    tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), sig, 0xffffffff))
+    tx.vout.append(CTxOut(value, ""))
+    tx.calc_sha256()
+    return tx