Fix regression tests

Taught bitcoind to close the HTTP connection after it gets a 'stop' command,
to make it easier for the regression tests to cleanly stop.
Move bitcoinrpc files to correct location.
Tidied up the python-based regression tests.

Author: gavinandresen

qa/rpc-tests/listtransactions.py

@@ -118,6 +118,7 @@ def main():
     check_json_precision()
 
     success = False
+    nodes = []
     try:
         print("Initializing test directory "+options.tmpdir)
         if not os.path.isdir(options.tmpdir):
@@ -127,6 +128,7 @@ def main():
         nodes = start_nodes(2, options.tmpdir)
         connect_nodes(nodes[1], 0)
         sync_blocks(nodes)
+
         run_test(nodes)
 
         success = True
@@ -135,12 +137,12 @@ def main():
         print("Assertion failed: "+e.message)
     except Exception as e:
         print("Unexpected exception caught during testing: "+str(e))
-        stack = traceback.extract_tb(sys.exc_info()[2])
-        print(stack[-1])
+        traceback.print_tb(sys.exc_info()[2])
 
     if not options.nocleanup:
         print("Cleaning up")
-        stop_nodes()
+        stop_nodes(nodes)
+        wait_bitcoinds()
         shutil.rmtree(options.tmpdir)
 
     if success:

qa/rpc-tests/python-bitcoinrpc/bitcoinrpc/.gitignore

@@ -0,0 +1 @@
+*.pyc

qa/rpc-tests/python-bitcoinrpc/bitcoinrpc/__init__.py

@@ -0,0 +1,140 @@
+
+"""
+  Copyright 2011 Jeff Garzik
+
+  AuthServiceProxy has the following improvements over python-jsonrpc's
+  ServiceProxy class:
+
+  - HTTP connections persist for the life of the AuthServiceProxy object
+    (if server supports HTTP/1.1)
+  - sends protocol 'version', per JSON-RPC 1.1
+  - sends proper, incrementing 'id'
+  - sends Basic HTTP authentication headers
+  - parses all JSON numbers that look like floats as Decimal
+  - uses standard Python json lib
+
+  Previous copyright, from python-jsonrpc/jsonrpc/proxy.py:
+
+  Copyright (c) 2007 Jan-Klaas Kollhof
+
+  This file is part of jsonrpc.
+
+  jsonrpc is free software; you can redistribute it and/or modify
+  it under the terms of the GNU Lesser General Public License as published by
+  the Free Software Foundation; either version 2.1 of the License, or
+  (at your option) any later version.
+
+  This software is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public License
+  along with this software; if not, write to the Free Software
+  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+try:
+    import http.client as httplib
+except ImportError:
+    import httplib
+import base64
+import json
+import decimal
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urlparse
+
+USER_AGENT = "AuthServiceProxy/0.1"
+
+HTTP_TIMEOUT = 30
+
+
+class JSONRPCException(Exception):
+    def __init__(self, rpc_error):
+        Exception.__init__(self)
+        self.error = rpc_error
+
+
+class AuthServiceProxy(object):
+    def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None):
+        self.__service_url = service_url
+        self.__service_name = service_name
+        self.__url = urlparse.urlparse(service_url)
+        if self.__url.port is None:
+            port = 80
+        else:
+            port = self.__url.port
+        self.__id_count = 0
+        (user, passwd) = (self.__url.username, self.__url.password)
+        try:
+            user = user.encode('utf8')
+        except AttributeError:
+            pass
+        try:
+            passwd = passwd.encode('utf8')
+        except AttributeError:
+            pass
+        authpair = user + b':' + passwd
+        self.__auth_header = b'Basic ' + base64.b64encode(authpair)
+
+        if connection:
+            # Callables re-use the connection of the original proxy
+            self.__conn = connection
+        elif self.__url.scheme == 'https':
+            self.__conn = httplib.HTTPSConnection(self.__url.hostname, port,
+                                                  None, None, False,
+                                                  timeout)
+        else:
+            self.__conn = httplib.HTTPConnection(self.__url.hostname, port,
+                                                 False, timeout)
+
+    def __getattr__(self, name):
+        if name.startswith('__') and name.endswith('__'):
+            # Python internal stuff
+            raise AttributeError
+        if self.__service_name is not None:
+            name = "%s.%s" % (self.__service_name, name)
+        return AuthServiceProxy(self.__service_url, name, connection=self.__conn)
+
+    def __call__(self, *args):
+        self.__id_count += 1
+
+        postdata = json.dumps({'version': '1.1',
+                               'method': self.__service_name,
+                               'params': args,
+                               'id': self.__id_count})
+        self.__conn.request('POST', self.__url.path, postdata,
+                            {'Host': self.__url.hostname,
+                             'User-Agent': USER_AGENT,
+                             'Authorization': self.__auth_header,
+                             'Content-type': 'application/json'})
+
+        response = self._get_response()
+        if response['error'] is not None:
+            raise JSONRPCException(response['error'])
+        elif 'result' not in response:
+            raise JSONRPCException({
+                'code': -343, 'message': 'missing JSON-RPC result'})
+        else:
+            return response['result']
+
+    def _batch(self, rpc_call_list):
+        postdata = json.dumps(list(rpc_call_list))
+        self.__conn.request('POST', self.__url.path, postdata,
+                            {'Host': self.__url.hostname,
+                             'User-Agent': USER_AGENT,
+                             'Authorization': self.__auth_header,
+                             'Content-type': 'application/json'})
+
+        return self._get_response()
+
+    def _get_response(self):
+        http_response = self.__conn.getresponse()
+        if http_response is None:
+            raise JSONRPCException({
+                'code': -342, 'message': 'missing HTTP response from server'})
+
+        return json.loads(http_response.read().decode('utf8'),
+                          parse_float=decimal.Decimal)

qa/rpc-tests/python-bitcoinrpc/bitcoinrpc/authproxy.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+
+setup(name='python-bitcoinrpc',
+      version='0.1',
+      description='Enhanced version of python-jsonrpc for use with Bitcoin',
+      long_description=open('README').read(),
+      author='Jeff Garzik',
+      author_email='<[email protected]>',
+      maintainer='Jeff Garzik',
+      maintainer_email='<[email protected]>',
+      url='http://www.github.com/jgarzik/python-bitcoinrpc',
+      packages=['bitcoinrpc'],
+      classifiers=['License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)', 'Operating System :: OS Independent'])

qa/rpc-tests/python-bitcoinrpc/setup.py

@@ -45,6 +45,7 @@ def main():
     check_json_precision()
 
     success = False
+    nodes = []
     try:
         print("Initializing test directory "+options.tmpdir)
         if not os.path.isdir(options.tmpdir):
@@ -63,12 +64,12 @@ def main():
         print("Assertion failed: "+e.message)
     except Exception as e:
         print("Unexpected exception caught during testing: "+str(e))
-        stack = traceback.extract_tb(sys.exc_info()[2])
-        print(stack[-1])
+        traceback.print_tb(sys.exc_info()[2])
 
     if not options.nocleanup:
         print("Cleaning up")
-        stop_nodes()
+        stop_nodes(nodes)
+        wait_bitcoinds()
         shutil.rmtree(options.tmpdir)
 
     if success:

qa/rpc-tests/skeleton.py

@@ -55,6 +55,8 @@ def sync_mempools(rpc_connections):
         time.sleep(1)
 
 
+bitcoind_processes = []
+
 def initialize_chain(test_dir):
     """
     Create (or copy from cache) a 200-block-long chain and
@@ -64,7 +66,6 @@ def initialize_chain(test_dir):
 
     if not os.path.isdir(os.path.join("cache", "node0")):
         # Create cache directories, run bitcoinds:
-        bitcoinds = []
         for i in range(4):
             datadir = os.path.join("cache", "node"+str(i))
             os.makedirs(datadir)
@@ -77,7 +78,7 @@ def initialize_chain(test_dir):
             args = [ "bitcoind", "-keypool=1", "-datadir="+datadir ]
             if i > 0:
                 args.append("-connect=127.0.0.1:"+str(START_P2P_PORT))
-            bitcoinds.append(subprocess.Popen(args))
+            bitcoind_processes.append(subprocess.Popen(args))
             subprocess.check_output([ "bitcoin-cli", "-datadir="+datadir,
                                       "-rpcwait", "getblockcount"])
 
@@ -90,8 +91,6 @@ def initialize_chain(test_dir):
                 sys.stderr.write("Error connecting to "+url+"\n")
                 sys.exit(1)
 
-        import pdb; pdb.set_trace()
-
         # Create a 200-block-long chain; each of the 4 nodes
         # gets 25 mature blocks and 25 immature.
         for i in range(4):
@@ -100,17 +99,18 @@ def initialize_chain(test_dir):
         for i in range(4):
             rpcs[i].setgenerate(True, 25)
             sync_blocks(rpcs)
-        # Shut them down
+
+        # Shut them down, and remove debug.logs:
+        stop_nodes(rpcs)
+        wait_bitcoinds()
         for i in range(4):
-            rpcs[i].stop()
+            os.remove(debug_log("cache", i))
 
     for i in range(4):
         from_dir = os.path.join("cache", "node"+str(i))
         to_dir = os.path.join(test_dir,  "node"+str(i))
         shutil.copytree(from_dir, to_dir)
 
-bitcoind_processes = []
-
 def start_nodes(num_nodes, dir):
     # Start bitcoinds, and wait for RPC interface to be up and running:
     for i in range(num_nodes):
@@ -126,9 +126,19 @@ def start_nodes(num_nodes, dir):
         rpc_connections.append(AuthServiceProxy(url))
     return rpc_connections
 
-def stop_nodes():
-    for process in bitcoind_processes:
-        process.kill()
+def debug_log(dir, n_node):
+    return os.path.join(dir, "node"+str(n_node), "regtest", "debug.log")
+
+def stop_nodes(nodes):
+    for i in range(len(nodes)):
+        nodes[i].stop()
+    del nodes[:] # Emptying array closes connections as a side effect
+
+def wait_bitcoinds():
+    # Wait for all bitcoinds to cleanly exit
+    for bitcoind in bitcoind_processes:
+        bitcoind.wait()
+    del bitcoind_processes[:]
 
 def connect_nodes(from_connection, node_num):
     ip_port = "127.0.0.1:"+str(START_P2P_PORT+node_num)

qa/rpc-tests/util.py

@@ -732,7 +732,7 @@ static string JSONRPCExecBatch(const Array& vReq)
 void ServiceConnection(AcceptedConnection *conn)
 {
     bool fRun = true;
-    while (fRun)
+    while (fRun && !ShutdownRequested())
     {
         int nProto = 0;
         map<string, string> mapHeaders;