Make linearize scripts Python 3-compatible.

Author: droark

contrib/linearize/README.md

@@ -1,5 +1,6 @@
 # Linearize
-Construct a linear, no-fork, best version of the Bitcoin blockchain.
+Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts
+run using Python 3 but are compatible with Python 2.
 
 ## Step 1: Download hash list
 
@@ -9,28 +10,32 @@ Required configuration file settings for linearize-hashes:
 * RPC: `rpcuser`, `rpcpassword`
 
 Optional config file setting for linearize-hashes:
-* RPC: `host`, `port`  (Default: `127.0.0.1:8332`)
+* RPC: `host`  (Default: `127.0.0.1`)
+* RPC: `port`  (Default: `8332`)
 * Blockchain: `min_height`, `max_height`
 * `rev_hash_bytes`: If true, the written block hash list will be
 byte-reversed. (In other words, the hash returned by getblockhash will have its
 bytes reversed.) False by default. Intended for generation of
 standalone hash lists but safe to use with linearize-data.py, which will output
 the same data no matter which byte format is chosen.
 
+The `linearize-hashes` script requires a connection, local or remote, to a
+JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient.
+
 ## Step 2: Copy local block data
 
     $ ./linearize-data.py linearize.cfg
 
 Required configuration file settings:
 * `output_file`: The file that will contain the final blockchain.
       or
-* `output`: Output directory for linearized blocks/blkNNNNN.dat output.
+* `output`: Output directory for linearized `blocks/blkNNNNN.dat` output.
 
 Optional config file setting for linearize-data:
 * `file_timestamp`: Set each file's last-modified time to that of the most
 recent block in that file.
 * `genesis`: The hash of the genesis block in the blockchain.
-* `input: bitcoind blocks/ directory containing blkNNNNN.dat
+* `input`: bitcoind blocks/ directory containing blkNNNNN.dat
 * `hashlist`: text file containing list of block hashes created by
 linearize-hashes.py.
 * `max_out_sz`: Maximum size for files created by the `output_file` option.

contrib/linearize/linearize-data.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # linearize-data.py: Construct a linear, no-fork version of the chain.
 #
@@ -8,29 +8,33 @@
 #
 
 from __future__ import print_function, division
+try: # Python 3
+    import http.client as httplib
+except ImportError: # Python 2
+    import httplib
 import json
 import struct
 import re
 import os
 import os.path
 import base64
-import httplib
 import sys
 import hashlib
 import datetime
 import time
 from collections import namedtuple
+from binascii import hexlify, unhexlify
 
 settings = {}
 
 ##### Switch endian-ness #####
 def hex_switchEndian(s):
 	""" Switches the endianness of a hex string (in pairs of hex chars) """
-	pairList = [s[i]+s[i+1] for i in range(0,len(s),2)]
-	return ''.join(pairList[::-1])
+	pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
+	return b''.join(pairList[::-1]).decode()
 
 def uint32(x):
-	return x & 0xffffffffL
+	return x & 0xffffffff
 
 def bytereverse(x):
 	return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
@@ -41,14 +45,14 @@ def bufreverse(in_buf):
 	for i in range(0, len(in_buf), 4):
 		word = struct.unpack('@I', in_buf[i:i+4])[0]
 		out_words.append(struct.pack('@I', bytereverse(word)))
-	return ''.join(out_words)
+	return b''.join(out_words)
 
 def wordreverse(in_buf):
 	out_words = []
 	for i in range(0, len(in_buf), 4):
 		out_words.append(in_buf[i:i+4])
 	out_words.reverse()
-	return ''.join(out_words)
+	return b''.join(out_words)
 
 def calc_hdr_hash(blk_hdr):
 	hash1 = hashlib.sha256()
@@ -65,7 +69,7 @@ def calc_hash_str(blk_hdr):
 	hash = calc_hdr_hash(blk_hdr)
 	hash = bufreverse(hash)
 	hash = wordreverse(hash)
-	hash_str = hash.encode('hex')
+	hash_str = hexlify(hash).decode('utf-8')
 	return hash_str
 
 def get_blk_dt(blk_hdr):
@@ -217,7 +221,7 @@ def run(self):
 
 			inMagic = inhdr[:4]
 			if (inMagic != self.settings['netmagic']):
-				print("Invalid magic: " + inMagic.encode('hex'))
+				print("Invalid magic: " + hexlify(inMagic).decode('utf-8'))
 				return
 			inLenLE = inhdr[4:]
 			su = struct.unpack("<I", inLenLE)
@@ -294,14 +298,14 @@ def run(self):
 	if 'split_timestamp' not in settings:
 		settings['split_timestamp'] = 0
 	if 'max_out_sz' not in settings:
-		settings['max_out_sz'] = 1000L * 1000 * 1000
+		settings['max_out_sz'] = 1000 * 1000 * 1000
 	if 'out_of_order_cache_sz' not in settings:
 		settings['out_of_order_cache_sz'] = 100 * 1000 * 1000
 
-	settings['max_out_sz'] = long(settings['max_out_sz'])
+	settings['max_out_sz'] = int(settings['max_out_sz'])
 	settings['split_timestamp'] = int(settings['split_timestamp'])
 	settings['file_timestamp'] = int(settings['file_timestamp'])
-	settings['netmagic'] = settings['netmagic'].decode('hex')
+	settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8'))
 	settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz'])
 
 	if 'output_file' not in settings and 'output' not in settings:

contrib/linearize/linearize-hashes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # linearize-hashes.py:  List blocks in a linear, no-fork version of the chain.
 #
@@ -8,38 +8,47 @@
 #
 
 from __future__ import print_function
+try: # Python 3
+    import http.client as httplib
+except ImportError: # Python 2
+    import httplib
 import json
 import struct
 import re
 import base64
-import httplib
 import sys
 
 settings = {}
 
 ##### Switch endian-ness #####
 def hex_switchEndian(s):
 	""" Switches the endianness of a hex string (in pairs of hex chars) """
-	pairList = [s[i]+s[i+1] for i in range(0,len(s),2)]
-	return ''.join(pairList[::-1])
+	pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
+	return b''.join(pairList[::-1]).decode()
 
 class BitcoinRPC:
 	def __init__(self, host, port, username, password):
 		authpair = "%s:%s" % (username, password)
-		self.authhdr = "Basic %s" % (base64.b64encode(authpair))
-		self.conn = httplib.HTTPConnection(host, port, False, 30)
+		authpair = authpair.encode('utf-8')
+		self.authhdr = b"Basic " + base64.b64encode(authpair)
+		self.conn = httplib.HTTPConnection(host, port=port, timeout=30)
 
 	def execute(self, obj):
-		self.conn.request('POST', '/', json.dumps(obj),
-			{ 'Authorization' : self.authhdr,
-			  'Content-type' : 'application/json' })
+		try:
+			self.conn.request('POST', '/', json.dumps(obj),
+				{ 'Authorization' : self.authhdr,
+				  'Content-type' : 'application/json' })
+		except ConnectionRefusedError:
+			print('RPC connection refused. Check RPC settings and the server status.',
+			      file=sys.stderr)
+			return None
 
 		resp = self.conn.getresponse()
 		if resp is None:
 			print("JSON-RPC: no response", file=sys.stderr)
 			return None
 
-		body = resp.read()
+		body = resp.read().decode('utf-8')
 		resp_obj = json.loads(body)
 		return resp_obj
 
@@ -70,6 +79,9 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
 			batch.append(rpc.build_request(x, 'getblockhash', [height + x]))
 
 		reply = rpc.execute(batch)
+		if reply is None:
+			print('Cannot continue. Program will halt.')
+			return None
 
 		for x,resp_obj in enumerate(reply):
 			if rpc.response_is_error(resp_obj):