Patch electrum to support Bubbles network upgrade (#17)

* Patch electrum to support Bubbles network upgrade
Summary
=======
- Handle dynamic header size
- Update the branch id used for signing the transaction
- Handle verification of blocks following the difficulty adjustment
- Update block explorers link
- Add new electrum server ip

Author: hTrap

lib/blockchain.py

@@ -22,6 +22,7 @@
 # SOFTWARE.
 import os
 import threading
+from time import sleep
 
 from . import util
 from . import bitcoin
@@ -32,6 +33,7 @@
 HDR_EH_192_7_LEN = 543
 CHUNK_LEN = 100
 BUBBLES_ACTIVATION_HEIGHT = 585318
+DIFFADJ_ACTIVATION_HEIGHT = 585322
 
 MAX_TARGET = 0x0007FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
 POW_AVERAGING_WINDOW = 17
@@ -52,6 +54,14 @@
     (100 + POW_MAX_ADJUST_DOWN) // 100
 
 
+def is_post_equihash_fork(height):
+    return height >= BUBBLES_ACTIVATION_HEIGHT
+
+def get_header_size(height):
+    if is_post_equihash_fork(height):
+        return HDR_EH_192_7_LEN
+    return HDR_LEN
+
 def serialize_header(res):
     s = int_to_hex(res.get('version'), 4) \
         + rev_hex(res.get('prev_block_hash')) \
@@ -67,7 +77,7 @@ def serialize_header(res):
 def deserialize_header(s, height):
     if not s:
         raise Exception('Invalid header: {}'.format(s))
-    if len(s) != HDR_LEN and len(s) != HDR_EH_192_7_LEN:
+    if len(s) != get_header_size(height):
         raise Exception('Invalid header length: {}'.format(len(s)))
     hex_to_int = lambda s: int('0x' + bh2u(s[::-1]), 16)
     h = {}
@@ -79,10 +89,7 @@ def deserialize_header(s, height):
     h['bits'] = hex_to_int(s[104:108])
     h['nonce'] = hash_encode(s[108:140])
     h['sol_size'] = hash_encode(s[140:143])
-    if height < BUBBLES_ACTIVATION_HEIGHT:
-        h['solution'] = hash_encode(s[143:HDR_LEN])
-    else:
-        h['solution'] = hash_encode(s[143:HDR_EH_192_7_LEN])
+    h['solution'] = hash_encode(s[143:])
     h['block_height'] = height
     return h
 
@@ -142,7 +149,7 @@ def __init__(self, config, checkpoint, parent_id):
         self.parent_id = parent_id
         self.lock = threading.Lock()
         with self.lock:
-            self.update_size()
+            self.update_size(0)
 
     def parent(self):
         return blockchains[self.parent_id]
@@ -180,9 +187,33 @@ def size(self):
         with self.lock:
             return self._size
 
-    def update_size(self):
+    def update_size(self, height):
         p = self.path()
-        self._size = os.path.getsize(p)//HDR_LEN if os.path.exists(p) else 0
+        if os.path.exists(p):
+            with open(p, 'rb') as f:
+                size = f.seek(0, 2)
+            self._size = self.calculate_size(height, size)
+        else:
+            self._size = 0
+
+    def calculate_size(self, checkpoint, size_in_bytes):
+        size_before_fork = 0
+        size_after_fork = 0
+
+        if not is_post_equihash_fork(checkpoint):
+            size_before_fork = size_in_bytes//HDR_LEN
+            if is_post_equihash_fork(size_before_fork):
+                size_before_fork = BUBBLES_ACTIVATION_HEIGHT
+                checkpoint = BUBBLES_ACTIVATION_HEIGHT
+                size_in_bytes -= size_before_fork * HDR_LEN
+        else:
+            size_before_fork = BUBBLES_ACTIVATION_HEIGHT
+            size_in_bytes -= size_before_fork * HDR_LEN
+
+        if is_post_equihash_fork(checkpoint):
+            size_after_fork = size_in_bytes//HDR_EH_192_7_LEN
+
+        return size_before_fork + size_after_fork
 
     def verify_header(self, header, prev_hash, target):
         _hash = hash_header(header)
@@ -191,18 +222,29 @@ def verify_header(self, header, prev_hash, target):
         if constants.net.TESTNET:
             return
         bits = self.target_to_bits(target)
+        height = header.get('block_height')
+        if height >= DIFFADJ_ACTIVATION_HEIGHT and height < DIFFADJ_ACTIVATION_HEIGHT + POW_AVERAGING_WINDOW:
+            valid_bits = [
+                0x1f07ffff, 0x1e0ffffe, 0x1e0ffffe, 0x1f07ffff, 0x1f014087, 0x1f01596b,
+                0x1f01743d, 0x1f019124, 0x1f01b049, 0x1f01d1da, 0x1f01f606, 0x1f021d01,
+                0x1f024703, 0x1f027448, 0x1f02a510, 0x1f02d9a3, 0x1f03124a,
+                ]
+            bits = valid_bits[height%DIFFADJ_ACTIVATION_HEIGHT]
+            target = self.bits_to_target(bits)
         if bits != header.get('bits'):
             raise Exception("bits mismatch: %s vs %s" % (bits, header.get('bits')))
         if int('0x' + _hash, 16) > target:
             raise Exception("insufficient proof of work: %s vs target %s" % (int('0x' + _hash, 16), target))
 
-    def verify_chunk(self, index, data):
-        num = len(data) // HDR_LEN
-        prev_hash = self.get_hash(index * CHUNK_LEN - 1)
+    def verify_chunk(self, height, data):
+        size = len(data)
+        prev_hash = self.get_hash(height-1)
         chunk_headers = {'empty': True}
-        for i in range(num):
-            raw_header = data[i*HDR_LEN:(i+1) * HDR_LEN]
-            height = index * CHUNK_LEN + i
+        offset = 0
+        i = 0
+        while offset < size:
+            header_size = get_header_size(height)
+            raw_header = data[offset:offset+header_size]
             header = deserialize_header(raw_header, height)
             target = self.get_target(height, chunk_headers)
             self.verify_header(header, prev_hash, target)
@@ -213,20 +255,26 @@ def verify_chunk(self, index, data):
                 chunk_headers['empty'] = False
             chunk_headers['max_height'] = height
             prev_hash = hash_header(header)
+            offset += header_size
+            height += 1
+            i += 1
+            sleep(0.001)
 
     def path(self):
         d = util.get_headers_dir(self.config)
         filename = 'blockchain_headers' if self.parent_id is None else os.path.join('forks', 'fork_%d_%d'%(self.parent_id, self.checkpoint))
         return os.path.join(d, filename)
 
-    def save_chunk(self, index, chunk):
-        filename = self.path()
-        d = (index * CHUNK_LEN - self.checkpoint) * HDR_LEN
-        if d < 0:
-            chunk = chunk[-d:]
-            d = 0
-        truncate = index >= len(self.checkpoints)
-        self.write(chunk, d, truncate)
+    def save_chunk(self, height, chunk):
+        delta = height - self.checkpoint
+
+        if delta < 0:
+            chunk = chunk[-delta:]
+            height = self.checkpoint
+
+        offset = self.get_offset(self.checkpoint, height)
+        truncate = (height / CHUNK_LEN) >= len(self.checkpoints)
+        self.write(chunk, offset, truncate)
         self.swap_with_parent()
 
     def swap_with_parent(self):
@@ -241,11 +289,12 @@ def swap_with_parent(self):
         parent = self.parent()
         with open(self.path(), 'rb') as f:
             my_data = f.read()
+        offset = self.get_offset(parent.checkpoint, checkpoint)
         with open(parent.path(), 'rb') as f:
-            f.seek((checkpoint - parent.checkpoint)*HDR_LEN)
-            parent_data = f.read(parent_branch_size*HDR_LEN)
+            f.seek(offset)
+            parent_data = f.read()
         self.write(parent_data, 0)
-        parent.write(my_data, (checkpoint - parent.checkpoint)*HDR_LEN)
+        parent.write(my_data, offset)
         # store file path
         for b in blockchains.values():
             b.old_path = b.path()
@@ -265,23 +314,29 @@ def swap_with_parent(self):
 
     def write(self, data, offset, truncate=True):
         filename = self.path()
+        current_offset = self.get_offset(self.checkpoint, self.size())
+
         with self.lock:
             with open(filename, 'rb+') as f:
-                if truncate and offset != self._size*HDR_LEN:
+                if truncate and offset != current_offset:
                     f.seek(offset)
                     f.truncate()
                 f.seek(offset)
                 f.write(data)
                 f.flush()
                 os.fsync(f.fileno())
-            self.update_size()
+        self.update_size(self.size())
 
     def save_header(self, header):
-        delta = header.get('block_height') - self.checkpoint
+        height = header.get('block_height')
+        delta = height - self.checkpoint
         data = bfh(serialize_header(header))
+        offset = self.get_offset(self.checkpoint, height)
+        header_size = get_header_size(height)
+
         assert delta == self.size()
-        assert len(data) == HDR_LEN or len(data) == HDR_EH_192_7_LEN
-        self.write(data, delta*HDR_LEN)
+        assert len(data) == header_size
+        self.write(data, offset)
         self.swap_with_parent()
 
     def read_header(self, height):
@@ -292,19 +347,20 @@ def read_header(self, height):
             return self.parent().read_header(height)
         if height > self.height():
             return
-        delta = height - self.checkpoint
+        offset = self.get_offset(self.checkpoint, height)
+        header_size = get_header_size(height)
         name = self.path()
         if os.path.exists(name):
             with open(name, 'rb') as f:
-                f.seek(delta * HDR_LEN)
-                h = f.read(HDR_LEN)
-                if len(h) < HDR_LEN:
+                f.seek(offset)
+                h = f.read(header_size)
+                if len(h) < header_size:
                     raise Exception('Expected to read a full header. This was only {} bytes'.format(len(h)))
         elif not os.path.exists(util.get_headers_dir(self.config)):
             raise Exception('Electrum datadir does not exist. Was it deleted while running?')
         else:
             raise Exception('Cannot find headers file but datadir is there. Should be at {}'.format(name))
-        if h == bytes([0])*HDR_LEN:
+        if h == bytes([0])*header_size:
             return None
         return deserialize_header(h, height)
 
@@ -430,9 +486,9 @@ def can_connect(self, header, check_height=True):
     def connect_chunk(self, idx, hexdata):
         try:
             data = bfh(hexdata)
-            self.verify_chunk(idx, data)
-            #self.print_error("validated chunk %d" % idx)
-            self.save_chunk(idx, data)
+            self.verify_chunk(idx * CHUNK_LEN, data)
+            # self.print_error("validated chunk %d" % idx)
+            self.save_chunk(idx * CHUNK_LEN, data)
             return True
         except BaseException as e:
             self.print_error('verify_chunk %d failed'%idx, str(e))
@@ -453,12 +509,27 @@ def get_checkpoints(self):
                 with open(self.path(), 'rb') as f:
                     lower_header = height - TARGET_CALC_BLOCKS
                     for height in range(height, lower_header-1, -1):
-                        f.seek(height*HDR_LEN)
-                        hd = f.read(HDR_LEN)
-                        if len(hd) < HDR_LEN:
+                        f.seek(height*get_header_size(height))
+                        hd = f.read(get_header_size(height))
+                        if len(hd) < get_header_size(height):
                             raise Exception(
                                 'Expected to read a full header.'
                                 ' This was only {} bytes'.format(len(hd)))
                         extra_headers.append((height, bh2u(hd)))
             cp.append((h, target, extra_headers))
         return cp
+
+    def get_offset(self, checkpoint, height):
+        offset_before_fork = 0
+        offset_after_fork = 0
+
+        if not is_post_equihash_fork(height):
+            offset_before_fork = height - checkpoint
+        else:
+            offset_before_fork = BUBBLES_ACTIVATION_HEIGHT
+
+        if is_post_equihash_fork(height):
+            offset_after_fork = height - max(checkpoint, BUBBLES_ACTIVATION_HEIGHT)
+
+        offset = (offset_before_fork * HDR_LEN) + (offset_after_fork * HDR_EH_192_7_LEN)
+        return offset

lib/network.py

@@ -37,7 +37,7 @@
 from . import util
 from . import bitcoin
 from .bitcoin import *
-from .blockchain import HDR_LEN, HDR_EH_192_7_LEN, CHUNK_LEN
+from .blockchain import CHUNK_LEN, get_header_size
 from . import constants
 from .interface import Connection, Interface
 from . import blockchain
@@ -831,7 +831,7 @@ def on_get_header(self, interface, response, height):
             self.connection_down(interface.server)
             return
 
-        if len(hex_header) != HDR_LEN*2 and len(hex_header) != HDR_EH_192_7_LEN*2:
+        if len(hex_header) != get_header_size(height)*2:
             interface.print_error('wrong header length', interface.request)
             self.connection_down(interface.server)
             return
@@ -984,17 +984,19 @@ def wait_on_sockets(self):
     def init_headers_file(self):
         b = self.blockchains[0]
         filename = b.path()
-        len_checkpoints = len(constants.net.CHECKPOINTS)
-        length = HDR_LEN * len_checkpoints * CHUNK_LEN
-        if not os.path.exists(filename) or os.path.getsize(filename) < length:
+        # len_checkpoints = len(constants.net.CHECKPOINTS)
+        len_checkpoints = 0
+        # length = HDR_LEN * len_checkpoints * CHUNK_LEN
+        # if not os.path.exists(filename) or os.path.getsize(filename) < length:
+        if not os.path.exists(filename):
             with open(filename, 'wb') as f:
                 for i in range(len_checkpoints):
                     for height, header_data in b.checkpoints[i][2]:
-                        f.seek(height*HDR_LEN)
+                        f.seek(height*get_header_size(height))
                         bin_header = bfh(header_data)
                         f.write(bin_header)
         with b.lock:
-            b.update_size()
+            b.update_size(0)
 
     def run(self):
         self.init_headers_file()
@@ -1013,10 +1015,10 @@ def on_notify_header(self, interface, header):
         if not height or not hex_header:
             return
 
-        if len(hex_header) != HDR_LEN*2 and len(hex_header) != HDR_EH_192_7_LEN*2:
-            interface.print_error('wrong header length', interface.request)
-            self.connection_down(interface.server)
-            return
+        # if len(hex_header) != get_header_size(height)*2:
+        #     interface.print_error('wrong header length', interface.request)
+        #     self.connection_down(interface.server)
+        #     return
 
         header = blockchain.deserialize_header(bfh(hex_header), height)
         if height < self.max_checkpoint():

lib/servers.json

@@ -1,5 +1,5 @@
 {
-  "electrum.zclassic-ce.org": {
+  "178.128.27.40": {
       "pruning": "-",
       "s": "50002",
       "t": "50001",

lib/util.py

@@ -506,10 +506,8 @@ def time_difference(distance_in_time, include_seconds):
         return "over %d years" % (round(distance_in_minutes / 525600))
 
 mainnet_block_explorers = {
-    'zclele.duckdns.org:3001': ('http://zclele.duckdns.org:3001',
-                        {'tx': 'tx', 'addr': 'address'}),
-    'zclassic-ce.io': ('http://zclassic-ce.io',
-                        {'tx': 'tx', 'addr': 'address'}),
+    'zeltrez.io': ('https://explorer.zcl.zeltrez.io/',
+                        {'tx': 'tx/', 'addr': 'address/'})
 }
 
 testnet_block_explorers = {
@@ -524,7 +522,7 @@ def block_explorer_info():
     return testnet_block_explorers if constants.net.TESTNET else mainnet_block_explorers
 
 def block_explorer(config):
-    return config.get('block_explorer', 'zclassic-ce.io')
+    return config.get('block_explorer', 'zeltrez.io')
 
 def block_explorer_tuple(config):
     return block_explorer_info().get(block_explorer(config))

lib/version.py

@@ -1,4 +1,4 @@
-ELECTRUM_VERSION = 'v3.2.3'   # version of the client package
+ELECTRUM_VERSION = 'v3.2.4'   # version of the client package
 PROTOCOL_VERSION = '1.2'     # protocol version requested
 
 # The hash of the mnemonic seed must begin with this