Added partial state cache

Author: vbuterin

ethereum/genesis_helpers.py

@@ -21,7 +21,7 @@ def block_from_genesis_declaration(genesis_data, env):
                     gas_limit=parse_as_int(genesis_data["gasLimit"]))
     return Block(h, [], [])
 
-def state_from_genesis_declaration(genesis_data, env, block=None, allow_empties=False):
+def state_from_genesis_declaration(genesis_data, env, block=None, allow_empties=False, executing_on_head=False):
     if block:
         assert isinstance(block, Block)
     else:
@@ -43,12 +43,15 @@ def state_from_genesis_declaration(genesis_data, env, block=None, allow_empties=
             for k, v in data['storage'].items():
                 state.set_storage_data(addr, big_endian_to_int(parse_as_bin(k)), big_endian_to_int(parse_as_bin(v)))
     get_consensus_strategy(state.config).initialize(state, block)
+    if executing_on_head:
+        state.executing_on_head = True
     state.commit(allow_empties=allow_empties)
     print('deleting %d' % len(state.deletes))
     rdb = RefcountDB(state.db)
     for delete in state.deletes:
         rdb.delete(delete)
     block.header.state_root = state.trie.root_hash
+    state.changed = {}
     state.prev_headers=[block.header]
     return state
 
@@ -103,8 +106,8 @@ def mk_genesis_block(env, **kwargs):
     return block
 
 
-def mk_basic_state(alloc, header=None, env=None):
-    state = State(env=env or Env())
+def mk_basic_state(alloc, header=None, env=None, executing_on_head=False):
+    state = State(env=env or Env(), executing_on_head=executing_on_head)
     if not header:
         header = {
             "number": 0, "gas_limit": 4712388, "gas_used": 0,

ethereum/pow/chain.py

@@ -35,6 +35,7 @@ def __init__(self, genesis=None, env=None, \
         # Initialize the state
         if 'head_hash' in self.db:  # new head tag
             self.state = self.mk_poststate_of_blockhash(self.db.get('head_hash'))
+            self.state.executing_on_head = True
             print('Initializing chain from saved head, #%d (%s)' % \
                 (self.state.prev_headers[0].number, encode_hex(self.state.prev_headers[0].hash)))
         elif genesis is None:
@@ -47,11 +48,11 @@ def __init__(self, genesis=None, env=None, \
             reset_genesis = True
         elif "extraData" in genesis:
             self.state = state_from_genesis_declaration(
-                genesis, self.env)
+                genesis, self.env, executing_on_head=True)
             reset_genesis = True
             print('Initializing chain from provided genesis declaration')
         elif "prev_headers" in genesis:
-            self.state = State.from_snapshot(genesis, self.env)
+            self.state = State.from_snapshot(genesis, self.env, executing_on_head=True)
             reset_genesis = True
             print('Initializing chain from provided state snapshot, %d (%s)' % \
                 (self.state.block_number, encode_hex(self.state.prev_headers[0].hash[:8])))
@@ -248,9 +249,10 @@ def add_block(self, block):
                      (now, block.header.timestamp, block.header.timestamp - now))
             return False
         # Is the block being added to the head?
-        self.state.deletes = []
         if block.header.prevhash == self.head_hash:
             log.info('Adding to head', head=encode_hex(block.header.prevhash))
+            self.state.deletes = []
+            self.state.changed = {}
             try:
                 apply_block(self.state, block)
             except (AssertionError, KeyError, ValueError, InvalidTransaction, VerificationFailed) as e:
@@ -264,10 +266,11 @@ def add_block(self, block):
                 self.db.put(b'txindex:' + tx.hash, rlp.encode([block.number, i]))
             assert self.get_blockhash_by_number(block.header.number) == block.header.hash
             deletes = self.state.deletes
+            changed = self.state.changed
         # Or is the block being added to a chain that is not currently the head?
         elif block.header.prevhash in self.env.db:
-            log.info('Receiving block not on head, adding to secondary post state',
-                     prevhash=encode_hex(block.header.prevhash))
+            log.info('Receiving block not on head (%s), adding to secondary post state %s' %
+                     (encode_hex(self.head_hash), encode_hex(block.header.prevhash)))
             temp_state = self.mk_poststate_of_blockhash(block.header.prevhash)
             try:
                 apply_block(temp_state, block)
@@ -277,6 +280,7 @@ def add_block(self, block):
                 return False
             deletes = temp_state.deletes
             block_score = self.get_score(block)
+            changed = temp_state.changed
             # If the block should be the new head, replace the head
             if block_score > self.get_score(self.head):
                 b = block
@@ -291,7 +295,8 @@ def add_block(self, block):
                     if b.prevhash not in self.db or self.db.get(b.prevhash) == 'GENESIS':
                         break
                     b = self.get_parent(b)
-                # Replace block index and tx indices
+                # Replace block index and tx indices, and edit the state cache
+                changed_accts = {}
                 replace_from = b.header.number
                 for i in itertools.count(replace_from):
                     log.info('Rewriting height %d' % i)
@@ -300,19 +305,40 @@ def add_block(self, block):
                     if orig_at_height:
                         self.db.delete(key)
                         orig_block_at_height = self.get_block(orig_at_height)
+                        log.info('%s no longer in main chain' % encode_hex(orig_block_at_height.header.hash))
                         for tx in orig_block_at_height.transactions:
                             if b'txindex:' + tx.hash in self.db:
                                 self.db.delete(b'txindex:' + tx.hash)
+                        acct_list = self.db.get(b'changed:'+orig_block_at_height.hash)
+                        for j in range(0, len(acct_list), 20):
+                            changed_accts[acct_list[j: j+20]] = True
                     if i in new_chain:
                         new_block_at_height = new_chain[i]
+                        log.info('%s now in main chain' % encode_hex(new_block_at_height.header.hash))
                         self.db.put(key, new_block_at_height.header.hash)
                         for i, tx in enumerate(new_block_at_height.transactions):
                             self.db.put(b'txindex:' + tx.hash,
                                         rlp.encode([new_block_at_height.number, i]))
+                        if i < b.number:
+                            acct_list = self.db.get(b'changed:'+new_block_at_height.hash)
+                            for j in range(0, len(acct_list), 20):
+                                changed_accts[acct_list[j: j+20]] = True
                     if i not in new_chain and not orig_at_height:
                         break
+                for c in changed.keys():
+                    changed_accts[c] = True
+                for addr in changed_accts.keys():
+                    data = temp_state.trie.get(addr)
+                    if data:
+                        self.state.db.put(b'address:'+addr, data)
+                    else:
+                        try:
+                            self.state.db.delete(b'address:'+addr)
+                        except KeyError:
+                            pass
                 self.head_hash = block.header.hash
                 self.state = temp_state
+                self.state.executing_on_head = True
         # Block has no parent yet
         else:
             if block.header.prevhash not in self.parent_queue:
@@ -324,6 +350,8 @@ def add_block(self, block):
         self.add_child(block)
         self.db.put('head_hash', self.head_hash)
         self.db.put(block.hash, rlp.encode(block))
+        self.db.put(b'changed:'+block.hash, b''.join(list(changed.keys())))
+        print('Saved %d address change logs' % len(changed.keys()))
         self.db.put(b'deletes:'+block.hash, b''.join(deletes))
         print('Saved %d trie node deletes for block %d (%s)' % (len(deletes), block.number, utils.encode_hex(block.hash)))
         # Delete old junk data
@@ -336,6 +364,7 @@ def add_block(self, block):
                 for i in range(0, len(deletes), 32):
                     rdb.delete(deletes[i: i+32])
                 self.db.delete(b'deletes:'+old_block_hash)
+                self.db.delete(b'changed:'+old_block_hash)
             except KeyError as e:
                 print(e)
                 pass

ethereum/state.py

@@ -128,7 +128,7 @@ def to_dict(self):
 #from ethereum.state import State
 class State():
 
-    def __init__(self, root=b'', env=Env(), **kwargs):
+    def __init__(self, root=b'', env=Env(), executing_on_head=False, **kwargs):
         self.env = env
         self.trie = SecureTrie(Trie(RefcountDB(self.db), root))
         for k, v in STATE_DEFAULTS.items():
@@ -137,6 +137,8 @@ def __init__(self, root=b'', env=Env(), **kwargs):
         self.cache = {}
         self.log_listeners = []
         self.deletes = []
+        self.changed = {}
+        self.executing_on_head = executing_on_head
 
     @property
     def db(self):
@@ -159,7 +161,13 @@ def add_block_header(self, block_header):
     def get_and_cache_account(self, address):
         if address in self.cache:
             return self.cache[address]
-        rlpdata = self.trie.get(address)
+        if self.executing_on_head:
+            try:
+                rlpdata = self.db.get(b'address:'+address)
+            except KeyError:
+                rlpdata = b''
+        else:
+            rlpdata = self.trie.get(address)
         if rlpdata != trie.BLANK_NODE:
             o = rlp.decode(rlpdata, Account, env=self.env, address=address)
         else:
@@ -319,13 +327,15 @@ def commit(self, allow_empties=False):
             if acct.touched or acct.deleted:
                 acct.commit()
                 self.deletes.extend(acct.storage_trie.deletes)
+                self.changed[addr] = True
                 if self.account_exists(addr) or allow_empties:
                     self.trie.update(addr, rlp.encode(acct))
-                    #self.db.update(b'acct:'+addr, rlp.encode(acct))
+                    if self.executing_on_head:
+                        self.db.put(b'address:'+addr, rlp.encode(acct))
                 else:
                     self.trie.delete(addr)
-                    # self.db.update(b'acct:'+addr, rlp.encode(acct))
-                    # self.trie.db.
+                    if self.executing_on_head:
+                        self.db.delete(b'address:'+addr)
         self.deletes.extend(self.trie.deletes)
         self.trie.deletes = []
         self.cache = {}
@@ -380,7 +390,7 @@ def to_snapshot(self, root_only=False, no_prevblocks=False):
 
     # Creates a state from a snapshot
     @classmethod
-    def from_snapshot(cls, snapshot_data, env):
+    def from_snapshot(cls, snapshot_data, env, executing_on_head=False):
         state = State(env = env)
         if "alloc" in snapshot_data:
             for addr, data in snapshot_data["alloc"].items():
@@ -425,7 +435,10 @@ def from_snapshot(cls, snapshot_data, env):
                 else:
                     uncles = default
                 setattr(state, k, uncles)
+        if executing_on_head:
+            state.executing_on_head = True
         state.commit()
+        state.changed = {}
         return state