Switched state tree pruning to be based on reference counting

Author: vbuterin

ethereum/db.py

@@ -1,6 +1,12 @@
 from ethereum import utils
 from ethereum.slogging import get_logger
 from rlp.utils import str_to_bytes
+import sys
+if sys.version_info.major == 2:
+    from repoze.lru import lru_cache
+else:
+    from functools import lru_cache
+
 log = get_logger('db')
 
 
@@ -41,25 +47,6 @@ def __eq__(self, other):
     def __hash__(self):
         return utils.big_endian_to_int(str_to_bytes(self.__repr__()))
 
-    def inc_refcount(self, key, value):
-        self.put(key, value)
-
-    def dec_refcount(self, key):
-        pass
-
-    def revert_refcount_changes(self, epoch):
-        pass
-
-    def commit_refcount_changes(self, epoch):
-        pass
-
-    def cleanup(self, epoch):
-        pass
-
-    def put_temporarily(self, key, value):
-        self.inc_refcount(key, value)
-        self.dec_refcount(key)
-
 
 DB = EphemDB = _EphemDB
 
@@ -136,21 +123,62 @@ def __eq__(self, other):
     def __hash__(self):
         return utils.big_endian_to_int(str_to_bytes(self.__repr__()))
 
-    def inc_refcount(self, key, value):
-        self.put(key, value)
+@lru_cache(128)
+def add1(b):
+    v = utils.big_endian_to_int(b)
+    return utils.zpad(utils.encode_int(v + 1), 4)
 
-    def dec_refcount(self, key):
-        pass
+@lru_cache(128)
+def sub1(b):
+    v = utils.big_endian_to_int(b)
+    return utils.zpad(utils.encode_int(v - 1), 4)
 
-    def revert_refcount_changes(self, epoch):
-        pass
+class RefcountDB(BaseDB):
 
-    def commit_refcount_changes(self, epoch):
-        pass
+    def __init__(self, db):
+        self.db = db
+        self.kv = None
+
+    def get(self, key):
+        return self.db.get(key)[4:]
 
-    def cleanup(self, epoch):
+    def get_refcount(self, key):
+        try:
+            return utils.big_endian_to_int(self.db.get(key)[:4])
+        except KeyError:
+            return 0
+
+    def put(self, key, value):
+        try:
+            existing = self.db.get(key)
+            assert existing[4:] == value
+            self.db.put(key, add1(existing[:4]) + value)
+            # print('putin', key, utils.big_endian_to_int(existing[:4]) + 1)
+        except KeyError:
+            self.db.put(key, b'\x00\x00\x00\x01' + value)
+            # print('putin', key, 1)
+
+    def delete(self, key):
+        existing = self.db.get(key)
+        if existing[:4] == b'\x00\x00\x00\x01':
+            # print('deletung')
+            self.db.delete(key)
+        else:
+            # print(repr(existing[:4]))
+            self.db.put(key, sub1(existing[:4]) + existing[4:])
+
+    def commit(self):
         pass
 
-    def put_temporarily(self, key, value):
-        self.inc_refcount(key, value)
-        self.dec_refcount(key)
+    def _has_key(self, key):
+        return key in self.db
+
+    def __contains__(self, key):
+        return self._has_key(key)
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.db == other.db
+
+    def __hash__(self):
+        return utils.big_endian_to_int(str_to_bytes(self.__repr__()))
+

ethereum/genesis_helpers.py

@@ -4,7 +4,7 @@
     parse_as_bin, parse_as_int, normalize_address
 from ethereum.config import Env
 from ethereum.consensus_strategy import get_consensus_strategy
-from ethereum.db import OverlayDB
+from ethereum.db import OverlayDB, RefcountDB
 import rlp
 import json
 
@@ -44,6 +44,10 @@ def state_from_genesis_declaration(genesis_data, env, block=None, allow_empties=
                 state.set_storage_data(addr, big_endian_to_int(parse_as_bin(k)), big_endian_to_int(parse_as_bin(v)))
     get_consensus_strategy(state.config).initialize(state, block)
     state.commit(allow_empties=allow_empties)
+    print('deleting %d' % len(state.deletes))
+    rdb = RefcountDB(state.db)
+    for delete in state.deletes:
+        rdb.delete(delete)
     block.header.state_root = state.trie.root_hash
     state.prev_headers=[block.header]
     return state

ethereum/pow/chain.py

@@ -18,6 +18,7 @@
 from ethereum.pow.consensus import initialize
 from ethereum.genesis_helpers import mk_basic_state, state_from_genesis_declaration, \
         initialize_genesis_keys
+from ethereum.db import RefcountDB
 
 
 log = get_logger('eth.chain')
@@ -330,9 +331,10 @@ def add_block(self, block):
         if old_block_hash:
             try:
                 deletes = self.db.get(b'deletes:'+old_block_hash)
-                print('Deleting %d trie nodes' % (len(deletes) // 32))
+                print('Deleting up to %d trie nodes' % (len(deletes) // 32))
+                rdb = RefcountDB(self.db)
                 for i in range(0, len(deletes), 32):
-                    self.db.delete(deletes[i: i+32])
+                    rdb.delete(deletes[i: i+32])
                 self.db.delete(b'deletes:'+old_block_hash)
             except KeyError as e:
                 print(e)

ethereum/securetrie.py

@@ -9,7 +9,7 @@ def __init__(self, t):
 
     def update(self, k, v):
         h = utils.sha3(k)
-        self.db.put(h, k)
+        self.db.put(h, utils.str_to_bytes(k))
         self.trie.update(h, v)
 
     def get(self, k):

ethereum/state.py

@@ -10,7 +10,7 @@
 from ethereum.securetrie import SecureTrie
 from ethereum.config import default_config, Env
 from ethereum.block import FakeHeader
-from ethereum.db import BaseDB, EphemDB, OverlayDB
+from ethereum.db import BaseDB, EphemDB, OverlayDB, RefcountDB
 from ethereum.specials import specials as default_specials
 import copy
 import sys
@@ -64,7 +64,7 @@ def __init__(self, nonce, balance, storage, code_hash, env, address):
         self.address = address
         super(Account, self).__init__(nonce, balance, storage, code_hash)
         self.storage_cache = {}
-        self.storage_trie = SecureTrie(Trie(self.env.db, prefix=address))
+        self.storage_trie = SecureTrie(Trie(RefcountDB(self.env.db)))
         self.storage_trie.root_hash = self.storage
         self.touched = False
         self.existent_at_start = True
@@ -131,7 +131,7 @@ class State():
 
     def __init__(self, root=b'', env=Env(), **kwargs):
         self.env = env
-        self.trie = SecureTrie(Trie(self.db, root))
+        self.trie = SecureTrie(Trie(RefcountDB(self.db), root))
         for k, v in STATE_DEFAULTS.items():
             setattr(self, k, kwargs.get(k, copy.copy(v)))
         self.journal = []

ethereum/trie.py

@@ -139,7 +139,7 @@ def is_key_value_type(node_type):
 
 class Trie(object):
 
-    def __init__(self, db, root_hash=BLANK_ROOT, prefix=b''):
+    def __init__(self, db, root_hash=BLANK_ROOT):
         """it also present a dictionary like interface
 
         :param db key value database
@@ -148,7 +148,6 @@ def __init__(self, db, root_hash=BLANK_ROOT, prefix=b''):
         self.db = db  # Pass in a database object directly
         self.set_root_hash(root_hash)
         self.deletes = []
-        self.prefix = prefix
 
     # def __init__(self, dbfile, root_hash=BLANK_ROOT):
     #     """it also present a dictionary like interface
@@ -175,7 +174,7 @@ def get_root_hash(self):
     def _update_root_hash(self):
         val = rlp_encode(self.root_node)
         key = utils.sha3(val)
-        self.db.put(self.prefix+key, val)
+        self.db.put(key, str_to_bytes(val))
         self._root_hash = key
 
     @root_hash.setter
@@ -208,7 +207,7 @@ def _delete_child_storage(self, node):
         elif node_type == NODE_TYPE_EXTENSION:
             self._delete_child_storage(self._decode_to_node(node[1]))
 
-    def _encode_node(self, node):
+    def _encode_node(self, node, put_in_db=True):
         if node == BLANK_NODE:
             return BLANK_NODE
         # assert isinstance(node, list)
@@ -217,15 +216,16 @@ def _encode_node(self, node):
             return node
 
         hashkey = utils.sha3(rlpnode)
-        self.db.put(self.prefix+hashkey, rlpnode)
+        if put_in_db:
+            self.db.put(hashkey, str_to_bytes(rlpnode))
         return hashkey
 
     def _decode_to_node(self, encoded):
         if encoded == BLANK_NODE:
             return BLANK_NODE
         if isinstance(encoded, list):
             return encoded
-        o = rlp.decode(self.db.get(self.prefix+encoded))
+        o = rlp.decode(self.db.get(encoded))
         return o
 
     def _get_node_type(self, node):
@@ -600,7 +600,7 @@ def _delete_node_storage(self, node):
         if node == BLANK_NODE:
             return
         # assert isinstance(node, list)
-        encoded = self._encode_node(node)
+        encoded = self._encode_node(node, put_in_db=False)
         if len(encoded) < 32:
             return
         """
@@ -609,6 +609,7 @@ def _delete_node_storage(self, node):
         thus we can not safely delete nodes for now
         """
         self.deletes.append(encoded)
+        # print('del', encoded, self.db.get_refcount(encoded))
 
     def _delete(self, node, key):
         """ update item inside a node