Merge commit '6658f70dfffc86149764d3b04f335ec09a74ee24' into develop_with_state_revamp

Author: Jan Xie

ethereum/abi.py

@@ -259,7 +259,7 @@ def encode_single(typ, arg):  # pylint: disable=too-many-return-statements,too-m
         if not -2 ** bits <= i < 2 ** bits:
             raise ValueOutOfBounds(repr(arg))
 
-        value = i % 2 ** sub  # convert negative to "equivalent" positive
+        value = i % 2 ** 256  # convert negative to "equivalent" positive
         value_encoded = int_to_big_endian(value)
         return zpad(value_encoded, 32)
 
@@ -309,6 +309,11 @@ def encode_single(typ, arg):  # pylint: disable=too-many-return-statements,too-m
         value = fixed_point % 2 ** 256
         return zpad(int_to_big_endian(value), 32)
 
+    # Decimals
+    if base == 'decimal':
+        val_to_encode = int(arg * 10**int(sub))
+        return zpad(encode_int(val_to_encode % 2**256), 32)
+
     if base == 'string':
         if isinstance(arg, utils.unicode):
             arg = arg.encode('utf8')
@@ -724,7 +729,7 @@ def encode_abi(types, args):
 def decode_single(typ, data):
     base, sub, _ = typ
     if base == 'address':
-        return encode_hex(data[12:])
+        return '0x' + encode_hex(data[12:])
     elif base == 'hash':
         return data[32 - int(sub):]
     elif base == 'string' or base == 'bytes':
@@ -746,8 +751,14 @@ def decode_single(typ, data):
         o = big_endian_to_int(data)
         i = (o - 2 ** (high + low)) if o >= 2 ** (high + low - 1) else o
         return (i * 1.0 // 2 ** low)
+    elif base == 'decimal':
+        o = big_endian_to_int(data)
+        i = (o - 2 ** 256 if o > 2 ** 255 else o)
+        return i / 10 ** int(sub)
     elif base == 'bool':
         return bool(int(encode_hex(data), 16))
+    else:
+        raise EncodingError("Unhandled type: %r %r" % (base, sub))
 
 
 # Decodes multiple arguments using the head/tail mechanism
@@ -799,7 +810,7 @@ def dec(typ, arg):
     # Dynamic-sized strings are encoded as <len(str)> + <str>
     if base in ('string', 'bytes') and not sub:
         L = big_endian_to_int(arg[:32])
-        assert len(arg[32:]) == ceil32(L), "Wrong data size for string/bytes object"
+        assert len(arg[32:]) == ceil32(L), "Wrong data size for string/bytes object: expected %d actual %d" % (ceil32(L), len(arg[32:]))
         return arg[32:][:L]
     # Dynamic-sized arrays
     elif sz is None:

ethereum/chain.py

@@ -1,6 +1,7 @@
 import json
 import random
 import time
+import itertools
 
 import rlp
 
@@ -10,7 +11,9 @@
 from ethereum.slogging import get_logger
 from ethereum.state import State
 from ethereum.state_transition import apply_block, initialize, \
-    update_block_env_variables
+    pre_seal_finalize, post_seal_finalize, apply_transaction, mk_receipt_sha, \
+    mk_transaction_sha, calc_difficulty, Receipt, mk_receipt, \
+    update_block_env_variables, validate_uncles, validate_block_header
 from ethereum.utils import encode_hex, parse_as_bin, big_endian_to_int
 
 log = get_logger('eth.chain')
@@ -70,9 +73,9 @@ def __init__(self, genesis=None, env=None, coinbase=b'\x00' * 20, \
             self.db.put('GENESIS_HASH', str(self.genesis.header.hash))
             self.db.put('GENESIS_STATE', json.dumps(self.state.to_snapshot()))
             self.db.put('GENESIS_RLP', rlp.encode(self.genesis))
-            self.db.put('score:' + self.genesis.header.hash, "0")
-            self.db.put('state:' + self.genesis.header.hash, self.state.trie.root_hash)
-            self.db.put('block:0', self.genesis.header.hash)
+            self.db.put(b'score:' + self.genesis.header.hash, "0")
+            self.db.put(b'state:' + self.genesis.header.hash, self.state.trie.root_hash)
+            self.db.put(b'block:0', self.genesis.header.hash)
             self.db.put(self.head_hash, 'GENESIS')
             self.db.commit()
         else:
@@ -105,7 +108,7 @@ def mk_poststate_of_blockhash(self, blockhash, convert=False):
         block = rlp.decode(block_rlp, Block)
 
         state = State(env=self.env)
-        state.trie.root_hash = block.header.state_root if convert else self.db.get('state:'+blockhash)
+        state.trie.root_hash = block.header.state_root if convert else self.db.get(b'state:'+blockhash)
         update_block_env_variables(state, block)
         state.gas_used = block.header.gas_used
         state.txindex = len(block.transactions)
@@ -158,18 +161,18 @@ def get_block(self, blockhash):
     # parent hash and see that it is one of its children
     def add_child(self, child):
         try:
-            existing = self.db.get('ci:' + child.header.prevhash)
+            existing = self.db.get(b'child:' + child.header.prevhash)
         except:
-            existing = ''
+            existing = b''
         existing_hashes = []
         for i in range(0, len(existing), 32):
             existing_hashes.append(existing[i: i+32])
         if child.header.hash not in existing_hashes:
-            self.db.put('ci:' + child.header.prevhash, existing + child.header.hash)
+            self.db.put(b'child:' + child.header.prevhash, existing + child.header.hash)
 
     def get_blockhash_by_number(self, number):
         try:
-            return self.db.get('block:' + str(number))
+            return self.db.get(b'block:' + str(number))
         except:
             return None
 
@@ -180,7 +183,7 @@ def get_block_by_number(self, number):
     def get_child_hashes(self, blockhash):
         o = []
         try:
-            data = self.db.get('child:' + blockhash)
+            data = self.db.get(b'child:' + blockhash)
             for i in range(0, len(data), 32):
                 o.append(data[i:i + 32])
             return o
@@ -198,16 +201,16 @@ def get_children(self, block):
     def get_score(self, block):
         if not block:
             return 0
-        key = 'score:' + block.header.hash
+        key = b'score:' + block.header.hash
 
         fills = []
         while key not in self.db:
             fills.insert(0, (block.header.hash, block.difficulty))
-            key = 'score:' + block.header.prevhash
+            key = b'score:' + block.header.prevhash
             block = self.get_parent(block)
         score = int(self.db.get(key))
         for h,d in fills:
-            key = 'score:' + h
+            key = b'score:' + h
             score = score + d + random.randrange(d // 10**6 + 1)
             self.db.put(key, str(score))
 
@@ -255,52 +258,52 @@ def add_block(self, block):
             except (KeyError, ValueError) as e:  # FIXME add relevant exceptions here
                 log.info('Block %s with parent %s invalid, reason: %s' % (encode_hex(block.header.hash), encode_hex(block.header.prevhash), e))
                 return False
-            self.db.put('block:' + str(block.header.number), block.header.hash)
-            self.db.put('state:' + block.header.hash, self.state.trie.root_hash)
+            self.db.put(b'block:' + str(block.header.number), block.header.hash)
+            self.db.put(b'state:' + block.header.hash, self.state.trie.root_hash)
             block_score = self.get_score(block)  # side effect: put 'score:' cache in db
             self.head_hash = block.header.hash
             for i, tx in enumerate(block.transactions):
-                self.db.put('txindex:' + tx.hash, rlp.encode([block.number, i]))
+                self.db.put(b'txindex:' + tx.hash, rlp.encode([block.number, i]))
         elif block.header.prevhash in self.env.db:
             log.info('Receiving block not on head, adding to secondary post state',
                      prevhash=encode_hex(block.header.prevhash))
             temp_state = self.mk_poststate_of_blockhash(block.header.prevhash)
             try:
                 apply_block(temp_state, block)
-            except (KeyError, ValueError), e:  # FIXME add relevant exceptions here
+            except (KeyError, ValueError) as e:  # FIXME add relevant exceptions here
                 log.info('Block %s with parent %s invalid, reason: %s' % (encode_hex(block.header.hash), encode_hex(block.header.prevhash), e))
                 return False
-            self.db.put('state:' + block.header.hash, temp_state.trie.root_hash)
+            self.db.put(b'state:' + block.header.hash, temp_state.trie.root_hash)
             block_score = self.get_score(block)
             # Replace the head
             if block_score > self.get_score(self.head):
                 b = block
                 new_chain = {}
                 while b.header.number >= int(self.db.get('GENESIS_NUMBER')):
                     new_chain[b.header.number] = b
-                    key = 'block:' + str(b.header.number)
+                    key = b'block:' + str(b.header.number)
                     orig_at_height = self.db.get(key) if key in self.db else None
                     if orig_at_height == b.header.hash:
                         break
                     if b.prevhash not in self.db or self.db.get(b.prevhash) == 'GENESIS':
                         break
                     b = self.get_parent(b)
                 replace_from = b.header.number
-                for i in xrange(replace_from, 2**63 - 1):
+                for i in itertools.count(replace_from):
                     log.info('Rewriting height %d' % i)
-                    key = 'block:' + str(i)
+                    key = b'block:' + str(i)
                     orig_at_height = self.db.get(key) if key in self.db else None
                     if orig_at_height:
                         self.db.delete(key)
                         orig_block_at_height = self.get_block(orig_at_height)
                         for tx in orig_block_at_height.transactions:
-                            if 'txindex:' + tx.hash in self.db:
-                                self.db.delete('txindex:' + tx.hash)
+                            if b'txindex:' + tx.hash in self.db:
+                                self.db.delete(b'txindex:' + tx.hash)
                     if i in new_chain:
                         new_block_at_height = new_chain[i]
                         self.db.put(key, new_block_at_height.header.hash)
                         for i, tx in enumerate(new_block_at_height.transactions):
-                            self.db.put('txindex:' + tx.hash,
+                            self.db.put(b'txindex:' + tx.hash,
                                         rlp.encode([new_block_at_height.number, i]))
                     if i not in new_chain and not orig_at_height:
                         break
@@ -346,7 +349,7 @@ def get_chain(self, frm=None, to=2**63 - 1):
         if frm is None:
             frm = int(self.db.get('GENESIS_NUMBER')) + 1
         chain = []
-        for i in xrange(frm, to):
+        for i in itertools.islice(itertools.count(), frm, to):
             h = self.get_blockhash_by_number(i)
             if not h:
                 return chain
@@ -356,8 +359,8 @@ def get_chain(self, frm=None, to=2**63 - 1):
     def get_transaction(self, tx):
         if not isinstance(tx, (str, bytes)):
             tx = tx.hash
-        if 'txindex:' + tx in self.db:
-            data = rlp.decode(self.db.get('txindex:' + tx))
+        if b'txindex:' + tx in self.db:
+            data = rlp.decode(self.db.get(b'txindex:' + tx))
             blk, index = self.get_block_by_number(
                 big_endian_to_int(data[0])), big_endian_to_int(data[1])
             tx = blk.transactions[index]

ethereum/config.py

@@ -28,6 +28,8 @@
     GASLIMIT_ADJMAX_FACTOR=1024,
     BLKLIM_FACTOR_NOM=3,
     BLKLIM_FACTOR_DEN=2,
+    # Network ID
+    NETWORK_ID=1,
     # Block reward
     BLOCK_REWARD=5000 * utils.denoms.finney,
     NEPHEW_REWARD=5000 * utils.denoms.finney // 32,  # BLOCK_REWARD / 32

ethereum/consensus_strategy.py

@@ -10,7 +10,7 @@ def __init__(self, header_check, header_validate, uncle_validate, block_setup, b
 
 def get_consensus_strategy(config):
     if config['CONSENSUS_STRATEGY'] in ('pow', 'ethpow', 'ethash', 'ethereum1'):
-        from ethpow_utils import ethereum1_check_header, ethereum1_validate_header, \
+        from ethereum.ethpow_utils import ethereum1_check_header, ethereum1_validate_header, \
                                  ethereum1_validate_uncle, ethereum1_pre_finalize_block, \
                                  ethereum1_post_finalize_block, ethereum1_setup_block
         return ConsensusStrategy(
@@ -23,7 +23,7 @@ def get_consensus_strategy(config):
             state_initialize=None
         )
     elif config['CONSENSUS_STRATEGY'] == 'casper':
-        from casper_utils import casper_validate_header, casper_state_initialize, casper_post_finalize_block, casper_setup_block
+        from ethereum.casper_utils import casper_validate_header, casper_state_initialize, casper_post_finalize_block, casper_setup_block
         return ConsensusStrategy(
             header_check=None,
             header_validate=casper_validate_header,

ethereum/new_statetest_utils.py

@@ -0,0 +1,139 @@
+from ethereum.state import State
+from ethereum.block import FakeHeader, Block
+from ethereum.utils import decode_hex, parse_int_or_hex, sha3, to_string, \
+    remove_0x_head, encode_hex, big_endian_to_int
+from ethereum.config import default_config, Env
+from ethereum.exceptions import InvalidTransaction
+import ethereum.transactions as transactions
+import ethereum.state_transition as state_transition
+import copy
+
+from ethereum.slogging import LogRecorder, configure_logging, set_level
+config_string = ':info,eth.vm.log:trace,eth.vm.op:trace,eth.vm.stack:trace,eth.vm.exit:trace,eth.pb.msg:trace,eth.pb.tx:debug'
+
+configure_logging(config_string=config_string)
+
+fake_headers = {}
+
+def mk_fake_header(blknum):
+    if blknum not in fake_headers:
+        fake_headers[blknum] = FakeHeader(sha3(to_string(blknum)))
+    return fake_headers[blknum]
+
+basic_env = {
+    "currentCoinbase": "2adc25665018aa1fe0e6bc666dac8fc2697ff9ba",
+    "currentDifficulty": "256",
+    "currentGasLimit": "1000000000",
+    "currentNumber": "257",
+    "currentTimestamp": "1",
+    "previousHash": "5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6"
+}
+
+konfig = copy.copy(default_config)
+
+konfig_homestead = copy.copy(konfig)
+konfig_homestead["HOMESTEAD_FORK_BLKNUM"] = 0
+konfig_homestead["ANTI_DOS_FORK_BLKNUM"] = 2**99
+konfig_homestead["SPURIOUS_DRAGON_FORK_BLKNUM"] = 2**99
+konfig_homestead["METROPOLIS_FORK_BLKNUM"] = 2**99
+
+konfig_tangerine = copy.copy(konfig)
+konfig_tangerine["HOMESTEAD_FORK_BLKNUM"] = 0
+konfig_tangerine["ANTI_DOS_FORK_BLKNUM"] = 0
+konfig_tangerine["SPURIOUS_DRAGON_FORK_BLKNUM"] = 2**99
+konfig_tangerine["METROPOLIS_FORK_BLKNUM"] = 2**99
+
+konfig_spurious = copy.copy(konfig)
+konfig_spurious["HOMESTEAD_FORK_BLKNUM"] = 0
+konfig_spurious["ANTI_DOS_FORK_BLKNUM"] = 0
+konfig_spurious["SPURIOUS_DRAGON_FORK_BLKNUM"] = 0
+konfig_spurious["METROPOLIS_FORK_BLKNUM"] = 2**99
+
+konfig_metropolis = copy.copy(konfig)
+konfig_metropolis["HOMESTEAD_FORK_BLKNUM"] = 0
+konfig_metropolis["ANTI_DOS_FORK_BLKNUM"] = 0
+konfig_metropolis["SPURIOUS_DRAGON_FORK_BLKNUM"] = 0
+konfig_metropolis["METROPOLIS_FORK_BLKNUM"] = 0
+
+configs = {
+    #"Homestead": konfig_homestead,
+    "EIP150": konfig_tangerine,
+    "EIP158": konfig_spurious,
+    "Metropolis": konfig_metropolis
+}
+
+def compute_state_test_unit(state, txdata, indices, konfig):
+    state.env.config = konfig
+    s = state.snapshot()
+    try:
+        # Create the transaction
+        tx = transactions.Transaction(
+            nonce=parse_int_or_hex(txdata['nonce'] or b"0"),
+            gasprice=parse_int_or_hex(txdata['gasPrice'] or b"0"),
+            startgas=parse_int_or_hex(txdata['gasLimit'][indices["gas"]] or b"0"),
+            to=decode_hex(txdata['to']),
+            value=parse_int_or_hex(txdata['value'][indices["value"]] or b"0"),
+            data=decode_hex(remove_0x_head(txdata['data'][indices["data"]])))
+        tx.sign(decode_hex(txdata['secretKey']))
+        # Run it
+        success, output = state_transition.apply_transaction(state, tx)
+        print("Applied tx")
+    except InvalidTransaction as e:
+        print("Exception: %r" % e)
+        success, output = False, b''
+    state.commit()
+    output_decl = {
+        "hash": encode_hex(state.trie.root_hash),
+        "indexes": indices
+    }
+    state.revert(s)
+    return output_decl
+
+
+def init_state(env, pre):
+    # Setup env
+    state = State(
+        env=Env(config=konfig),
+        block_prevhash=decode_hex(env['previousHash']),
+        prev_headers=[mk_fake_header(i) for i in range(parse_int_or_hex(env['currentNumber']) -1,
+                                                       max(-1, parse_int_or_hex(env['currentNumber']) -257), -1)],
+        block_number=parse_int_or_hex(env['currentNumber']),
+        block_coinbase=decode_hex(env['currentCoinbase']),
+        block_difficulty=parse_int_or_hex(env['currentDifficulty']),
+        gas_limit=parse_int_or_hex(env['currentGasLimit']),
+        timestamp=parse_int_or_hex(env['currentTimestamp']))
+
+    # Fill up pre
+    for address, h in list(pre.items()):
+        assert len(address) == 40
+        address = decode_hex(address)
+        assert set(h.keys()) == set(['code', 'nonce', 'balance', 'storage'])
+        state.set_nonce(address, parse_int_or_hex(h['nonce']))
+        state.set_balance(address, parse_int_or_hex(h['balance']))
+        state.set_code(address, decode_hex(h['code'][2:]))
+        for k, v in h['storage'].items():
+            state.set_storage_data(address,
+                                   big_endian_to_int(decode_hex(k[2:])),
+                                   decode_hex(v[2:]))
+
+    state.commit()
+    return state
+
+def verify_state_test(test):
+    print("Verifying state test")
+    _state = init_state(test["env"], test["pre"])
+    for config_name, results in test["post"].items():
+        # Old protocol versions may not be supported
+        if config_name not in configs:
+            continue
+        print("Testing for %s" % config_name)
+        for result in results:
+            print("Checking for values: g %d v %d d %s" % (
+                  parse_int_or_hex(test["transaction"]['gasLimit'][result["indexes"]["gas"]]),
+                  parse_int_or_hex(test["transaction"]['value'][result["indexes"]["value"]]),
+                  test["transaction"]['data'][result["indexes"]["data"]]))
+            computed = compute_state_test_unit(_state, test["transaction"], result["indexes"], configs[config_name])
+            if computed["hash"] != result["hash"]:
+                raise Exception("Hash mismatch, computed: %s, supplied: %s" % (computed["hash"], result["hash"]))
+            else:
+                print("Hash matched!: %s" % computed["hash"])