Merge branch 'state_revamp' into state_revamp_for_pyethapp

Author: Jan Xie

ethereum/casper_contract.py

@@ -20,7 +20,6 @@
 rlp_decoder_path = os.path.join(mydir, 'rlp_decoder_contract.py')
 hash_without_ed_path = os.path.join(mydir, 'hash_without_ed_contract.py')
 finalizer_path = os.path.join(mydir, 'finalizer_contract.py')
-validator_sizes = [64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072]
 
 
 # Get the final saved code of a contract from the init code
@@ -100,7 +99,7 @@ def get_parent(self, val):
                 assert utils.sha3(o) == origval
                 return o
             val = utils.sha3(val)
-        raise Exception("parent not found")
+        raise Exception("Randao parent not found")
 
 # Create the validation code for a given address
 def generate_validation_code(addr):
@@ -131,19 +130,19 @@ def call_casper(state, fun, args=[], gas=1000000, value=0):
 
 # Get the number of skips needed to make a block on the current
 # parent
-def get_skips_and_block_making_time(chain, my_indices, max_lookup=100):
+def get_skips_and_block_making_time(state, vchash, max_lookup=100):
     skips = 0
     while skips < max_lookup:
-        indices = call_casper(chain.state, 'getValidator', [skips])
-        if indices and (my_indices[0], my_indices[1]) == (indices[0], indices[1]):
-            return skips, call_casper(chain.state, 'getMinTimestamp', [skips]) + 3
+        vchash2 = call_casper(state, 'getValidator', [skips])
+        if vchash2 and vchash2 == vchash:
+            return skips, call_casper(state, 'getMinTimestamp', [skips]) + 3
         skips += 1
     return None, None
 
 # Check a particular count of skips
-def check_skips(chain, my_indices, skips):
-    indices = call_casper(chain.state, 'getValidator', [skips])
-    return indices and (my_indices[0], my_indices[1]) == (indices[0], indices[1])
+def check_skips(chain, vchash, skips):
+    vchash2 = call_casper(chain.state, 'getValidator', [skips])
+    return vchash2 and vchash2 == vchash
 
 # Get timestamp given a particular number of skips
 def get_timestamp(chain, skips):
@@ -152,12 +151,11 @@ def get_timestamp(chain, skips):
     return call_casper(chain.state, 'getMinTimestamp', [skips]) + 3 
 
 # Add a signature to a block
-def sign_block(block, key, randao_parent, indices, skips):
+def sign_block(block, key, randao_parent, vchash, skips):
     block.header.extra_data = \
         randao_parent + \
         utils.zpad(utils.encode_int(skips), 32) + \
-        utils.zpad(utils.encode_int(indices[0]), 32) + \
-        utils.zpad(utils.encode_int(indices[1]), 32)
+        vchash
     for val in utils.ecsign(block.header.signing_hash, key):
         block.header.extra_data += utils.zpad(utils.encode_int(val), 32)
     return block
@@ -176,12 +174,12 @@ def casper_contract_bootstrap(state, timestamp=0, epoch_length=100, number=0, ga
     success = apply_transaction(state, t)
     assert success
 
-def validator_inject(state, vcode, deposit_size, randao_commitment, nonce=0, ct=None):
+def validator_inject(state, vcode, deposit_size, randao_commitment, address, nonce=0, ct=None):
     if not ct:
         ct = get_casper_ct()
     state.set_balance(utils.int_to_addr(1), deposit_size)
     t = Transaction(nonce, 0, 10**8, casper_config['CASPER_ADDR'], deposit_size,
-                    ct.encode('deposit', [vcode, randao_commitment]))
+                    ct.encode('deposit', [vcode, randao_commitment, address]))
     t._sender = utils.int_to_addr(1)
     success = apply_transaction(state, t)
     assert success
@@ -215,14 +213,14 @@ def make_casper_genesis(validators, alloc, timestamp=0, epoch_length=100):
     casper_contract_bootstrap(state, ct=ct)
 
     # Add validators
-    for i, (vcode, deposit_size, randao_commitment) in enumerate(validators):
-        validator_inject(state, vcode, deposit_size, randao_commitment, i, ct)
+    for i, (vcode, deposit_size, randao_commitment, address) in enumerate(validators):
+        validator_inject(state, vcode, deposit_size, randao_commitment, address, i, ct)
 
     # Start the first epoch
     casper_start_epoch(state)
 
     assert call_casper(state, 'getEpoch', []) == 0
-    assert call_casper(state, 'getTotalDeposits', []) == sum([d for a,d,r in validators])
+    assert call_casper(state, 'getTotalDeposits', []) == sum([d for a,d,r,a in validators])
     state.commit()
     return state
 
@@ -233,21 +231,6 @@ def casper_start_epoch(state):
     t._sender = casper_config['CASPER_ADDR']
     apply_transaction(state, t)
 
-def find_indices(state, vcode):
-    for i in range(len(validator_sizes)):
-        epoch = state.block_number // call_casper(state, 'getEpochLength', [])
-        valcount = call_casper(state, 'getValidatorCount', [i])
-        for j in range(valcount):
-            valcode = call_casper(state, 'getValidationCode', [i, j])
-            if valcode == vcode:
-                start = call_casper(state, 'getStartEpoch', [i, j])
-                end = call_casper(state, 'getEndEpoch', [i, j])
-                if start <= epoch < end:
-                    return [i, j, True]
-                else:
-                    return [i, j, False]
-    return [None, None, False]
-
 
 def get_dunkle_candidates(chain, state, scan_limit=10):
     blknumber = call_casper(state, 'getBlockNumber')

ethereum/casper_utils.py

@@ -225,7 +225,8 @@ def add_block(self, block):
             while i < len(self.time_queue) and block.timestamp > self.time_queue[i].timestamp:
                 i += 1
             self.time_queue.insert(i, block)
-            log.info('Block received too early. Delaying for %d seconds' % (block.header.timestamp - now))
+            log.info('Block received too early (%d vs %d). Delaying for %d seconds' %
+                     (now, block.header.timestamp, block.header.timestamp - now))
             return False
         if block.header.prevhash == self.head_hash:
             log.info('Adding to head', head=encode_hex(block.header.prevhash))

ethereum/chain.py

@@ -279,11 +279,12 @@ def rp(what, actual, target):
     if not state.is_METROPOLIS() and not SKIP_MEDSTATES:
         state.commit()
     r = mk_receipt(state, state.logs)
+    _logs = list(state.logs)
     state.logs = []
     state.add_to_list('receipts', r)
     state.set_param('bloom', state.bloom | r.bloom)
     state.set_param('txindex', state.txindex + 1)
-    return success
+    return success, output
 
 
 def mk_receipt_sha(receipts):

ethereum/state_transition.py

@@ -1,4 +1,4 @@
-from ethereum import tester, blocks
+from ethereum import tester
 import ethereum.utils as utils
 import rlp
 import ethereum.testutils as testutils

ethereum/tests/test_difficulty.py

@@ -1,7 +1,6 @@
 import os
 import pytest
 import json
-import ethereum.blocks as blocks
 import ethereum.testutils as testutils
 from ethereum.testutils import fixture_to_bytes
 import ethereum.utils as utils
@@ -11,6 +10,10 @@
 logger = get_logger()
 
 
+def blocks_genesis(*args):  # FIXME!
+    pass
+
+
 @pytest.fixture(scope="module")
 def genesis_fixture():
     """
@@ -29,22 +32,22 @@ def genesis_fixture():
 
 @pytest.mark.xfail  # code not in sync with genesis fixtures
 def test_genesis_state_root(genesis_fixture):
-    genesis = blocks.genesis(new_env())
+    genesis = blocks_genesis(new_env())
     assert encode_hex(genesis.state_root) == utils.to_string(genesis_fixture['genesis_state_root'])
 
 
 def test_genesis_initial_alloc(genesis_fixture):
     env = new_env()
-    genesis = blocks.genesis(env)
+    genesis = blocks_genesis(env)
     for k, v in list(env.config['GENESIS_INITIAL_ALLOC'].items()):
         assert genesis.get_balance(k) == v.get("balance", 0) or v.get("wei", 0)
 
 
 @pytest.mark.xfail  # code not in sync with genesis fixtures
 def test_genesis_hash(genesis_fixture):
-    genesis = blocks.genesis(new_env())
+    genesis = blocks_genesis(new_env())
     assert genesis.hex_hash() == utils.to_string(genesis_fixture['genesis_hash'])
 
 
 if __name__ == '__main__':
-    print('current genesis:', blocks.genesis(new_env()).hex_hash())
+    print('current genesis:', blocks_genesis(new_env()).hex_hash())

ethereum/tests/test_genesis.py

@@ -7,9 +7,10 @@
 from ethereum.parse_genesis_declaration import mk_basic_state
 from ethereum import abi
 from ethereum.casper_utils import RandaoManager, generate_validation_code, call_casper, \
-    get_skips_and_block_making_time, sign_block, get_casper_ct, make_casper_genesis, \
-    validator_sizes, find_indices, get_timestamp, make_withdrawal_signature, \
-    casper_config
+    get_skips_and_block_making_time, sign_block, get_contract_code, \
+    casper_config, get_casper_ct, get_casper_code, get_rlp_decoder_code, \
+    get_hash_without_ed_code, make_casper_genesis, get_timestamp, \
+    make_withdrawal_signature
 from ethereum.block_creation import make_head_candidate
 from ethereum.transaction_queue import TransactionQueue
 from ethereum.slogging import LogRecorder, configure_logging, set_level
@@ -24,67 +25,73 @@
 configure_logging(config_string=config_string)
 
 NUM_PARTICIPANTS = 10
+BLOCK_MAKING_PPB = 10
 
 print 'Initializing privkeys, addresses and randaos for validators'
 privkeys = [utils.sha3(str(i)) for i in range(NUM_PARTICIPANTS)]
 addrs = [utils.privtoaddr(k) for k in privkeys]
 randaos = [RandaoManager(utils.sha3(str(i))) for i in range(NUM_PARTICIPANTS)]
-deposit_sizes = [256] * (NUM_PARTICIPANTS // 4) + [128] * (NUM_PARTICIPANTS - (NUM_PARTICIPANTS // 4))
-assert len(privkeys) == len(addrs) == len(randaos) == len(deposit_sizes) == NUM_PARTICIPANTS
+deposit_sizes = [i * 500 + 500 for i in range(NUM_PARTICIPANTS)]
+vcodes = [generate_validation_code(a) for a in addrs]
+vchashes = [utils.sha3(c) for c in vcodes]
+assert len(privkeys) == len(addrs) == len(randaos) == len(deposit_sizes) == len(vcodes) == len(vchashes) == NUM_PARTICIPANTS
 
 # Creating casper contract translator
 ct = get_casper_ct()
 assert ct
 print 'Constructing genesis'
-s = make_casper_genesis(validators=[(generate_validation_code(a), ds * 10**18, r.get(9999))
+s = make_casper_genesis(validators=[(generate_validation_code(a), ds * 10**18, r.get(9999), a)
                                     for a, ds, r in zip(addrs, deposit_sizes, randaos)][:-1],
                         alloc={a: {'balance': 10**18} for a in addrs},
                         timestamp=int(time.time() - 99999),
                         epoch_length=100)
 print 'Genesis constructed successfully'
 chains = [Chain(s.to_snapshot(), env=s.env) for i in range(NUM_PARTICIPANTS)]
+withdrawal_time_1 = call_casper(chains[0].state, 'getLockDuration', [vchashes[0]])
+
+# List of validator IDs that created each block
+vids = []
 
 # Create and sign a block
-def make_block(chain, key, randao, indices, skips):
+def make_block(chain, key, randao, vchash, skips):
     h = make_head_candidate(chain, TransactionQueue(), timestamp=get_timestamp(chain, skips))
-    return sign_block(h, key, randao.get_parent(call_casper(chain.state, 'getRandao', [indices[0], indices[1]])), indices, skips)
+    return sign_block(h, key, randao.get_parent(call_casper(chain.state, 'getRandao', [vchash])), vchash, skips)
 
 next_validator = call_casper(s, 'getValidator', [0])
-print 'Next validator:', next_validator
-indices = [find_indices(s, generate_validation_code(addrs[i]))[:2] for i in range(len(addrs) - 1)]
-print indices, next_validator
-next_validator_id = indices.index(next_validator)
-
-print 'Index in set:', next_validator_id
+print 'Next validator:', next_validator.encode('hex')
+next_validator_id = vchashes.index(next_validator)
+print 'Next validator index:', next_validator_id
 
-skip_count, timestamp = get_skips_and_block_making_time(chains[next_validator_id], indices[next_validator_id])
+skip_count, timestamp = get_skips_and_block_making_time(chains[0].state, next_validator)
 assert skip_count == 0
 b = make_block(chains[0], privkeys[next_validator_id],
-               randaos[next_validator_id], indices[next_validator_id], skip_count)
+               randaos[next_validator_id], vchashes[next_validator_id], skip_count)
 # Validate it
 print 'Block timestamp:', b.header.timestamp
 initialize(s, b)
 print 'Validating block'
 assert validate_block_header(s, b.header)
 print 'Validation successful'
 assert chains[0].add_block(b)
+vids.append(next_validator_id)
 print 'Block added to chain'
 # Make another block
 next_validator = call_casper(chains[0].state, 'getValidator', [0])
-next_validator_id = indices.index(next_validator)
-skip_count, timestamp = get_skips_and_block_making_time(chains[0], next_validator)
+next_validator_id = vchashes.index(next_validator)
+skip_count, timestamp = get_skips_and_block_making_time(chains[0].state, next_validator)
 assert skip_count == 0
 b2 = make_block(chains[0], privkeys[next_validator_id],
-                randaos[next_validator_id], indices[next_validator_id], skip_count)
+                randaos[next_validator_id], vchashes[next_validator_id], skip_count)
 assert chains[0].add_block(b2)
+vids.append(next_validator_id)
 print 'Second block added to chain'
 # Make a dunkle and include it in a transaction
 next_validator = call_casper(chains[1].state, 'getValidator', [1])
-next_validator_id = indices.index(next_validator)
-skip_count, timestamp = get_skips_and_block_making_time(chains[next_validator_id], next_validator)
+next_validator_id = vchashes.index(next_validator)
+skip_count, timestamp = get_skips_and_block_making_time(chains[1].state, next_validator)
 assert skip_count == 1
 b3 = make_block(chains[1], privkeys[next_validator_id],
-                randaos[next_validator_id], indices[next_validator_id], skip_count)
+                randaos[next_validator_id], vchashes[next_validator_id], skip_count)
 print 'Dunkle produced'
 t = Transaction(0, 0, 10**6, casper_config['CASPER_ADDR'], 0, ct.encode('includeDunkle', [rlp.encode(b3.header)])).sign(privkeys[0])
 apply_transaction(chains[0].state, t)
@@ -104,51 +111,61 @@ def make_block(chain, key, randao, indices, skips):
 t2 = Transaction(chains[0].state.get_nonce(a), 0, 1000000, casper_config['CASPER_ADDR'],
                  ds * 10**18, ct.encode('deposit', [vc, r.get(9999)])).sign(k)
 apply_transaction(chains[0].state, t2)
-indices.append(find_indices(chains[0].state, vc)[:2])
-assert indices[-1]
-assert call_casper(chains[0].state, 'getStartEpoch', indices[-1]) == 2
+assert call_casper(chains[0].state, 'getStartEpoch', [vchashes[-1]]) == 2
 chains[0].state.commit()
-print 'Added new validator "in-flight", indices:', indices[-1]
+print 'Added new validator "in-flight", indices:', vchashes[-1].encode('hex')
 # Create some blocks
-vids = []
 bn = call_casper(chains[0].state, 'getBlockNumber')
 for i in range(bn + 1, 200):
     next_validator = call_casper(chains[0].state, 'getValidator', [0])
-    next_validator_id = indices.index(next_validator)
+    next_validator_id = vchashes.index(next_validator)
     b = make_block(chains[0], privkeys[next_validator_id], randaos[next_validator_id],
-                   indices[next_validator_id], 0)
+                   vchashes[next_validator_id], 0)
     assert chains[0].add_block(b)
     vids.append(next_validator_id)
 print 'Created 200 blocks before any deposits/widthdraws, created by validators:', vids
-assert len(indices) - 1 not in vids
+assert len(vchashes) - 1 not in vids
 assert 0 in vids
 # Remove a validator
 sigdata = make_withdrawal_signature(privkeys[0])
-txdata = ct.encode('startWithdrawal', [indices[0][0], indices[0][1], sigdata])
+txdata = ct.encode('startWithdrawal', [vchashes[0], sigdata])
 t3 = Transaction(chains[0].state.get_nonce(addrs[0]), 0, 1000000, casper_config['CASPER_ADDR'], 0, txdata).sign(privkeys[0])
 apply_transaction(chains[0].state, t3)
-assert call_casper(chains[0].state, 'getEndEpoch', indices[0]) == 4
+assert call_casper(chains[0].state, 'getEndEpoch', [vchashes[0]]) == 4
 chains[0].state.commit()
 print 'Withdrew a validator'
+print '%d blocks before ETH becomes available' % withdrawal_time_1
 for i in range(200, 400):
     next_validator = call_casper(chains[0].state, 'getValidator', [0])
-    next_validator_id = indices.index(next_validator)
+    next_validator_id = vchashes.index(next_validator)
     b = make_block(chains[0], privkeys[next_validator_id], randaos[next_validator_id],
-                   indices[next_validator_id], 0)
+                   vchashes[next_validator_id], 0)
     assert b.header.number == i
     assert chains[0].add_block(b)
     vids.append(next_validator_id)
 print 'Created 200 blocks after the deposit, created by validators:', vids[-200:]
-assert len(indices) - 1 in vids
+assert len(vchashes) - 1 in vids
 assert 0 in vids
-for i in range(400, 600):
+for i in range(400, 400 + withdrawal_time_1 + 1):
     next_validator = call_casper(chains[0].state, 'getValidator', [0])
-    next_validator_id = indices.index(next_validator)
+    next_validator_id = vchashes.index(next_validator)
     b = make_block(chains[0], privkeys[next_validator_id], randaos[next_validator_id],
-                   indices[next_validator_id], 0)
+                   vchashes[next_validator_id], 0)
     assert chains[0].add_block(b)
     vids.append(next_validator_id)
-print 'Created 200 blocks after the withdrawal, created by validators:', vids[-200:]
-assert len(indices) - 1 in vids
+print 'Created %d blocks after the withdrawal, created by validators:' % (withdrawal_time_1 + 1), vids[-200:]
+assert len(vchashes) - 1 in vids
 assert 0 not in vids[-200:]
+pre_bal = chains[0].state.get_balance(addrs[0])
+txdata = ct.encode('withdraw', [vchashes[0]])
+t4 = Transaction(chains[0].state.get_nonce(addrs[0]), 0, 1000000, casper_config['CASPER_ADDR'], 0, txdata).sign(privkeys[0])
+apply_transaction(chains[0].state, t4)
+post_bal = chains[0].state.get_balance(addrs[0])
+print 'Wei withdrawn:', post_bal - pre_bal
+blocks_by_v0_in_stage1 = len([x for x in vids[:200] if x == 0])
+expected_revenue_in_stage1 = blocks_by_v0_in_stage1 * max(sum(deposit_sizes[:-1]), 1000000) * 10**18 * BLOCK_MAKING_PPB / 10**9
+blocks_by_v0_in_stage2 = len([x for x in vids[200:400] if x == 0])
+expected_revenue_in_stage2 = blocks_by_v0_in_stage2 * max(sum(deposit_sizes), 1000000) * 10**18 * BLOCK_MAKING_PPB / 10**9
+
+assert post_bal - pre_bal == deposit_sizes[0] * 10**18 + expected_revenue_in_stage1 + expected_revenue_in_stage2
 print 'PoS test fully passed'

ethereum/tests/test_pos.py

@@ -26,7 +26,7 @@ def test_transaction(filename, testname, testdata):
         tx = rlp.decode(rlpdata, transactions.Transaction)
         blknum = int(testdata["blocknumber"])
         if blknum >= 1000000:  # config.default_config["HOMESTEAD_FORK_BLKNUM"]:
-            tx.check_low_s()
+            tx.check_low_s_homestead()
         o["sender"] = tx.sender
         o["transaction"] = {
             "data": b'0x' * (len(tx.data) > 0) + encode_hex(tx.data),