Test w/ pre-built db fixtures unzipped to a tmpdir

Author: carver

tests/trinity/core/integration_fixture_builders.py

@@ -0,0 +1,23 @@
+from eth_keys import keys
+from eth_utils import decode_hex
+
+from tests.trinity.core.integration_test_helpers import FUNDED_ACCT, load_mining_chain
+
+RECEIVER = keys.PrivateKey(
+    decode_hex("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"))
+
+
+def build_pow_fixture(write_db, num_blocks=20):
+    chain = load_mining_chain(write_db)
+    for i in range(num_blocks):
+        tx = chain.create_unsigned_transaction(
+            nonce=i,
+            gas_price=1234,
+            gas=123400,
+            to=RECEIVER.public_key.to_canonical_address(),
+            value=i,
+            data=b'',
+        )
+        chain.apply_transaction(tx.as_signed_transaction(FUNDED_ACCT))
+        chain.mine_block()
+    return chain.chaindb

tests/trinity/core/integration_test_helpers.py

@@ -1,19 +1,34 @@
 import asyncio
+from enum import Enum
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from zipfile import ZipFile
 
 from cancel_token import OperationCancelled
+from eth_keys import keys
+from eth_utils import decode_hex
 
-from eth import MainnetChain, RopstenChain
+from eth import MainnetChain, RopstenChain, constants
 from eth.chains.base import (
     MiningChain,
 )
 from eth.db.backends.level import LevelDB
 from eth.db.backends.memory import MemoryDB
 from eth.db.atomic import AtomicDB
+from eth.tools.builder.chain import (
+    build,
+    byzantium_at,
+    enable_pow_mining,
+    genesis,
+)
+from eth.vm.forks.byzantium import ByzantiumVM
 
 from trinity.db.base import AsyncBaseDB
 from trinity.db.chain import AsyncChainDB
 from trinity.db.header import AsyncHeaderDB
 
+ZIPPED_FIXTURES_PATH = Path(__file__).parent.parent / 'integration' / 'fixtures'
+
 
 async def connect_to_peers_loop(peer_pool, nodes):
     """Loop forever trying to connect to one of the given nodes if the pool is not yet full."""
@@ -96,3 +111,54 @@ class FakeAsyncChain(MiningChain):
     coro_import_block = coro_import_block
     coro_validate_chain = async_passthrough('validate_chain')
     coro_validate_receipt = async_passthrough('validate_receipt')
+    chaindb_class = FakeAsyncChainDB
+
+
+class ByzantiumTestChain(FakeAsyncChain):
+    vm_configuration = ((0, ByzantiumVM),)
+    network_id = 999
+
+
+FUNDED_ACCT = keys.PrivateKey(
+    decode_hex("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee"))
+
+
+def load_mining_chain(db):
+    GENESIS_PARAMS = {
+        'coinbase': constants.ZERO_ADDRESS,
+        'difficulty': 5,
+        'gas_limit': 3141592,
+        'timestamp': 1514764800,
+    }
+
+    GENESIS_STATE = {
+        FUNDED_ACCT.public_key.to_canonical_address(): {
+            "balance": 100000000000000000,
+        }
+    }
+
+    return build(
+        FakeAsyncChain,
+        byzantium_at(0),
+        enable_pow_mining(),
+        genesis(db=db, params=GENESIS_PARAMS, state=GENESIS_STATE),
+    )
+
+
+class DBFixture(Enum):
+    twenty_pow_headers = '20pow_headers.ldb'
+
+
+def load_fixture_db(db_fixture, db_class=LevelDB):
+    """
+    Extract the database from the zip file to a temp directory, which has two benefits and one cost:
+    - B1. works with xdist, multiple access to the ldb files at the same time
+    - B2. prevents dirty-ing the git index
+    - C1. slows down test, because of time to extract files
+    """
+    assert isinstance(db_fixture, DBFixture)
+    zipped_path = ZIPPED_FIXTURES_PATH / f"{db_fixture.value}.zip"
+
+    with ZipFile(zipped_path, 'r') as zipped, TemporaryDirectory() as tmpdir:
+        zipped.extractall(tmpdir)
+        yield db_class(Path(tmpdir) / db_fixture.value)

tests/trinity/core/p2p-proto/test_sync.py

@@ -2,14 +2,6 @@
 
 import pytest
 
-from eth_keys import keys
-from eth_utils import decode_hex
-
-from eth import constants
-from eth.tools.mining import POWMiningMixin
-from eth.vm.forks.frontier import FrontierVM
-
-
 from trinity.protocol.eth.servers import ETHRequestServer
 from trinity.protocol.les.peer import LESPeer
 from trinity.protocol.les.servers import LightRequestServer
@@ -18,10 +10,13 @@
 from trinity.sync.light.chain import LightChainSyncer
 
 from tests.trinity.core.integration_test_helpers import (
-    FakeAsyncChain,
+    ByzantiumTestChain,
     FakeAsyncChainDB,
     FakeAsyncHeaderDB,
     FakeAsyncAtomicDB,
+    load_fixture_db,
+    load_mining_chain,
+    DBFixture,
 )
 from tests.trinity.core.peer_helpers import (
     get_directly_linked_peers,
@@ -46,10 +41,10 @@ async def test_fast_syncer(request, event_loop, chaindb_fresh, chaindb_20):
         alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
         bob_headerdb=FakeAsyncHeaderDB(chaindb_20.db))
     client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer])
-    client = FastChainSyncer(FrontierTestChain(chaindb_fresh.db), chaindb_fresh, client_peer_pool)
+    client = FastChainSyncer(ByzantiumTestChain(chaindb_fresh.db), chaindb_fresh, client_peer_pool)
     server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer])
     server = RegularChainSyncer(
-        FrontierTestChain(chaindb_20.db),
+        ByzantiumTestChain(chaindb_20.db),
         chaindb_20,
         server_peer_pool,
     )
@@ -86,12 +81,12 @@ async def test_regular_syncer(request, event_loop, chaindb_fresh, chaindb_20):
         alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
         bob_headerdb=FakeAsyncHeaderDB(chaindb_20.db))
     client = RegularChainSyncer(
-        FrontierTestChain(chaindb_fresh.db),
+        ByzantiumTestChain(chaindb_fresh.db),
         chaindb_fresh,
         MockPeerPoolWithConnectedPeers([client_peer]))
     server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer])
     server = RegularChainSyncer(
-        FrontierTestChain(chaindb_20.db),
+        ByzantiumTestChain(chaindb_20.db),
         chaindb_20,
         server_peer_pool,
     )
@@ -126,12 +121,12 @@ async def test_light_syncer(request, event_loop, chaindb_fresh, chaindb_20):
         alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
         bob_headerdb=FakeAsyncHeaderDB(chaindb_20.db))
     client = LightChainSyncer(
-        FrontierTestChain(chaindb_fresh.db),
+        ByzantiumTestChain(chaindb_fresh.db),
         chaindb_fresh,
         MockPeerPoolWithConnectedPeers([client_peer]))
     server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer])
     server = LightChainSyncer(
-        FrontierTestChain(chaindb_20.db),
+        ByzantiumTestChain(chaindb_20.db),
         chaindb_20,
         server_peer_pool,
     )
@@ -157,72 +152,24 @@ def finalizer():
 
 
 @pytest.fixture
-def chaindb_20():
-    chain = PoWMiningChain.from_genesis(FakeAsyncAtomicDB(), GENESIS_PARAMS, GENESIS_STATE)
-    for i in range(20):
-        tx = chain.create_unsigned_transaction(
-            nonce=i,
-            gas_price=1234,
-            gas=1234000,
-            to=RECEIVER.public_key.to_canonical_address(),
-            value=i,
-            data=b'',
-        )
-        chain.apply_transaction(tx.as_signed_transaction(SENDER))
-        chain.mine_block()
+def leveldb_20():
+    yield from load_fixture_db(DBFixture.twenty_pow_headers)
+
+
+@pytest.fixture
+def chaindb_20(leveldb_20):
+    chain = load_mining_chain(FakeAsyncAtomicDB(leveldb_20))
+    assert chain.chaindb.get_canonical_head().block_number == 20
     return chain.chaindb
 
 
 @pytest.fixture
 def chaindb_fresh():
-    chain = PoWMiningChain.from_genesis(FakeAsyncAtomicDB(), GENESIS_PARAMS, GENESIS_STATE)
+    chain = load_mining_chain(FakeAsyncAtomicDB())
     assert chain.chaindb.get_canonical_head().block_number == 0
     return chain.chaindb
 
 
-SENDER = keys.PrivateKey(
-    decode_hex("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee"))
-RECEIVER = keys.PrivateKey(
-    decode_hex("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"))
-GENESIS_PARAMS = {
-    'parent_hash': constants.GENESIS_PARENT_HASH,
-    'uncles_hash': constants.EMPTY_UNCLE_HASH,
-    'coinbase': constants.ZERO_ADDRESS,
-    'transaction_root': constants.BLANK_ROOT_HASH,
-    'receipt_root': constants.BLANK_ROOT_HASH,
-    'bloom': 0,
-    'difficulty': 5,
-    'block_number': constants.GENESIS_BLOCK_NUMBER,
-    'gas_limit': 3141592,
-    'gas_used': 0,
-    'timestamp': 1514764800,
-    'extra_data': constants.GENESIS_EXTRA_DATA,
-    'nonce': constants.GENESIS_NONCE
-}
-GENESIS_STATE = {
-    SENDER.public_key.to_canonical_address(): {
-        "balance": 100000000000000000,
-        "code": b"",
-        "nonce": 0,
-        "storage": {}
-    }
-}
-
-
-class FrontierTestChain(FakeAsyncChain):
-    vm_configuration = ((0, FrontierVM),)
-    chaindb_class = FakeAsyncChainDB
-    network_id = 999
-
-
-class POWFrontierVM(POWMiningMixin, FrontierVM):
-    pass
-
-
-class PoWMiningChain(FrontierTestChain):
-    vm_configuration = ((0, POWFrontierVM),)
-
-
 async def wait_for_head(headerdb, header):
     # A full header sync may involve several round trips, so we must be willing to wait a little
     # bit for them.