Merge pull request #1169 from pipermerriam/piper/expand-round-trip-api-to-block-bodies

Expand round trip api to block bodies

Author: pipermerriam

eth/db/trie.py

@@ -1,20 +1,24 @@
 import functools
-from typing import Dict, List, Tuple, Union
+from typing import Dict, Tuple, Union
 
 import rlp
 from trie import (
     HexaryTrie,
 )
 
+from eth_typing import Hash32
+
 from eth.constants import (
     BLANK_ROOT_HASH,
 )
 from eth.rlp.receipts import Receipt
 from eth.rlp.transactions import BaseTransaction
 
+TransactionsOrReceipts = Union[Tuple[Receipt, ...], Tuple[BaseTransaction, ...]]
+TrieRootAndData = Tuple[Hash32, Dict[Hash32, bytes]]
+
 
-def make_trie_root_and_nodes(
-        items: Union[List[Receipt], List[BaseTransaction]]) -> Tuple[bytes, Dict[bytes, bytes]]:
+def make_trie_root_and_nodes(items: TransactionsOrReceipts) -> TrieRootAndData:
     return _make_trie_root_and_nodes(tuple(rlp.encode(item) for item in items))
 
 
@@ -23,8 +27,8 @@ def make_trie_root_and_nodes(
 # as it's common for them to have duplicate receipt_roots. Given that, it probably makes sense to
 # use a relatively small cache size here.
 @functools.lru_cache(128)
-def _make_trie_root_and_nodes(items: Tuple[bytes, ...]) -> Tuple[bytes, Dict[bytes, bytes]]:
-    kv_store = {}  # type: Dict[bytes, bytes]
+def _make_trie_root_and_nodes(items: Tuple[bytes, ...]) -> TrieRootAndData:
+    kv_store = {}  # type: Dict[Hash32, bytes]
     trie = HexaryTrie(kv_store, BLANK_ROOT_HASH)
     with trie.squash_changes() as memory_trie:
         for index, item in enumerate(items):

p2p/peer.py

@@ -1063,16 +1063,16 @@ async def request_stuff() -> None:
             peer_pool.logger.info("Waiting for peer connection...")
             await asyncio.sleep(0.2)
         peer = peer_pool.highest_td_peer
-        headers = await peer.requests.get_block_headers(2440319, max_headers=100)  # type: ignore
-        hashes = [header.hash for header in headers]
+        headers = await cast(ETHPeer, peer).requests.get_block_headers(2440319, max_headers=100)
+        hashes = tuple(header.hash for header in headers)
         if peer_class == ETHPeer:
             peer = cast(ETHPeer, peer)
-            peer.sub_proto.send_get_block_bodies(hashes)
-            peer.sub_proto.send_get_receipts(hashes)
+            peer.sub_proto._send_get_block_bodies(hashes)
+            peer.sub_proto._send_get_receipts(hashes)
         else:
             peer = cast(LESPeer, peer)
             request_id = 1
-            peer.sub_proto.send_get_block_bodies(hashes, request_id + 1)
+            peer.sub_proto.send_get_block_bodies(list(hashes), request_id + 1)
             peer.sub_proto.send_get_receipts(hashes[0], request_id + 2)
 
     sigint_received = asyncio.Event()

tests/trinity/core/p2p-proto/test_block_bodies_request_object.py

@@ -0,0 +1,143 @@
+import os
+import random
+import time
+
+import pytest
+
+import rlp
+
+from eth_hash.auto import keccak
+
+from eth_utils import (
+    to_tuple,
+    big_endian_to_int,
+)
+
+from eth.db.trie import make_trie_root_and_nodes
+from eth.rlp.headers import BlockHeader
+from eth.rlp.transactions import BaseTransactionFields
+
+from p2p.exceptions import ValidationError
+
+from trinity.rlp.block_body import BlockBody
+from trinity.protocol.eth.requests import BlockBodiesRequest
+
+
+def mk_uncle(block_number):
+    return BlockHeader(
+        state_root=os.urandom(32),
+        difficulty=1000000,
+        block_number=block_number,
+        gas_limit=3141592,
+        timestamp=int(time.time()),
+    )
+
+
+def mk_transaction():
+    return BaseTransactionFields(
+        nonce=0,
+        gas=21000,
+        gas_price=1,
+        to=os.urandom(20),
+        value=random.randint(0, 100),
+        data=b'',
+        v=27,
+        r=big_endian_to_int(os.urandom(32)),
+        s=big_endian_to_int(os.urandom(32)),
+    )
+
+
+def mk_header_and_body(block_number, num_transactions, num_uncles):
+    transactions = tuple(mk_transaction() for _ in range(num_transactions))
+    uncles = tuple(mk_uncle(block_number - 1) for _ in range(num_uncles))
+
+    transaction_root, trie_data = make_trie_root_and_nodes(transactions)
+    uncles_hash = keccak(rlp.encode(uncles))
+
+    body = BlockBody(transactions=transactions, uncles=uncles)
+
+    header = BlockHeader(
+        difficulty=1000000,
+        block_number=block_number,
+        gas_limit=3141592,
+        timestamp=int(time.time()),
+        transaction_root=transaction_root,
+        uncles_hash=uncles_hash,
+    )
+
+    return header, body, transaction_root, trie_data, uncles_hash
+
+
+@to_tuple
+def mk_headers(*counts):
+    for idx, (num_transactions, num_uncles) in enumerate(counts, 1):
+        yield mk_header_and_body(idx, num_transactions, num_uncles)
+
+
+def test_block_bodies_request_empty_response_is_valid():
+    headers_bundle = mk_headers((2, 3), (8, 4), (0, 1), (0, 0))
+    headers, _, _, _, _ = zip(*headers_bundle)
+    request = BlockBodiesRequest(headers)
+    request.validate_response(tuple(), tuple())
+
+
+def test_block_bodies_request_valid_with_full_response():
+    headers_bundle = mk_headers((2, 3), (8, 4), (0, 1), (0, 0))
+    headers, bodies, transactions_roots, trie_data_dicts, uncles_hashes = zip(*headers_bundle)
+    transactions_bundles = tuple(zip(transactions_roots, trie_data_dicts))
+    bodies_bundle = tuple(zip(bodies, transactions_bundles, uncles_hashes))
+    request = BlockBodiesRequest(headers)
+    request.validate_response(bodies, bodies_bundle)
+
+
+def test_block_bodies_request_valid_with_partial_response():
+    headers_bundle = mk_headers((2, 3), (8, 4), (0, 1), (0, 0))
+    headers, bodies, transactions_roots, trie_data_dicts, uncles_hashes = zip(*headers_bundle)
+    transactions_bundles = tuple(zip(transactions_roots, trie_data_dicts))
+    bodies_bundle = tuple(zip(bodies, transactions_bundles, uncles_hashes))
+    request = BlockBodiesRequest(headers)
+
+    request.validate_response(bodies[:2], bodies_bundle[:2])
+    request.validate_response(bodies[2:], bodies_bundle[2:])
+    request.validate_response(
+        (bodies[0], bodies[2], bodies[3]),
+        (bodies_bundle[0], bodies_bundle[2], bodies_bundle[3]),
+    )
+
+
+def test_block_bodies_request_with_fully_invalid_response():
+    headers_bundle = mk_headers((2, 3), (8, 4), (0, 1), (0, 0))
+    headers, _, _, _, _ = zip(*headers_bundle)
+
+    wrong_headers_bundle = mk_headers((3, 2), (4, 8), (1, 0), (0, 0))
+    w_headers, w_bodies, w_transactions_roots, w_trie_data_dicts, w_uncles_hashes = zip(
+        *wrong_headers_bundle
+    )
+    w_transactions_bundles = tuple(zip(w_transactions_roots, w_trie_data_dicts))
+    w_bodies_bundle = tuple(zip(w_bodies, w_transactions_bundles, w_uncles_hashes))
+
+    request = BlockBodiesRequest(headers)
+    with pytest.raises(ValidationError):
+        request.validate_response(w_bodies, w_bodies_bundle)
+
+
+def test_block_bodies_request_with_extra_unrequested_bodies():
+    headers_bundle = mk_headers((2, 3), (8, 4), (0, 1), (0, 0))
+    headers, bodies, transactions_roots, trie_data_dicts, uncles_hashes = zip(*headers_bundle)
+    transactions_bundles = tuple(zip(transactions_roots, trie_data_dicts))
+    bodies_bundle = tuple(zip(bodies, transactions_bundles, uncles_hashes))
+    request = BlockBodiesRequest(headers)
+
+    wrong_headers_bundle = mk_headers((3, 2), (4, 8), (1, 0), (0, 0))
+    w_headers, w_bodies, w_transactions_roots, w_trie_data_dicts, w_uncles_hashes = zip(
+        *wrong_headers_bundle
+    )
+    w_transactions_bundles = tuple(zip(w_transactions_roots, w_trie_data_dicts))
+    w_bodies_bundle = tuple(zip(w_bodies, w_transactions_bundles, w_uncles_hashes))
+
+    request = BlockBodiesRequest(headers)
+    with pytest.raises(ValidationError):
+        request.validate_response(
+            bodies + w_bodies,
+            bodies_bundle + w_bodies_bundle,
+        )

tests/trinity/core/p2p-proto/test_node_data_request_object.py

@@ -28,25 +28,28 @@ def test_node_data_request_empty_response_is_valid():
     node_keys, _ = mk_node_data(10)
     request = NodeDataRequest(node_keys)
 
-    request.validate_response(tuple())
+    request.validate_response(tuple(), tuple())
 
 
 def test_node_data_request_with_full_response():
     node_keys, nodes = mk_node_data(10)
     request = NodeDataRequest(node_keys)
     node_data = tuple(zip(node_keys, nodes))
 
-    request.validate_response(node_data)
+    request.validate_response(nodes, node_data)
 
 
 def test_node_data_request_with_partial_response():
     node_keys, nodes = mk_node_data(10)
     request = NodeDataRequest(node_keys)
     node_data = tuple(zip(node_keys, nodes))
 
-    request.validate_response(node_data[3:])
-    request.validate_response(node_data[:3])
-    request.validate_response((node_data[1], node_data[8], node_data[4]))
+    request.validate_response(nodes[3:], node_data[3:])
+    request.validate_response(nodes[:3], node_data[:3])
+    request.validate_response(
+        (nodes[1], nodes[8], nodes[4]),
+        (node_data[1], node_data[8], node_data[4]),
+    )
 
 
 def test_node_data_request_with_fully_invalid_response():
@@ -58,7 +61,7 @@ def test_node_data_request_with_fully_invalid_response():
     other_node_data = tuple((keccak(node), node) for node in other_nodes)
 
     with pytest.raises(ValidationError):
-        request.validate_response(other_node_data)
+        request.validate_response(other_nodes, other_node_data)
 
 
 def test_node_data_request_with_extra_unrequested_nodes():
@@ -71,4 +74,7 @@ def test_node_data_request_with_extra_unrequested_nodes():
     other_node_data = tuple((keccak(node), node) for node in other_nodes)
 
     with pytest.raises(ValidationError):
-        request.validate_response(node_data + other_node_data)
+        request.validate_response(
+            nodes + other_nodes,
+            node_data + other_node_data,
+        )