Merge pull request #1149 from pipermerriam/piper/expand-round-trip-to-other-apis

Expand round trip to GetNodeData and NodeData commands

Author: pipermerriam

tests/conftest.py

@@ -27,7 +27,7 @@
 
 
 @pytest.fixture(autouse=True, scope="session")
-def vm_logger(namespaces=LOGGING_NAMESPACES):
+def _stdout_logging(namespaces=LOGGING_NAMESPACES):
     for namespace in namespaces:
         logger = logging.getLogger(namespace)
 
@@ -43,14 +43,14 @@ def vm_logger(namespaces=LOGGING_NAMESPACES):
 
         logger.addHandler(handler)
 
-        return logger
+        logger.info('Set level for logger: %s', namespace)
 
 
 # Uncomment this to have logs from tests written to a file.  This is useful for
 # debugging when you need to dump the VM output from test runs.
 """
 @pytest.yield_fixture(autouse=True)
-def vm_file_logger(request):
+def _file_logging(request):
     import datetime
     import os
 

tests/json-fixtures/test_state.py

@@ -1,3 +1,4 @@
+import logging
 import os
 
 import pytest
@@ -51,16 +52,14 @@
     ForkName
 )
 
-from tests.conftest import vm_logger
-
 
 ROOT_PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 
 
 BASE_FIXTURE_PATH = os.path.join(ROOT_PROJECT_DIR, 'fixtures', 'GeneralStateTests')
 
 
-LOGGER = vm_logger()
+logger = logging.getLogger('eth.tests.fixtures.GeneralStateTests')
 
 
 @to_tuple
@@ -314,7 +313,7 @@ def test_state_fixtures(fixture, fixture_vm_class):
     except ValidationError as err:
         block = vm.block
         transaction_error = err
-        LOGGER.warn("Got transaction error", exc_info=True)
+        logger.warn("Got transaction error", exc_info=True)
     else:
         transaction_error = False
 

tests/p2p/test_peer_collect_sub_proto_msgs.py

@@ -5,7 +5,10 @@
 
 from trinity.protocol.eth.peer import ETHPeer
 from trinity.protocol.eth.commands import GetBlockHeaders, GetNodeData
-from trinity.protocol.eth.requests import HeaderRequest
+from trinity.protocol.eth.requests import (
+    HeaderRequest,
+    NodeDataRequest,
+)
 
 from tests.trinity.core.peer_helpers import (
     get_directly_linked_peers,
@@ -27,11 +30,11 @@ async def test_peer_subscriber_filters_messages(request, event_loop):
 
     with peer.collect_sub_proto_messages() as collector:
         assert collector in peer._subscribers
-        remote.sub_proto.send_get_node_data([b'\x00' * 32])
+        remote.sub_proto.send_get_node_data(NodeDataRequest([b'\x00' * 32]))
         remote.sub_proto.send_get_block_headers(HeaderRequest(0, 1, 0, False))
-        remote.sub_proto.send_get_node_data([b'\x00' * 32])
+        remote.sub_proto.send_get_node_data(NodeDataRequest([b'\x00' * 32]))
         remote.sub_proto.send_get_block_headers(HeaderRequest(1, 1, 0, False))
-        remote.sub_proto.send_get_node_data([b'\x00' * 32])
+        remote.sub_proto.send_get_node_data(NodeDataRequest([b'\x00' * 32]))
         await asyncio.sleep(0.01)
 
     assert collector not in peer._subscribers

tests/p2p/test_peer_subscriber.py

@@ -8,7 +8,10 @@
 
 from trinity.protocol.eth.peer import ETHPeer
 from trinity.protocol.eth.commands import GetBlockHeaders
-from trinity.protocol.eth.requests import HeaderRequest
+from trinity.protocol.eth.requests import (
+    HeaderRequest,
+    NodeDataRequest,
+)
 
 from tests.trinity.core.peer_helpers import (
     get_directly_linked_peers,
@@ -45,11 +48,11 @@ async def test_peer_subscriber_filters_messages(request, event_loop):
     peer.add_subscriber(header_subscriber)
     peer.add_subscriber(all_subscriber)
 
-    remote.sub_proto.send_get_node_data([b'\x00' * 32])
+    remote.sub_proto.send_get_node_data(NodeDataRequest([b'\x00' * 32]))
     remote.sub_proto.send_get_block_headers(HeaderRequest(0, 1, 0, False))
-    remote.sub_proto.send_get_node_data([b'\x00' * 32])
+    remote.sub_proto.send_get_node_data(NodeDataRequest([b'\x00' * 32]))
     remote.sub_proto.send_get_block_headers(HeaderRequest(1, 1, 0, False))
-    remote.sub_proto.send_get_node_data([b'\x00' * 32])
+    remote.sub_proto.send_get_node_data(NodeDataRequest([b'\x00' * 32]))
 
     # yeild to let remote and peer transmit.
     await asyncio.sleep(0.01)

tests/trinity/core/p2p-proto/test_node_data_request_object.py

@@ -0,0 +1,74 @@
+import os
+import random
+
+import pytest
+
+from eth_utils import keccak
+
+from p2p.exceptions import ValidationError
+
+from trinity.protocol.eth.requests import NodeDataRequest
+
+
+def mk_node():
+    node_length = random.randint(0, 2048)
+    node = os.urandom(node_length)
+    return node
+
+
+def mk_node_data(n):
+    if n == 0:
+        return tuple(), tuple()
+    nodes = tuple(set(mk_node() for _ in range(n)))
+    node_keys = tuple(keccak(node) for node in nodes)
+    return node_keys, nodes
+
+
+def test_node_data_request_empty_response_is_valid():
+    node_keys, _ = mk_node_data(10)
+    request = NodeDataRequest(node_keys)
+
+    request.validate_response(tuple())
+
+
+def test_node_data_request_with_full_response():
+    node_keys, nodes = mk_node_data(10)
+    request = NodeDataRequest(node_keys)
+    node_data = tuple(zip(node_keys, nodes))
+
+    request.validate_response(node_data)
+
+
+def test_node_data_request_with_partial_response():
+    node_keys, nodes = mk_node_data(10)
+    request = NodeDataRequest(node_keys)
+    node_data = tuple(zip(node_keys, nodes))
+
+    request.validate_response(node_data[3:])
+    request.validate_response(node_data[:3])
+    request.validate_response((node_data[1], node_data[8], node_data[4]))
+
+
+def test_node_data_request_with_fully_invalid_response():
+    node_keys, nodes = mk_node_data(10)
+    request = NodeDataRequest(node_keys)
+
+    # construct a unique set of other nodes
+    other_nodes = tuple(set(mk_node() for _ in range(10)).difference(nodes))
+    other_node_data = tuple((keccak(node), node) for node in other_nodes)
+
+    with pytest.raises(ValidationError):
+        request.validate_response(other_node_data)
+
+
+def test_node_data_request_with_extra_unrequested_nodes():
+    node_keys, nodes = mk_node_data(10)
+    request = NodeDataRequest(node_keys)
+    node_data = tuple(zip(node_keys, nodes))
+
+    # construct a unique set of other nodes
+    other_nodes = tuple(set(mk_node() for _ in range(10)).difference(nodes))
+    other_node_data = tuple((keccak(node), node) for node in other_nodes)
+
+    with pytest.raises(ValidationError):
+        request.validate_response(node_data + other_node_data)

tests/trinity/core/p2p-proto/test_peer_node_data_request_and_response_api.py

@@ -0,0 +1,133 @@
+import asyncio
+import os
+import random
+
+import pytest
+
+from eth_utils import (
+    keccak,
+)
+
+from trinity.protocol.eth.peer import ETHPeer
+
+from tests.trinity.core.peer_helpers import (
+    get_directly_linked_peers,
+)
+
+
+@pytest.fixture
+async def eth_peer_and_remote(request, event_loop):
+    peer, remote = await get_directly_linked_peers(
+        request,
+        event_loop,
+        peer1_class=ETHPeer,
+        peer2_class=ETHPeer,
+    )
+    return peer, remote
+
+
+def mk_node():
+    node_length = random.randint(0, 2048)
+    node = os.urandom(node_length)
+    return node
+
+
+def mk_node_data(n):
+    if n == 0:
+        return tuple(), tuple()
+    nodes = tuple(set(mk_node() for _ in range(n)))
+    node_keys = tuple(keccak(node) for node in nodes)
+    return node_keys, nodes
+
+
+@pytest.mark.parametrize(
+    'node_keys,nodes',
+    (
+        (
+            (keccak(b''),),
+            (b'',),
+        ),
+        mk_node_data(1),
+        mk_node_data(4),
+        mk_node_data(20),
+        mk_node_data(128),
+        mk_node_data(384),
+    )
+)
+@pytest.mark.asyncio
+async def test_eth_peer_get_node_data_round_trip(eth_peer_and_remote, node_keys, nodes):
+    peer, remote = eth_peer_and_remote
+    node_data = tuple(zip(node_keys, nodes))
+
+    async def send_node_data():
+        remote.sub_proto.send_node_data(nodes)
+
+    asyncio.ensure_future(send_node_data())
+    response = await peer.requests.get_node_data(node_keys)
+
+    assert len(response) == len(node_keys)
+    assert response == node_data
+
+
+@pytest.mark.asyncio
+async def test_eth_peer_get_headers_round_trip_partial_response(eth_peer_and_remote):
+    peer, remote = eth_peer_and_remote
+
+    node_keys, nodes = mk_node_data(32)
+    node_data = tuple(zip(node_keys, nodes))
+
+    async def send_responses():
+        remote.sub_proto.send_transactions([])
+        await asyncio.sleep(0)
+        remote.sub_proto.send_node_data(nodes[:10])
+        await asyncio.sleep(0)
+
+    asyncio.ensure_future(send_responses())
+    response = await peer.requests.get_node_data(node_keys)
+
+    assert len(response) == 10
+    assert response[:10] == node_data[:10]
+
+
+@pytest.mark.asyncio
+async def test_eth_peer_get_headers_round_trip_with_noise(eth_peer_and_remote):
+    peer, remote = eth_peer_and_remote
+
+    node_keys, nodes = mk_node_data(32)
+    node_data = tuple(zip(node_keys, nodes))
+
+    async def send_responses():
+        remote.sub_proto.send_transactions([])
+        await asyncio.sleep(0)
+        remote.sub_proto.send_node_data(nodes)
+        await asyncio.sleep(0)
+
+    asyncio.ensure_future(send_responses())
+    response = await peer.requests.get_node_data(node_keys)
+
+    assert len(response) == len(nodes)
+    assert response == node_data
+
+
+@pytest.mark.asyncio
+async def test_eth_peer_get_headers_round_trip_does_not_match_invalid_response(eth_peer_and_remote):
+    peer, remote = eth_peer_and_remote
+
+    node_keys, nodes = mk_node_data(32)
+    node_data = tuple(zip(node_keys, nodes))
+
+    wrong_nodes = tuple(set(mk_node() for _ in range(32)).difference(nodes))
+
+    async def send_responses():
+        remote.sub_proto.send_node_data(wrong_nodes)
+        await asyncio.sleep(0)
+        remote.sub_proto.send_transactions([])
+        await asyncio.sleep(0)
+        remote.sub_proto.send_node_data(nodes)
+        await asyncio.sleep(0)
+
+    asyncio.ensure_future(send_responses())
+    response = await peer.requests.get_node_data(node_keys)
+
+    assert len(response) == len(nodes)
+    assert response == node_data

tests/trinity/core/p2p-proto/test_server.py

@@ -130,7 +130,7 @@ async def mock_do_handshake(peer):
 async def test_peer_pool_connect(monkeypatch, event_loop, receiver_server_with_dumb_peer):
     started_peers = []
 
-    def mock_start_peer(peer):
+    async def mock_start_peer(peer):
         nonlocal started_peers
         started_peers.append(peer)
 

trinity/p2p/handlers.py

@@ -77,7 +77,7 @@ async def handle_get_node_data(self, peer: ETHPeer, node_hashes: List[Hash32]) -
                 continue
             nodes.append(node)
         self.logger.trace("Replying to %s with %d trie nodes", peer, len(nodes))
-        peer.sub_proto.send_node_data(nodes)
+        peer.sub_proto.send_node_data(tuple(nodes))
 
     async def lookup_headers(self,
                              request: BaseHeaderRequest) -> Tuple[BlockHeader, ...]: