Merge pull request #1140 from carver/simpler-sequence-builder

Simplify sequence_builder

Author: carver

tests/trinity/core/header-utils/test_block_number_sequence_builder.py

@@ -1,6 +1,8 @@
 import pytest
 
 from eth.constants import UINT_256_MAX
+
+from trinity.exceptions import OversizeObject
 from trinity.utils.headers import sequence_builder
 
 
@@ -22,7 +24,22 @@
         (9, 5, 1, True, (9, 7, 5, 3, 1)),
         (1, 9, 0, True, (1, 0)),
         (UINT_256_MAX - 1, 4, 0, False, (UINT_256_MAX - 1, UINT_256_MAX, )),
+        # can handle mildly large numbers
+        (400000000, 1000000, 0, False, tuple(range(400000000, 401000000))),
     ),
 )
 def test_sequence(start_num, max_length, skip, reverse, expected):
     assert sequence_builder(start_num, max_length, skip, reverse) == expected
+
+
+TOO_LONG = 2000000
+
+
+@pytest.mark.parametrize('reverse', (True, False))
+@pytest.mark.parametrize('start_num', (0, 400000000))
+@pytest.mark.parametrize('skip', (0, 10000))
+def test_oversize_sequence(start_num, skip, reverse):
+    # Instead of using the specific constant, just use a rough TOO_LONG number
+    # We don't need to worry about edge cases for this gut check
+    with pytest.raises(OversizeObject):
+        sequence_builder(start_num, TOO_LONG, skip, reverse)

trinity/constants.py

@@ -1,2 +1,4 @@
 SYNC_FULL = 'full'
 SYNC_LIGHT = 'light'
+
+MAXIMUM_OBJECT_MEMORY_BYTES = 10000000

trinity/exceptions.py

@@ -37,3 +37,10 @@ class SyncRequestAlreadyProcessed(BaseTrinityError):
     Raised when a trie SyncRequest has already been processed.
     """
     pass
+
+
+class OversizeObject(BaseTrinityError):
+    """
+    Raised when an object is bigger than comfortably fits in memory.
+    """
+    pass

trinity/utils/headers.py

@@ -1,26 +1,31 @@
 from typing import (
-    Iterator,
+    Tuple,
 )
 
-from eth_utils import to_tuple
-
 from eth.constants import UINT_256_MAX
 
+from trinity.constants import MAXIMUM_OBJECT_MEMORY_BYTES
+from trinity.exceptions import OversizeObject
+
 
-@to_tuple
 def sequence_builder(start_number: int,
                      max_length: int,
                      skip: int,
-                     reverse: bool) -> Iterator[int]:
+                     reverse: bool) -> Tuple[int, ...]:
+    # Limit the in-memory size of this sequence.
+    # A tuple of 64-bit ints is about 8 bytes per value
+    # Ignore the cutoffs at 0 and UINT_256_MAX, because this is just a gut check anyway,
+    # we should never be approaching this value.
+    if max_length > MAXIMUM_OBJECT_MEMORY_BYTES // 8:
+        raise OversizeObject("Sequence is too big to fit in memory: {}".format(max_length))
+
     if reverse:
         step = -1 * (skip + 1)
     else:
         step = skip + 1
 
     cutoff_number = start_number + step * max_length
 
-    for number in range(start_number, cutoff_number, step):
-        if number < 0 or number > UINT_256_MAX:
-            return
-        else:
-            yield number
+    whole_range = range(start_number, cutoff_number, step)
+
+    return tuple(number for number in whole_range if 0 <= number <= UINT_256_MAX)