Merge pull request #1930 from carver/gap-filling-test

Handle a lot more scenarios of filling header gaps, created by checkpoints

Author: carver

eth/db/chain.py

@@ -4,6 +4,7 @@
 from typing import (
     Dict,
     Iterable,
+    Sequence,
     Tuple,
     Type,
 )
@@ -94,14 +95,14 @@ def get_block_uncles(self, uncles_hash: Hash32) -> Tuple[BlockHeaderAPI, ...]:
     def _decanonicalize_old_headers(
         cls,
         db: DatabaseAPI,
-        new_canonical_headers: Tuple[BlockHeaderAPI, ...]
+        numbers_to_decanonicalize: Sequence[BlockNumber],
     ) -> Tuple[BlockHeaderAPI, ...]:
         old_canonical_headers = []
 
         # remove transaction lookups for blocks that are no longer canonical
-        for h in new_canonical_headers:
+        for block_number in numbers_to_decanonicalize:
             try:
-                old_hash = cls._get_canonical_block_hash(db, h.block_number)
+                old_hash = cls._get_canonical_block_hash(db, block_number)
             except HeaderNotFound:
                 # no old block, and no more possible
                 break

eth/db/chain_gaps.py

@@ -1,7 +1,11 @@
 import enum
-from typing import Tuple
+from typing import Iterable, Tuple
 
 from eth_typing import BlockNumber
+from eth_utils import (
+    ValidationError,
+    to_tuple,
+)
 
 from eth.exceptions import GapTrackingCorrupted
 from eth.typing import BlockRange, ChainGaps
@@ -12,17 +16,87 @@ class GapChange(enum.Enum):
     NewGap = enum.auto()
     GapFill = enum.auto()
     GapSplit = enum.auto()
-    GapShrink = enum.auto()
+    GapLeftShrink = enum.auto()
+    GapRightShrink = enum.auto()
     TailWrite = enum.auto()
 
 
-GAP_WRITES = (GapChange.GapFill, GapChange.GapSplit, GapChange.GapShrink)
+GAP_WRITES = (
+    GapChange.GapFill,
+    GapChange.GapSplit,
+    GapChange.GapLeftShrink,
+    GapChange.GapRightShrink,
+)
 GENESIS_CHAIN_GAPS = ((), BlockNumber(1))
 
 GapInfo = Tuple[GapChange, ChainGaps]
 
 
-def calculate_gaps(newly_persisted: BlockNumber, base_gaps: ChainGaps) -> GapInfo:
+@to_tuple
+def _join_overlapping_gaps(unjoined_gaps: Tuple[BlockRange, ...]) -> Iterable[BlockRange]:
+    """
+    After introducing a new gap, join any that overlap.
+    Input must already be sorted.
+    """
+    unyielded_low = None
+    unyielded_high = None
+    for low, high in unjoined_gaps:
+        if unyielded_high is not None:
+            if low < unyielded_low:
+                raise ValidationError(f"Unsorted input! {unjoined_gaps!r}")
+            elif unyielded_low <= low <= unyielded_high + 1:
+                unyielded_high = max(high, unyielded_high)
+                continue
+            else:
+                yield unyielded_low, unyielded_high
+
+        unyielded_low = low
+        unyielded_high = high
+
+    if unyielded_high is not None:
+        yield unyielded_low, unyielded_high
+
+
+def reopen_gap(decanonicalized: BlockNumber, base_gaps: ChainGaps) -> ChainGaps:
+    """
+    Add a new gap, for a header that was decanonicalized.
+    """
+    current_gaps, tip_child = base_gaps
+
+    if tip_child <= decanonicalized:
+        return base_gaps
+
+    new_raw_gaps = current_gaps + ((decanonicalized, decanonicalized), )
+
+    # join overlapping gaps
+    joined_gaps = _join_overlapping_gaps(sorted(new_raw_gaps))
+
+    # is the last gap overlapping with the tip child? if so, merge it
+    if joined_gaps[-1][1] + 1 >= tip_child:
+        return joined_gaps[:-1], joined_gaps[-1][0]
+    else:
+        return joined_gaps, tip_child
+
+
+def is_block_number_in_gap(block_number: BlockNumber, gaps: ChainGaps) -> bool:
+    """
+    Check if a block number is found in the given gaps
+    """
+    gap_ranges, tip_child = gaps
+    for low, high in gap_ranges:
+        if low > block_number:
+            return False
+        elif high >= block_number:
+            return True
+        # this range was below the block number, continue looking at the next range
+
+    return block_number >= tip_child
+
+
+def fill_gap(newly_persisted: BlockNumber, base_gaps: ChainGaps) -> GapInfo:
+    """
+    Remove a gap, for a new header that was canonicalized.
+    """
 
     current_gaps, tip_child = base_gaps
 
@@ -45,11 +119,12 @@ def calculate_gaps(newly_persisted: BlockNumber, base_gaps: ChainGaps) -> GapInf
         ]
 
         if len(matching_gaps) > 1:
+            first_match, second_match, *_ = matching_gaps
             raise GapTrackingCorrupted(
                 "Corrupted chain gap tracking",
                 f"No. {newly_persisted} appears to be missing in multiple gaps",
-                f"1st gap goes from {matching_gaps[0][1][0]} to {matching_gaps[0][1][1]}"
-                f"2nd gap goes from {matching_gaps[1][1][0]} to {matching_gaps[1][1][1]}"
+                f"1st gap is {first_match[1]}, 2nd gap is {second_match[1]}",
+                f"all matching gaps: {matching_gaps}",
             )
         elif len(matching_gaps) == 0:
             # Looks like we are just overwriting an existing header.
@@ -62,11 +137,11 @@ def calculate_gaps(newly_persisted: BlockNumber, base_gaps: ChainGaps) -> GapInf
             elif newly_persisted == gap[0]:
                 # we are shrinking the gap at the start
                 updated_center = ((BlockNumber(gap[0] + 1), gap[1],),)
-                gap_change = GapChange.GapShrink
+                gap_change = GapChange.GapLeftShrink
             elif newly_persisted == gap[1]:
                 # we are shrinking the gap at the tail
                 updated_center = ((gap[0], BlockNumber(gap[1] - 1),),)
-                gap_change = GapChange.GapShrink
+                gap_change = GapChange.GapRightShrink
             elif gap[0] < newly_persisted < gap[1]:
                 # we are dividing the gap
                 first_new_gap = (gap[0], BlockNumber(newly_persisted - 1))