v0.1.18 made w3.eth.get_logs more robust and added unit tests for it.

simone1999 · simone1999 · commit 5430e63cea03 · 2024-11-17T22:31:49.000+02:00
diff --git a/IceCreamSwapWeb3/EthAdvanced.py b/IceCreamSwapWeb3/EthAdvanced.py
@@ -136,11 +136,36 @@ def get_logs(
             assert "fromBlock" not in filter_params and "toBlock" not in filter_params
             return self.get_logs_inner(filter_params, no_retry=no_retry)
 
+        from_block_original: BlockIdentifier = filter_params.get("fromBlock", "earliest")
+        to_block_original: BlockIdentifier = filter_params.get("toBlock", "latest")
+
         # sanitizing block numbers, could be strings like "latest"
-        filter_params["fromBlock"] = from_block = self.get_block_number_from_identifier(filter_params.get("fromBlock", "earliest"))
-        filter_params["toBlock"] = to_block = self.get_block_number_from_identifier(filter_params.get("toBlock", "latest"))
+        if isinstance(from_block_original, int):
+            from_block_body = None
+            from_block = from_block_original
+        else:
+            from_block_body = self.get_block(from_block_original)
+            from_block = from_block_body["number"]
+            filter_params = {**filter_params, "fromBlock": from_block}
+
+        if isinstance(to_block_original, int):
+            to_block_body = None
+            to_block = to_block_original
+        else:
+            to_block_body = self.get_block(to_block_original)
+            to_block = to_block_body["number"]
+            filter_params = {**filter_params, "toBlock": to_block}
+
         assert to_block >= from_block, f"{from_block=}, {to_block=}"
 
+        # if logs for a single block are queried, and we know the block hash, query by it
+        if from_block == to_block and (from_block_body or to_block_body):
+            block_body = from_block_body if from_block_body else to_block_body
+            single_hash_filter = {**filter_params, "blockHash": block_body["hash"]}
+            del single_hash_filter["fromBlock"]
+            del single_hash_filter["toBlock"]
+            return self.get_logs_inner(single_hash_filter, no_retry=no_retry)
+
         # note: fromBlock and toBlock are both inclusive. e.g. 5 to 6 are 2 blocks
         num_blocks = to_block - from_block + 1
 
@@ -162,18 +187,31 @@ def get_logs(
         # simply ignores logs from the missing block
         # to prevent this, we get the latest blocks individually by their hashes
         unstable_blocks = self.w3.unstable_blocks
-        if to_block > self.w3.latest_seen_block - unstable_blocks and to_block > self.block_number - unstable_blocks:
+        if to_block > self.w3.latest_seen_block - unstable_blocks and to_block > (last_stable_block := (self.get_block_number() - unstable_blocks)):
             results = []
-            while to_block > self.w3.latest_seen_block - unstable_blocks and to_block >= from_block:
-                single_hash_filter = {**filter_params, "blockHash": self.get_block(to_block)["hash"]}
-                del single_hash_filter["fromBlock"]
-                del single_hash_filter["toBlock"]
-                results += self.get_logs_inner(single_hash_filter, no_retry=no_retry)
-                to_block -= 1
+            if from_block <= last_stable_block:
+                results += self.get_logs({**filter_params, "toBlock": last_stable_block}, **kwargs)
+
+            # get all block hashes and ensure they build upon each other
+            block_hashes = []
+            for block_number in range(max(last_stable_block + 1, from_block), to_block + 1):
+                block = self.get_block(block_number, no_retry=no_retry)
+                if block_hashes:
+                    # make sure chain of blocks is consistent with each block building on the previous one
+                    assert block["parentHash"] == block_hashes[-1], f"{block_hashes[-1]=}, {block['parentHash']=}"
+                if block_number == from_block and from_block_body is not None:
+                    assert block["hash"] == from_block_body["hash"], f"{from_block_body['hash']=}, {block['hash']=}"
+                if block_number == to_block and to_block_body is not None:
+                    assert block["hash"] == to_block_body["hash"], f"{to_block_body['hash']=}, {block['hash']=}"
+                block_hashes.append(block["hash"])
+
+            single_hash_filter = filter_params.copy()
+            del single_hash_filter["fromBlock"]
+            del single_hash_filter["toBlock"]
+            for block_hash in block_hashes:
+                results += self.get_logs_inner({**single_hash_filter, "blockHash": block_hash}, no_retry=no_retry)
                 if p_bar is not None:
                     p_bar.update(1)
-            if to_block >= from_block:
-                results += self.get_logs({**filter_params, "toBlock": to_block}, **kwargs)
             return results
 
         # getting logs for a single block, which is not at the chain head. No drama
@@ -192,7 +230,7 @@ def get_logs(
                 )
                 if till_block >= to_block:
                     return results
-                return results + self.get_logs({**filter_params, "fromBlock": till_block+1}, **kwargs)
+                return results + self.get_logs({**filter_params, "fromBlock": till_block + 1}, **kwargs)
             except Exception as e:
                 print(f"Getting logs from SubSquid threw exception {repr(e)}, falling back to RPC")
 
diff --git a/IceCreamSwapWeb3/FilterTest.py b/IceCreamSwapWeb3/FilterTest.py
@@ -0,0 +1,223 @@
+import unittest
+from unittest.mock import MagicMock, patch
+from . import Web3Advanced
+
+
+class TestWeb3AdvancedGetLogs(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        # Instantiate the class only once
+        cls.eth_advanced = Web3Advanced(node_url="https://rpc-core.icecreamswap.com").eth
+
+    def setUp(self):
+        # Mock self.eth_advanced.w3 and its properties
+        self.eth_advanced.w3 = MagicMock()
+        self.eth_advanced.w3.filter_block_range = 1000  # Set a default filter block range
+        self.eth_advanced.w3.unstable_blocks = 10       # Set default unstable blocks
+        self.eth_advanced.w3.latest_seen_block = 1000   # Set the latest seen block
+
+        # Mock get_block_number
+        self.eth_advanced.get_block_number = MagicMock(return_value=1000)
+
+        # Mock get_block
+        def mock_get_block(block_identifier, no_retry=False):
+            if isinstance(block_identifier, int):
+                block_number = block_identifier
+            elif block_identifier == 'latest':
+                block_number = 1000
+            else:
+                # Handle other block identifiers as needed
+                block_number = None
+            if block_number is None:
+                raise Exception(f"Invalid block identifier: {block_identifier}")
+            block_hash = f"hash_{block_number}"
+            parent_hash = f"hash_{block_number - 1}" if block_number > 0 else None
+            return {'number': block_number, 'hash': block_hash, 'parentHash': parent_hash}
+        self.eth_advanced.get_block = MagicMock(side_effect=mock_get_block)
+
+        # Mock _get_logs
+        self.logs_storage = {}  # To simulate storage of logs per block
+        def mock__get_logs(filter_params):
+            if 'blockHash' in filter_params:
+                # Single block query
+                block_number = int(filter_params['blockHash'].split('_')[1])
+                logs = self.logs_storage.get(block_number, [])
+                return logs
+            else:
+                from_block = filter_params.get('fromBlock', 0)
+                to_block = filter_params.get('toBlock', 0)
+                logs = []
+                for block_number in range(from_block, to_block + 1):
+                    block_logs = self.logs_storage.get(block_number, [])
+                    logs.extend(block_logs)
+                return logs
+        self.eth_advanced._get_logs = MagicMock(side_effect=mock__get_logs)
+
+    def test_get_logs_no_duplicates_no_missing_blocks_correct_order(self):
+        # Prepare test data
+        from_block = 900
+        to_block = 950
+        filter_params = {'fromBlock': from_block, 'toBlock': to_block}
+
+        # Simulate logs for each block
+        for block_number in range(from_block, to_block + 1):
+            self.logs_storage[block_number] = [{'blockNumber': block_number, 'logIndex': 0}]
+
+        # Call get_logs
+        logs = self.eth_advanced.get_logs(filter_params, use_subsquid=False)
+
+        # Collect block numbers from logs
+        actual_block_numbers = [log['blockNumber'] for log in logs]
+        expected_block_numbers = list(range(from_block, to_block + 1))
+
+        # Assertions
+        self.assertEqual(len(actual_block_numbers), len(set(actual_block_numbers)), "Duplicate logs found")
+        self.assertEqual(sorted(actual_block_numbers), expected_block_numbers, "Missing or extra logs found")
+        self.assertEqual(actual_block_numbers, expected_block_numbers, "Logs are not in correct order")
+
+    def test_get_logs_range_exceeds_filter_block_range(self):
+        # Adjust filter_block_range to force splitting
+        self.eth_advanced.w3.filter_block_range = 10
+
+        # Prepare test data
+        from_block = 50
+        to_block = 100
+        filter_params = {'fromBlock': from_block, 'toBlock': to_block}
+
+        # Simulate logs for each block
+        for block_number in range(from_block, to_block + 1):
+            self.logs_storage[block_number] = [{'blockNumber': block_number, 'logIndex': 0}]
+
+        # Call get_logs
+        logs = self.eth_advanced.get_logs(filter_params, use_subsquid=False)
+
+        # Collect block numbers from logs
+        actual_block_numbers = [log['blockNumber'] for log in logs]
+        expected_block_numbers = list(range(from_block, to_block + 1))
+
+        # Assertions
+        self.assertEqual(len(actual_block_numbers), len(set(actual_block_numbers)), "Duplicate logs found")
+        self.assertEqual(sorted(actual_block_numbers), expected_block_numbers, "Missing or extra logs found")
+        self.assertEqual(actual_block_numbers, expected_block_numbers, "Logs are not in correct order")
+
+    def test_get_logs_splits_on_error(self):
+        # Prepare test data
+        from_block = 50
+        to_block = 100
+        filter_params = {'fromBlock': from_block, 'toBlock': to_block}
+
+        for block_number in range(from_block, to_block + 1):
+            self.logs_storage[block_number] = [{'blockNumber': block_number, 'logIndex': 0}]
+
+        # Simulate logs for each block except when more than 10 logs are requested, then raise exception
+        def mock__get_logs(filter_params):
+            from_block = filter_params.get('fromBlock', 0)
+            to_block = filter_params.get('toBlock', 0)
+            if to_block - from_block + 1 > 10:
+                raise Exception("Simulated RPC error")
+            logs = []
+            for block_number in range(from_block, to_block + 1):
+                logs.extend(self.logs_storage.get(block_number, []))
+            return logs
+        self.eth_advanced._get_logs.side_effect = mock__get_logs
+
+        # Call get_logs
+        logs = self.eth_advanced.get_logs(filter_params, use_subsquid=False)
+
+        # Collect block numbers from logs
+        actual_block_numbers = [log['blockNumber'] for log in logs]
+        expected_block_numbers = list(range(from_block, to_block + 1))
+
+        # Assertions
+        self.assertEqual(len(actual_block_numbers), len(set(actual_block_numbers)), "Duplicate logs found")
+        self.assertEqual(sorted(actual_block_numbers), expected_block_numbers, "Missing or extra logs found")
+        self.assertEqual(actual_block_numbers, expected_block_numbers, "Logs are not in correct order")
+
+    @patch('IceCreamSwapWeb3.EthAdvanced.get_filter')
+    def test_get_logs_uses_subsquid(self, mock_get_filter):
+        # Prepare test data
+        from_block = 800
+        to_block = 850
+        filter_params = {'fromBlock': from_block, 'toBlock': to_block}
+
+        for block_number in range(to_block - 10, to_block + 1):
+            self.logs_storage[block_number] = [{'blockNumber': block_number, 'logIndex': 0}]
+
+        # Simulate logs returned by get_filter
+        def mock_get_filter_func(chain_id, filter_params, partial_allowed, p_bar):
+            till_block = to_block - 10
+            logs = []
+            for block_number in range(from_block, till_block + 1):
+                logs.append({'blockNumber': block_number, 'logIndex': 0})
+            return till_block, logs
+        mock_get_filter.side_effect = mock_get_filter_func
+
+        # Call get_logs with use_subsquid=True
+        logs = self.eth_advanced.get_logs(filter_params, use_subsquid=True)
+
+        # Collect block numbers from logs
+        actual_block_numbers = [log['blockNumber'] for log in logs]
+        expected_block_numbers = list(range(from_block, to_block + 1))
+
+        # Assertions
+        self.assertEqual(len(actual_block_numbers), len(set(actual_block_numbers)), "Duplicate logs found")
+        self.assertEqual(sorted(actual_block_numbers), expected_block_numbers, "Missing or extra logs found")
+        self.assertEqual(actual_block_numbers, expected_block_numbers, "Logs are not in correct order")
+
+    def test_get_logs_unstable_blocks_handling(self):
+        # Prepare test data where to_block is within the latest unstable blocks
+        unstable_blocks = self.eth_advanced.w3.unstable_blocks
+        latest_block = self.eth_advanced.get_block_number()
+        from_block = latest_block - unstable_blocks - 5
+        to_block = latest_block  # This will be within the unstable blocks
+
+        filter_params = {'fromBlock': from_block, 'toBlock': to_block}
+
+        # Simulate logs for each block
+        for block_number in range(from_block, to_block + 1):
+            self.logs_storage[block_number] = [{'blockNumber': block_number, 'logIndex': 0}]
+
+        # Ensure get_block returns consistent block hashes and parent hashes
+        def mock_get_block(block_identifier, no_retry=False):
+            if isinstance(block_identifier, int):
+                block_number = block_identifier
+            elif block_identifier == 'latest':
+                block_number = latest_block
+            else:
+                # Handle other block identifiers as needed
+                block_number = None
+            if block_number is None:
+                raise Exception(f"Invalid block identifier: {block_identifier}")
+            block_hash = f"hash_{block_number}"
+            parent_hash = f"hash_{block_number - 1}" if block_number > 0 else None
+            return {'number': block_number, 'hash': block_hash, 'parentHash': parent_hash}
+
+        self.eth_advanced.get_block.side_effect = mock_get_block
+
+        # Mock get_logs_inner to simulate fetching logs by blockHash
+        def mock_get_logs_inner(filter_params, no_retry=False):
+            block_hash = filter_params.get('blockHash')
+            if block_hash:
+                block_number = int(block_hash.split('_')[1])
+                return self.logs_storage.get(block_number, [])
+            else:
+                return []
+
+        self.eth_advanced.get_logs_inner = MagicMock(side_effect=mock_get_logs_inner)
+
+        # Call get_logs
+        logs = self.eth_advanced.get_logs(filter_params, use_subsquid=False)
+
+        # Collect block numbers from logs
+        actual_block_numbers = [log['blockNumber'] for log in logs]
+        expected_block_numbers = list(range(from_block, to_block + 1))
+
+        # Assertions
+        self.assertEqual(len(actual_block_numbers), len(set(actual_block_numbers)), "Duplicate logs found")
+        self.assertEqual(sorted(actual_block_numbers), expected_block_numbers, "Missing or extra logs found")
+        self.assertEqual(actual_block_numbers, expected_block_numbers, "Logs are not in correct order")
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/IceCreamSwapWeb3/Web3Advanced.py b/IceCreamSwapWeb3/Web3Advanced.py
@@ -36,7 +36,7 @@ def __init__(
             self,
             node_url: str,
             should_retry: bool = True,
-            unstable_blocks: int = int(os.getenv("UNSTABLE_BLOCKS", 3)),  # not all nodes might have latest n blocks, these are seen as unstable
+            unstable_blocks: int = int(os.getenv("UNSTABLE_BLOCKS", 5)),  # not all nodes might have latest n blocks, these are seen as unstable
     ):
         patch_error_formatters()
         self.node_url = node_url
diff --git a/setup.py b/setup.py
@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '0.1.17'
+VERSION = '0.1.18'
 DESCRIPTION = 'IceCreamSwap Web3.py wrapper'
 LONG_DESCRIPTION = 'IceCreamSwap Web3.py wrapper with automatic retries, multicall and other advanced functionality'
 
