Fix "URI Too Long" error on fetching originations (#65)

Author: droserasprout

src/dipdup/datasources/tzkt/datasource.py

@@ -19,10 +19,12 @@
 from dipdup.datasources.proxy import DatasourceRequestProxy
 from dipdup.datasources.tzkt.enums import TzktMessageType
 from dipdup.models import BigMapAction, BigMapData, OperationData
+from dipdup.utils import split_by_chunks
 
 OperationID = int
 
 TZKT_HTTP_REQUEST_LIMIT = 10000
+TZKT_ORIGINATIONS_REQUEST_LIMIT = 100
 OPERATION_FIELDS = (
     "type",
     "id",
@@ -356,28 +358,34 @@ async def get_latest_block(self) -> Dict[str, Any]:
         self._logger.debug(block)
         return block
 
-    async def get_originations(self, addresses: Set[str], offset: int, first_level, last_level) -> List[OperationData]:
-        raw_originations = await self._proxy.http_request(
-            'get',
-            url=f'{self._url}/v1/operations/originations',
-            params={
-                "originatedContract.in": ','.join(addresses),
-                "offset": offset,
-                "limit": self.request_limit,
-                "level.gt": first_level,
-                "level.le": last_level,
-                "select": ','.join(ORIGINATION_OPERATION_FIELDS),
-                "status": "applied",
-            },
-        )
+    async def get_originations(self, addresses: Set[str], offset: int, first_level: int, last_level: int) -> List[OperationData]:
+        raw_originations = []
+        # NOTE: TzKT may hit URL length limit with hundreds of originations in a single request.
+        # NOTE: Chunk of 100 addresses seems like a reasonable choice - URL of ~3971 characters.
+        # NOTE: Other operation requests won't hit that limit.
+        for addresses_chunk in split_by_chunks(list(addresses), TZKT_ORIGINATIONS_REQUEST_LIMIT):
+            raw_originations += await self._proxy.http_request(
+                'get',
+                url=f'{self._url}/v1/operations/originations',
+                params={
+                    "originatedContract.in": ','.join(addresses_chunk),
+                    "offset": offset,
+                    "limit": self.request_limit,
+                    "level.gt": first_level,
+                    "level.le": last_level,
+                    "select": ','.join(ORIGINATION_OPERATION_FIELDS),
+                    "status": "applied",
+                },
+            )
+
         originations = []
         for op in raw_originations:
             # NOTE: `type` field needs to be set manually when requesting operations by specific type
             op['type'] = 'origination'
             originations.append(self.convert_operation(op))
         return originations
 
-    async def get_transactions(self, field: str, addresses: Set[str], offset: int, first_level, last_level) -> List[OperationData]:
+    async def get_transactions(self, field: str, addresses: Set[str], offset: int, first_level: int, last_level: int) -> List[OperationData]:
         raw_transactions = await self._proxy.http_request(
             'get',
             url=f'{self._url}/v1/operations/transactions',

src/dipdup/utils.py

@@ -6,7 +6,7 @@
 import time
 from contextlib import asynccontextmanager
 from logging import Logger
-from typing import AsyncIterator, Optional
+from typing import Any, AsyncIterator, Iterator, List, Optional, Sequence
 
 import aiohttp
 from tortoise import Tortoise
@@ -40,6 +40,13 @@ def pascal_to_snake(name: str) -> str:
     return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower()
 
 
+def split_by_chunks(input_: List[Any], size: int) -> Iterator[List[Any]]:
+    i = 0
+    while i < len(input_):
+        yield input_[i : i + size]
+        i += size
+
+
 @asynccontextmanager
 async def tortoise_wrapper(url: str, models: Optional[str] = None) -> AsyncIterator:
     """Initialize Tortoise with internal and project models, close connections when done"""