Fix TzktDatasource getters, add missing typehints (#274)

Author: droserasprout

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [unreleased]
+
+### Fixed
+
+* tzkt: Fixed `get_originated_contracts` and `get_similar_contracts` methods response.
+
 ## 5.0.0-rc3 - 2022-03-28
 
 ### Added

poetry.lock

@@ -18,5 +18,5 @@
             line = line.strip()
             if line.startswith(':param'):
                 parts = line.split(':')
-                param, desc = parts[1], parts[2].strip()
+                param, desc = parts[1], ':'.join(parts[2:]).strip()
                 print(f'| `{param[6:]}` | {desc} |')

scripts/extract_reference.py

@@ -119,7 +119,7 @@ def connection_string(self) -> str:
         return connection_string
 
     @validator('immune_tables')
-    def _valid_immune_tables(cls, v):
+    def _valid_immune_tables(cls, v) -> None:
         for table in v:
             if table.startswith('dipdup'):
                 raise ConfigurationError('Tables with `dipdup` prefix can\'t be immune')
@@ -184,15 +184,15 @@ class ContractConfig(NameMixin):
     address: str
     typename: Optional[str] = None
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(f'{self.address}{self.typename or ""}')
 
     @cached_property
     def module_name(self) -> str:
         return self.typename or self.name
 
     @validator('address', allow_reuse=True)
-    def _valid_address(cls, v):
+    def _valid_address(cls, v: str) -> str:
         # NOTE: Environment substitution was disabled during export, skip validation
         if '$' in v:
             return v
@@ -217,7 +217,7 @@ class TzktDatasourceConfig(NameMixin):
     url: str
     http: Optional[HTTPConfig] = None
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self.kind + self.url)
 
     def __post_init_post_parse__(self) -> None:
@@ -249,7 +249,7 @@ class CoinbaseDatasourceConfig(NameMixin):
     passphrase: Optional[str] = None
     http: Optional[HTTPConfig] = None
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self.kind)
 
 
@@ -268,7 +268,7 @@ class MetadataDatasourceConfig(NameMixin):
     url: str = DEFAULT_METADATA_URL
     http: Optional[HTTPConfig] = None
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self.kind + self.url + self.network.value)
 
 
@@ -285,7 +285,7 @@ class IpfsDatasourceConfig(NameMixin):
     url: str = DEFAULT_IPFS_URL
     http: Optional[HTTPConfig] = None
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self.kind + self.url)
 
 
@@ -302,7 +302,7 @@ class HttpDatasourceConfig(NameMixin):
     url: str
     http: Optional[HTTPConfig] = None
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self.kind + self.url)
 
 
@@ -344,7 +344,7 @@ def format_arguments(self) -> Iterator[str]:
 
     def locate_arguments(self) -> Dict[str, Optional[Type]]:
         """Try to resolve scope annotations for arguments"""
-        kwargs: Dict[str, Optional[Type]] = {}
+        kwargs: Dict[str, Optional[Type[Any]]] = {}
         for name, cls in self.iter_arguments():
             cls = cls.split(' as ')[0]
             kwargs[name] = cast(Optional[Type], locate(cls))
@@ -394,11 +394,11 @@ def format_untyped_operation_argument(cls, transaction_idx: int, optional: bool)
 class StorageTypeMixin:
     """`storage_type_cls` field"""
 
-    def __post_init_post_parse__(self):
-        self._storage_type_cls = None
+    def __post_init_post_parse__(self) -> None:
+        self._storage_type_cls: Optional[Type[Any]] = None
 
     @cached_property
-    def storage_type_cls(self) -> Type:
+    def storage_type_cls(self) -> Type[Any]:
         if self._storage_type_cls is None:
             raise ConfigInitializationException
         return self._storage_type_cls
@@ -429,7 +429,7 @@ def parent(self) -> Optional[T]:
 class ParameterTypeMixin:
     """`parameter_type_cls` field"""
 
-    def __post_init_post_parse__(self):
+    def __post_init_post_parse__(self) -> None:
         self._parameter_type_cls = None
 
     @cached_property
@@ -809,8 +809,8 @@ class BigMapHandlerConfig(HandlerConfig, kind='handler'):
 
     def __post_init_post_parse__(self):
         super().__post_init_post_parse__()
-        self._key_type_cls = None
-        self._value_type_cls = None
+        self._key_type_cls: Optional[Type[Any]] = None
+        self._value_type_cls: Optional[Type[Any]] = None
 
     @classmethod
     def format_key_import(cls, package: str, module_name: str, path: str) -> Tuple[str, str]:

src/dipdup/config.py

@@ -96,6 +96,14 @@ def token(cls) -> int:
 
 # TODO: Dataclasses are cool, everyone loves them. Resolve issue with pydantic serialization.
 class DipDupContext:
+    """Class to store application context
+
+    :param datasources: Mapping of available datasources
+    :param config: DipDup configuration
+    :param callbacks: Low-level callback interface (intented for internal use)
+    :param logger: Context-aware logger instance
+    """
+
     def __init__(
         self,
         datasources: Dict[str, Datasource],
@@ -118,6 +126,12 @@ async def fire_hook(
         *args,
         **kwargs: Any,
     ) -> None:
+        """Fire hook with given name and arguments.
+
+        :param name: Hook name
+        :param fmt: Format string for `ctx.logger` messages
+        :param wait: Wait for hook to finish or fire and forget
+        """
         await self.callbacks.fire_hook(self, name, fmt, wait, *args, **kwargs)
 
     async def fire_handler(
@@ -129,20 +143,28 @@ async def fire_handler(
         *args,
         **kwargs: Any,
     ) -> None:
+        """Fire handler with given name and arguments.
+
+        :param name: Handler name
+        :param index: Index name
+        :param datasource: An instance of datasource that triggered the handler
+        :param fmt: Format string for `ctx.logger` messages
+        """
         await self.callbacks.fire_handler(self, name, index, datasource, fmt, *args, **kwargs)
 
     async def execute_sql(self, name: str) -> None:
+        """Execute SQL script with given name
+
+        :param name: SQL script name within `<project>/sql` directory
+        """
         await self.callbacks.execute_sql(self, name)
 
     async def restart(self) -> None:
-        """Restart preserving CLI arguments"""
-        # NOTE: Remove --reindex from arguments to avoid reindexing loop
-        if '--reindex' in sys.argv:
-            sys.argv.remove('--reindex')
+        """Restart indexer preserving CLI arguments"""
         os.execl(sys.executable, sys.executable, *sys.argv)
 
     async def reindex(self, reason: Optional[Union[str, ReindexingReason]] = None, **context) -> None:
-        """Drop all tables or whole database and restart with the same CLI arguments"""
+        """Drop the whole database and restart with the same CLI arguments"""
         if not reason:
             reason = ReindexingReason.manual
         elif isinstance(reason, str):

src/dipdup/context.py

@@ -1,10 +1,10 @@
 import logging
 from abc import ABC
 from typing import Dict
+from typing import Optional
 from typing import Set
 
 from pydantic.dataclasses import dataclass
-from tortoise import Optional
 
 _logger = logging.getLogger('dipdup.datasource')
 

src/dipdup/datasources/subscription.py

@@ -305,12 +305,12 @@ async def get_similar_contracts(
             'get',
             url=f'v1/contracts/{address}/{entrypoint}',
             params={
-                'select': 'address',
+                'select': 'id,address',
                 'offset': offset,
                 'limit': limit,
             },
         )
-        return tuple(response)
+        return tuple(item['address'] for item in response)
 
     async def iter_similar_contracts(
         self,
@@ -333,12 +333,12 @@ async def get_originated_contracts(
             'get',
             url=f'v1/accounts/{address}/contracts',
             params={
-                'select': 'address',
+                'select': 'id,address',
                 'offset': offset,
                 'limit': limit,
             },
         )
-        return tuple(response)
+        return tuple(item['address'] for item in response)
 
     async def iter_originated_contracts(self, address: str) -> AsyncIterator[Tuple[str, ...]]:
         async for batch in self._iter_batches(self.get_originated_contracts, address, cursor=False):

src/dipdup/datasources/tzkt/datasource.py

@@ -28,7 +28,7 @@ def indent(text: str, indent: int = 2) -> str:
 class DipDupException(Exception):
     message: str
 
-    def __init__(self, *args) -> None:
+    def __init__(self, *args: Any) -> None:
         super().__init__(self.message, *args)
 
 
@@ -250,7 +250,7 @@ def _help(self) -> str:
 class InvalidDataError(DipDupError):
     """Failed to validate datasource message against generated type class"""
 
-    type_cls: Type
+    type_cls: Type[Any]
     data: Any
     parsed_object: Any
 
@@ -291,8 +291,8 @@ class CallbackTypeError(DipDupError):
     name: str
 
     arg: str
-    type_: Type
-    expected_type: Type
+    type_: Type[Any]
+    expected_type: Type[Any]
 
     def _help(self) -> str:
         return f"""

src/dipdup/exceptions.py

@@ -16,7 +16,7 @@
 
 from dipdup.enums import IndexStatus
 from dipdup.enums import IndexType
-from dipdup.exceptions import ReindexingReason
+from dipdup.enums import ReindexingReason
 
 ParameterType = TypeVar('ParameterType', bound=BaseModel)
 StorageType = TypeVar('StorageType', bound=BaseModel)

src/dipdup/models.py

@@ -1,5 +1,6 @@
 import time
 from contextlib import contextmanager
+from typing import Generator
 
 from prometheus_client import Counter  # type: ignore
 from prometheus_client import Gauge
@@ -74,31 +75,31 @@ def __new__(cls):
 
     @classmethod
     @contextmanager
-    def measure_level_sync_duration(cls):
+    def measure_level_sync_duration(cls) -> Generator[None, None, None]:
         with _index_level_sync_duration.time():
             yield
 
     @classmethod
     @contextmanager
-    def measure_level_realtime_duration(cls):
+    def measure_level_realtime_duration(cls) -> Generator[None, None, None]:
         with _index_level_realtime_duration.time():
             yield
 
     @classmethod
     @contextmanager
-    def measure_total_sync_duration(cls):
+    def measure_total_sync_duration(cls) -> Generator[None, None, None]:
         with _index_total_sync_duration.time():
             yield
 
     @classmethod
     @contextmanager
-    def measure_total_realtime_duration(cls):
+    def measure_total_realtime_duration(cls) -> Generator[None, None, None]:
         with _index_total_realtime_duration.time():
             yield
 
     @classmethod
     @contextmanager
-    def measure_callback_duration(cls, name: str):
+    def measure_callback_duration(cls, name: str) -> Generator[None, None, None]:
         with _callback_duration.labels(callback=name).time():
             yield
 
@@ -109,11 +110,11 @@ def set_indexes_count(cls, active: int, synced: int, realtime: int) -> None:
         _indexes_total.labels(status='realtime').set(realtime)
 
     @classmethod
-    def set_datasource_head_updated(cls, name: str):
+    def set_datasource_head_updated(cls, name: str) -> None:
         _datasource_head_updated.labels(datasource=name).observe(time.time())
 
     @classmethod
-    def set_datasource_rollback(cls, name: str):
+    def set_datasource_rollback(cls, name: str) -> None:
         _datasource_rollbacks.labels(datasource=name).inc()
 
     @classmethod
@@ -125,9 +126,9 @@ def set_index_handlers_matched(cls, amount: float) -> None:
         _index_handlers_matched.inc(amount)
 
     @classmethod
-    def set_levels_to_sync(cls, index: str, levels: int):
+    def set_levels_to_sync(cls, index: str, levels: int) -> None:
         _index_levels_to_sync.labels(index=index).observe(levels)
 
     @classmethod
-    def set_levels_to_realtime(cls, index: str, levels: int):
+    def set_levels_to_realtime(cls, index: str, levels: int) -> None:
         _index_levels_to_realtime.labels(index=index).observe(levels)