Set config annotations in runtime to allow unresolved links (#549)

Author: droserasprout

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog], and this project adheres to [Semantic Versioning].
 
+## [Unreleased]
+
+### Fixed
+
+- metadata: Add `limanet` to supported networks.
+
 ## [6.3.1] - 2022-11-25
 
 ### Fixed

docs/config/reference.md

@@ -163,8 +163,8 @@ async def _fetch_operation_pattern_schema(
         operation_pattern_config: OperationHandlerPatternConfigU,
         datasource_config: TzktDatasourceConfig,
     ) -> None:
-        if _is_typed_transaction(operation_pattern_config):
-            contract_config = operation_pattern_config.destination_contract_config
+        if _is_typed_transaction(operation_pattern_config) and operation_pattern_config.destination:
+            contract_config = operation_pattern_config.destination
         elif _is_typed_origination(operation_pattern_config):
             contract_config = operation_pattern_config.contract_config
         else:

src/dipdup/codegen.py

@@ -591,8 +591,8 @@ class OperationHandlerTransactionPatternConfig(PatternConfig, StorageTypeMixin,
     """
 
     type: Literal['transaction'] = 'transaction'
-    source: str | ContractConfig | None = None
-    destination: str | ContractConfig | None = None
+    source: ContractConfig | None = None
+    destination: ContractConfig | None = None
     entrypoint: str | None = None
     optional: bool = False
     alias: str | None = None
@@ -605,17 +605,17 @@ def __post_init_post_parse__(self) -> None:
             raise ConfigurationError('Transactions with entrypoint must also have destination')
 
     def iter_imports(self, package: str) -> Iterator[tuple[str, str]]:
-        if self.entrypoint:
-            module_name = self.destination_contract_config.module_name
+        if self.entrypoint and self.destination:
+            module_name = self.destination.module_name
             yield 'dipdup.models', 'Transaction'
             yield self.format_parameter_import(package, module_name, self.entrypoint, self.alias)
             yield self.format_storage_import(package, module_name)
         else:
             yield self.format_untyped_operation_import()
 
     def iter_arguments(self) -> Iterator[tuple[str, str]]:
-        if self.entrypoint:
-            module_name = self.destination_contract_config.module_name
+        if self.entrypoint and self.destination:
+            module_name = self.destination.module_name
             yield self.format_operation_argument(
                 module_name,
                 self.entrypoint,
@@ -630,18 +630,6 @@ def iter_arguments(self) -> Iterator[tuple[str, str]]:
                 self.alias,
             )
 
-    @cached_property
-    def source_contract_config(self) -> ContractConfig:
-        if not isinstance(self.source, ContractConfig):
-            raise ConfigInitializationException
-        return self.source
-
-    @cached_property
-    def destination_contract_config(self) -> ContractConfig:
-        if not isinstance(self.destination, ContractConfig):
-            raise ConfigInitializationException
-        return self.destination
-
 
 @dataclass
 class OperationHandlerOriginationPatternConfig(PatternConfig, StorageTypeMixin, SubgroupIndexMixin):
@@ -657,9 +645,9 @@ class OperationHandlerOriginationPatternConfig(PatternConfig, StorageTypeMixin,
     """
 
     type: Literal['origination'] = 'origination'
-    source: str | ContractConfig | None = None
-    similar_to: str | ContractConfig | None = None
-    originated_contract: str | ContractConfig | None = None
+    source: ContractConfig | None = None
+    similar_to: ContractConfig | None = None
+    originated_contract: ContractConfig | None = None
     optional: bool = False
     strict: bool = False
     alias: str | None = None
@@ -678,18 +666,18 @@ def __hash__(self) -> int:
         return hash(
             ''.join(
                 [
-                    self.source_contract_config.address if self.source else '',
-                    self.similar_to_contract_config.address if self.similar_to else '',
-                    self.originated_contract_config.address if self.originated_contract else '',
+                    self.source.address if self.source else '',
+                    self.similar_to.address if self.similar_to else '',
+                    self.originated_contract.address if self.originated_contract else '',
                 ]
             )
         )
 
     def iter_imports(self, package: str) -> Iterator[tuple[str, str]]:
         if self.originated_contract:
-            module_name = self.originated_contract_config.module_name
+            module_name = self.originated_contract.module_name
         elif self.similar_to:
-            module_name = self.similar_to_contract_config.module_name
+            module_name = self.similar_to.module_name
         elif self.source:
             yield 'dipdup.models', 'OperationData'
             return
@@ -723,31 +711,13 @@ def module_name(self) -> str:
     @cached_property
     def contract_config(self) -> ContractConfig:
         if self.originated_contract:
-            return self.originated_contract_config
+            return self.originated_contract
         if self.similar_to:
-            return self.similar_to_contract_config
+            return self.similar_to
         if self.source:
-            return self.source_contract_config
+            return self.source
         raise RuntimeError
 
-    @cached_property
-    def source_contract_config(self) -> ContractConfig:
-        if not isinstance(self.source, ContractConfig):
-            raise ConfigInitializationException
-        return self.source
-
-    @cached_property
-    def similar_to_contract_config(self) -> ContractConfig:
-        if not isinstance(self.similar_to, ContractConfig):
-            raise ConfigInitializationException
-        return self.similar_to
-
-    @cached_property
-    def originated_contract_config(self) -> ContractConfig:
-        if not isinstance(self.originated_contract, ContractConfig):
-            raise ConfigInitializationException
-        return self.originated_contract
-
 
 @dataclass
 class CallbackMixin(CodegenMixin):
@@ -870,7 +840,7 @@ class IndexConfig(ABC, TemplateValuesMixin, NameMixin, SubscriptionsMixin, Paren
     """
 
     kind: str
-    datasource: str | TzktDatasourceConfig
+    datasource: TzktDatasourceConfig
 
     def __post_init_post_parse__(self) -> None:
         TemplateValuesMixin.__post_init_post_parse__(self)
@@ -920,8 +890,8 @@ class OperationIndexConfig(IndexConfig):
 
     kind: Literal['operation']
     handlers: tuple[OperationHandlerConfig, ...]
+    contracts: list[ContractConfig] = field(default_factory=list)
     types: tuple[OperationType, ...] = (OperationType.transaction,)
-    contracts: list[str | ContractConfig] = field(default_factory=list)
 
     first_level: int = 0
     last_level: int = 0
@@ -968,10 +938,10 @@ def import_objects(self, package: str) -> None:
 
             for pattern_config in handler_config.pattern:
                 if isinstance(pattern_config, OperationHandlerTransactionPatternConfig):
-                    if not pattern_config.entrypoint:
+                    if not (pattern_config.entrypoint and pattern_config.destination):
                         continue
 
-                    module_name = pattern_config.destination_contract_config.module_name
+                    module_name = pattern_config.destination.module_name
                     pattern_config.initialize_parameter_cls(package, module_name, pattern_config.entrypoint)
                     pattern_config.initialize_storage_cls(package, module_name)
 
@@ -994,7 +964,7 @@ class BigMapHandlerConfig(HandlerConfig, kind='handler'):
     :param path: Path to big map (alphanumeric string with dots)
     """
 
-    contract: str | ContractConfig
+    contract: ContractConfig
     path: str
 
     def __post_init_post_parse__(self) -> None:
@@ -1077,7 +1047,7 @@ class BigMapIndexConfig(IndexConfig):
     """
 
     kind: Literal['big_map']
-    datasource: str | TzktDatasourceConfig
+    datasource: TzktDatasourceConfig
     handlers: tuple[BigMapHandlerConfig, ...]
 
     skip_history: SkipHistory = SkipHistory.never
@@ -1119,7 +1089,7 @@ class HeadIndexConfig(IndexConfig):
     """Head block index config"""
 
     kind: Literal['head']
-    datasource: str | TzktDatasourceConfig
+    datasource: TzktDatasourceConfig
     handlers: tuple[HeadHandlerConfig, ...]
 
     @property
@@ -1137,10 +1107,10 @@ def import_objects(self, package: str) -> None:
 
 @dataclass
 class TokenTransferHandlerConfig(HandlerConfig, kind='handler'):
-    contract: str | ContractConfig | None = None
+    contract: ContractConfig | None = None
     token_id: int | None = None
-    from_: str | ContractConfig | None = Field(default=None, alias='from')
-    to: str | ContractConfig | None = None
+    from_: ContractConfig | None = Field(default=None, alias='from')
+    to: ContractConfig | None = None
 
     def iter_imports(self, package: str) -> Iterator[tuple[str, str]]:
         yield 'dipdup.context', 'HandlerContext'
@@ -1157,7 +1127,7 @@ class TokenTransferIndexConfig(IndexConfig):
     """Token index config"""
 
     kind: Literal['token_transfer']
-    datasource: str | TzktDatasourceConfig
+    datasource: TzktDatasourceConfig
     handlers: tuple[TokenTransferHandlerConfig, ...] = field(default_factory=tuple)
 
     first_level: int = 0
@@ -1170,7 +1140,7 @@ def import_objects(self, package: str) -> None:
 
 @dataclass
 class EventHandlerConfig(HandlerConfig, kind='handler'):
-    contract: str | ContractConfig
+    contract: ContractConfig
     tag: str
 
     def __post_init_post_parse__(self) -> None:
@@ -1216,7 +1186,7 @@ def iter_arguments(self) -> Iterator[tuple[str, str]]:
 
 @dataclass
 class UnknownEventHandlerConfig(HandlerConfig, kind='handler'):
-    contract: str | ContractConfig
+    contract: ContractConfig
 
     @cached_property
     def contract_config(self) -> ContractConfig:
@@ -1240,7 +1210,7 @@ def iter_arguments(self) -> Iterator[tuple[str, str]]:
 @dataclass
 class EventIndexConfig(IndexConfig):
     kind: Literal['event']
-    datasource: str | TzktDatasourceConfig
+    datasource: TzktDatasourceConfig
     handlers: tuple[EventHandlerConfigU, ...] = field(default_factory=tuple)
 
     first_level: int = 0
@@ -1312,7 +1282,7 @@ class JobConfig(NameMixin):
     :param args: Arguments to pass to the hook
     """
 
-    hook: str | HookConfig
+    hook: HookConfig
     crontab: str | None = None
     interval: int | None = None
     daemon: bool = False
@@ -1818,7 +1788,10 @@ def _resolve_index_subscriptions(self, index_config: IndexConfigU) -> None:
             )
 
     def _resolve_index_links(self, index_config: ResolvedIndexConfigU) -> None:
-        """Resolve contract and datasource configs by aliases"""
+        """Resolve contract and datasource configs by aliases.
+
+        WARNING: str type checks are intentional! See `dipdup.config.patch_annotations`.
+        """
         # NOTE: Each index must have a corresponding (currently) TzKT datasource
         if isinstance(index_config.datasource, str):
             index_config.datasource = self.get_tzkt_datasource(index_config.datasource)
@@ -1896,3 +1869,41 @@ def _set_names(self) -> None:
         for named_configs in named_config_sections:
             for name, config in named_configs.items():
                 config.name = name
+
+
+yaml_annotations = {
+    'TzktDatasourceConfig': 'str | TzktDatasourceConfig',
+    'ContractConfig': 'str | ContractConfig',
+    'ContractConfig | None': 'str | ContractConfig | None',
+    'list[ContractConfig]': 'list[str | ContractConfig]',
+}
+orinal_annotations = {v: k for k, v in yaml_annotations.items()}
+
+
+def patch_annotations(replace_table: dict[str, str]) -> None:
+    """Patch dataclass annotations in runtime to allow using aliases in config files.
+
+    DipDup config allows to use string aliases for contracts and datasources. During `DipDupConfig.load`
+    these aliases are resolved to actual configs and never become strings again. This hack allows to add
+    `str` in Unions before loading config so we don't need to write isinstance checks everywhere.
+    """
+    self = importlib.import_module(__name__)
+
+    for attr in dir(self):
+        value = getattr(self, attr)
+        if not isinstance(value, type) or not hasattr(value, '__annotations__'):
+            continue
+
+        # NOTE: All annotations are strings now
+        reload = False
+        for name, annotation in value.__annotations__.items():
+            if new_annotation := replace_table.get(annotation):
+                value.__annotations__[name] = new_annotation
+                reload = True
+
+        # NOTE: Wrap dataclass again to recreate magic methods
+        if reload:
+            setattr(self, attr, dataclass(value))
+
+
+patch_annotations(yaml_annotations)

src/dipdup/config.py

@@ -39,7 +39,6 @@
 from dipdup.config import PostgresDatabaseConfig
 from dipdup.config import ResolvedIndexConfigU
 from dipdup.config import TokenTransferIndexConfig
-from dipdup.config import TzktDatasourceConfig
 from dipdup.datasources.coinbase.datasource import CoinbaseDatasource
 from dipdup.datasources.datasource import Datasource
 from dipdup.datasources.datasource import HttpDatasource
@@ -279,7 +278,7 @@ async def _spawn_index(self, name: str, state: Optional[Index] = None) -> None:
         index_config = cast(ResolvedIndexConfigU, self.config.get_index(name))
         index: OperationIndex | BigMapIndex | HeadIndex | TokenTransferIndex | EventIndex
 
-        datasource_name = cast(TzktDatasourceConfig, index_config.datasource).name
+        datasource_name = index_config.datasource.name
         datasource = self.get_tzkt_datasource(datasource_name)
 
         if isinstance(index_config, OperationIndexConfig):

src/dipdup/context.py

@@ -7,3 +7,4 @@ class MetadataNetwork(Enum):
     jakartanet = 'jakartanet'
     kathmandunet = 'kathmandunet'
     mainnet = 'mainnet'
+    limanet = 'limanet'