Fix merging bigmapdiffs (#13)

Author: droserasprout

aiosignalrcore-0.0.0-py3-none-any.whl

@@ -28,9 +28,8 @@ asyncpg = "^0.22.0"
 datamodel-code-generator = "^0.10.0"
 "ruamel.yaml" = "^0.17.2"
 tortoise-orm = "^0.17.1"
-websockets = "^8.1"
 pydantic = "^1.8.1"
-aiosignalrcore = {path = "aiosignalrcore-0.0.0-py3-none-any.whl"}
+aiosignalrcore = "^0.9.2"
 
 [tool.poetry.dev-dependencies]
 black = "^20.8b1"

poetry.lock

@@ -0,0 +1,22 @@
+  version: 1
+  disable_existing_loggers: false
+  formatters:
+    brief:
+      format: "%(levelname)-8s %(name)-35s %(message)s"
+  handlers:
+    console:
+      level: DEBUG
+      formatter: brief
+      class: logging.StreamHandler
+      stream : ext://sys.stdout
+  loggers:
+    SignalRCoreClient:
+      formatter: brief
+    dipdup.datasources.tzkt.datasource:
+      level: DEBUG
+    dipdup.datasources.tzkt.cache:
+      level: DEBUG
+  root:
+    level: DEBUG
+    handlers:
+      - console

pyproject.toml

@@ -1,9 +1,8 @@
 import logging
-from contextlib import suppress
 from copy import deepcopy
 from datetime import datetime
 from enum import Enum
-from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, get_args, get_origin
+from typing import Any, Dict, Generic, List, Optional, Type, TypeVar
 
 from pydantic import BaseModel
 from pydantic.dataclasses import dataclass
@@ -35,7 +34,7 @@ class Meta:
 
 @dataclass
 class OperationData:
-    # FIXME:
+    # FIXME: Bug in TzKT, shouldn't be optional
     type: Optional[str]
     id: int
     level: int
@@ -67,7 +66,7 @@ class OperationData:
 
     def _merge_bigmapdiffs(self, storage_dict: Dict[str, Any], bigmap_name: str) -> None:
         if self.bigmaps is None:
-            raise Exception('`bigaps` field missing')
+            raise Exception('`bigmaps` field missing')
         bigmapdiffs = [bm for bm in self.bigmaps if bm['path'] == bigmap_name]
         for diff in bigmapdiffs:
             if diff['action'] in ('add_key', 'update_key'):
@@ -80,20 +79,23 @@ def _merge_bigmapdiffs(self, storage_dict: Dict[str, Any], bigmap_name: str) ->
     def get_merged_storage(self, storage_type: Type[StorageType]) -> StorageType:
         if self.storage is None:
             raise Exception('`storage` field missing')
-        if self.bigmaps is None:
-            return storage_type.parse_obj(self.storage)
 
         storage = deepcopy(self.storage)
+        _logger.debug('Merging storage')
+        _logger.debug('Before: %s', storage)
         for key, field in storage_type.__fields__.items():
-            if field.type_ != int and isinstance(storage[key], int):
-                with suppress(AttributeError):
-                    if 'key' in field.type_.__fields__:
-                        storage[key] = []
-                    else:
-                        storage[key] = {}
-
-                    self._merge_bigmapdiffs(storage, key)
-
+            # NOTE: TzKT could return bigmaps as object or as array of key-value objects. We need to guess this from storage.
+            # TODO: This code should be a part of datasource module.
+            if field.type_ not in (int, bool) and isinstance(storage[key], int):
+                if hasattr(field.type_, '__fields__') and 'key' in field.type_.__fields__:
+                    storage[key] = []
+                else:
+                    storage[key] = {}
+
+            if self.bigmaps is not None:
+                self._merge_bigmapdiffs(storage, key)
+
+        _logger.debug('After: %s', storage)
         return storage_type.parse_obj(storage)
 
 

src/dipdup/configs/debug.yml

@@ -323,6 +323,7 @@ async def test_on_operation_match(self):
 
             callback_mock = AsyncMock()
             storage_type_mock = MagicMock()
+            storage_type_mock.__fields__ = MagicMock()
 
             self.index_config.handlers[0].callback_fn = callback_mock
             self.index_config.handlers[0].pattern[0].storage_type_cls = storage_type_mock