Update staging API format, fix parsing aliased Pydantic models (#25)

Author: droserasprout

src/dipdup/datasources/tzkt/datasource.py

@@ -45,7 +45,7 @@
     "hasInternals",
     # "quote",
     "parameters",
-    "bigmaps,",
+    "diffs,",
 )
 
 
@@ -354,7 +354,7 @@ def convert_operation(cls, operation_json: Dict[str, Any]) -> OperationData:
             initiator_address=operation_json['initiator']['address'] if operation_json.get('initiator') else None,
             parameter=operation_json.get('parameters'),
             storage=storage,
-            bigmaps=operation_json.get('bigmaps'),
+            diffs=operation_json.get('diffs'),
         )
 
     async def get_latest_block(self) -> Dict[str, Any]:

src/dipdup/models.py

@@ -62,20 +62,20 @@ class OperationData:
     initiator_address: Optional[str] = None
     parameter: Optional[str] = None
     storage: Optional[Dict[str, Any]] = None
-    bigmaps: Optional[List[Dict[str, Any]]] = None
+    diffs: Optional[List[Dict[str, Any]]] = None
 
     def _merge_bigmapdiffs(self, storage_dict: Dict[str, Any], bigmap_name: str, array: bool) -> None:
-        if self.bigmaps is None:
+        if self.diffs is None:
             raise Exception('`bigmaps` field missing')
-        bigmapdiffs = [bm for bm in self.bigmaps if bm['path'] == bigmap_name]
+        bigmapdiffs = [bm for bm in self.diffs if bm['path'] == bigmap_name]
         for diff in bigmapdiffs:
             _logger.debug('Applying bigmapdiff: %s', diff)
             if diff['action'] in ('add_key', 'update_key'):
-                key = diff['key']['key']
+                key = diff['content']['key']
                 if array is True:
-                    storage_dict[bigmap_name].append({'key': key, 'value': diff['key']['value']})
+                    storage_dict[bigmap_name].append({'key': key, 'value': diff['content']['value']})
                 else:
-                    storage_dict[bigmap_name][key] = diff['key']['value']
+                    storage_dict[bigmap_name][key] = diff['content']['value']
 
     def get_merged_storage(self, storage_type: Type[StorageType]) -> StorageType:
         if self.storage is None:
@@ -85,16 +85,20 @@ def get_merged_storage(self, storage_type: Type[StorageType]) -> StorageType:
         _logger.debug('Merging storage')
         _logger.debug('Before: %s', storage)
         for key, field in storage_type.__fields__.items():
+
+            if field.alias:
+                key = field.alias
+
             # NOTE: TzKT could return bigmaps as object or as array of key-value objects. We need to guess this from storage.
             # TODO: This code should be a part of datasource module.
             if field.type_ not in (int, bool) and isinstance(storage[key], int):
                 if hasattr(field.type_, '__fields__') and 'key' in field.type_.__fields__ and 'value' in field.type_.__fields__:
                     storage[key] = []
-                    if self.bigmaps:
+                    if self.diffs:
                         self._merge_bigmapdiffs(storage, key, array=True)
                 else:
                     storage[key] = {}
-                    if self.bigmaps:
+                    if self.diffs:
                         self._merge_bigmapdiffs(storage, key, array=False)
 
         _logger.debug('After: %s', storage)

src/dipdup/utils.py

@@ -1,4 +1,5 @@
 import asyncio
+import logging
 import os
 import re
 import sys
@@ -12,6 +13,8 @@
 
 from dipdup import __version__
 
+_logger = logging.getLogger(__name__)
+
 
 def snake_to_camel(value: str) -> str:
     return ''.join(map(lambda x: x[0].upper() + x[1:], value.split('_')))
@@ -55,6 +58,8 @@ async def http_request(method: str, **kwargs):
             headers=headers,
             **kwargs,
         ) as response:
+            request_string = kwargs['url'] + '?' + '&'.join([f'{key}={value}' for key, value in kwargs.get('params', {}).items()])
+            _logger.debug('Calling `%s`', request_string)
             yield response
 
 

tests/test_dipdup/test_datasources/test_tzkt/operations-storage.json

@@ -79,12 +79,12 @@
             "errors": null,
             "hasInternals": false,
             "parameters": "{\"entrypoint\":\"propose\",\"value\":{\"prim\":\"Pair\",\"args\":[{\"int\":\"10\"},{\"prim\":\"Left\",\"args\":[{\"prim\":\"Pair\",\"args\":[{\"int\":\"0\"},[{\"prim\":\"Pair\",\"args\":[{\"bytes\":\"\"},{\"prim\":\"Some\",\"args\":[{\"bytes\":\"6672\"}]}]}]]}]}]}}",
-            "bigmaps": [
+            "diffs": [
                 {
                     "bigmap": 36371,
                     "path": "proposals",
                     "action": "add_key",
-                    "key": {
+                    "content": {
                         "hash": "exprtrENeWqsFnt4E4mFvnonvhrhU3jKTuaFXiufCK45XwdgnP7D6p",
                         "key": "e710c1a066bbbf73692168e783607996785260cec4d60930579827298493b8b9",
                         "value": {
@@ -112,7 +112,7 @@
                     "bigmap": 36368,
                     "path": "ledger",
                     "action": "update_key",
-                    "key": {
+                    "content": {
                         "hash": "expruVX2dr7XzPeZwyNsKwVcf88r2FfUrTNHewFZWkDtfPyoHkzrdf",
                         "key": {
                             "nat": "0",
@@ -125,7 +125,7 @@
                     "bigmap": 36368,
                     "path": "ledger",
                     "action": "update_key",
-                    "key": {
+                    "content": {
                         "hash": "exprtjvuiPChDAAipwbGtmwZDA8JcEQ6y4pu2qTy93fpPJFtDkZFap",
                         "key": {
                             "nat": "1",