Fix crash while applying bigmap diffs with complex keys (#194)

* Apply diffs by id, initial and dirty af

* Tests, docs

* Typo

* Correct logic, but with wraps

* Replace unwraps with copypaste

* Lint

* remove some copypaste

* Docs, optimizations

* Refactoring

* Ignore missing fields

* Refactoring, disable edo tests

* CI

* Failing tests

* Another failing test

* Fix naming

* asdf-qwer

* Tests

* Simplify checks, fix hjkl test, structure test data

* Preprocess storage schema

* Bump just in case

* Remove --use-default to fix bigmap type

* Cleanup, comments

* Process unions in lists

* fix list types detector, lint

* Lint, docs

* lint

* Fix bazaar storage, refactoring

* new test

* fix test

* Lint

* failing test

* Fix last test

* Refactoring, docs

* Update changelog

* Cleanup

* Disable integration tests

Author: droserasprout

CHANGELOG.md

@@ -6,8 +6,15 @@ Please use [this](https://docs.gitlab.com/ee/development/changelog.html) documen
 
 ## Fixed
 
+* codegen: Fixed generating storage typeclasses with `Union` fields.
+* codegen: Fixed preprocessing contract JSONSchema.
 * index: Fixed processing reindexing reason saved in the database.
 * tzkt: Fixed processing operations with default entrypoint and empty parameter.
+* tzkt: Fixed crash while recursively applying bigmap diffs to the storage.
+
+## Performance
+
+* tzkt: Increased speed of applying bigmap diffs to operation storage.
 
 ## 4.0.0 - 2021-12-24
 

poetry.lock

@@ -40,6 +40,7 @@ sentry-sdk = "^1.1.0"
 sqlparse = "^0.4.1"
 tabulate = "^0.8.9"
 tortoise-orm = "0.17.8"
+typing-inspect = "^0.6.0"
 
 [tool.poetry.dev-dependencies]
 black = "^20.8b1"

pyproject.toml

@@ -0,0 +1,8 @@
+#$/bin/bash
+cd tests/test_dipdup
+for name in "asdf" "qwer" "hjkl" "zxcv" "rewq"
+do
+    dipdup -c $name.yml init
+    mv $name/types/$name/storage.py types/$name/storage.py
+    rm -r $name
+done

scripts/init_tests.sh

@@ -61,13 +61,25 @@
 _templates: Dict[str, Template] = {}
 
 
-def resolve_big_maps(schema: Dict[str, Any]) -> Dict[str, Any]:
+def preprocess_storage_jsonschema(schema: Dict[str, Any]) -> Dict[str, Any]:
     """Preprocess bigmaps in JSONSchema. Those are unions as could be pointers.
     We resolve bigmaps from diffs so no need to include int in type signature."""
+    if not isinstance(schema, dict):
+        return schema
     if 'properties' in schema:
         return {
             **schema,
-            'properties': {prop: resolve_big_maps(sub_schema) for prop, sub_schema in schema['properties'].items()},
+            'properties': {prop: preprocess_storage_jsonschema(sub_schema) for prop, sub_schema in schema['properties'].items()},
+        }
+    elif 'items' in schema:
+        return {
+            **schema,
+            'items': preprocess_storage_jsonschema(schema['items']),
+        }
+    elif 'additionalProperties' in schema:
+        return {
+            **schema,
+            'additionalProperties': preprocess_storage_jsonschema(schema['additionalProperties']),
         }
     elif schema.get('$comment') == 'big_map':
         return schema['oneOf'][1]
@@ -165,8 +177,8 @@ async def fetch_schemas(self) -> None:
                         mkdir_p(contract_schemas_path)
 
                         storage_schema_path = join(contract_schemas_path, 'storage.json')
+                        storage_schema = preprocess_storage_jsonschema(contract_schemas['storageSchema'])
 
-                        storage_schema = resolve_big_maps(contract_schemas['storageSchema'])
                         write(storage_schema_path, json.dumps(storage_schema, indent=4, sort_keys=True))
 
                         if not isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig):
@@ -279,7 +291,6 @@ async def generate_types(self, overwrite_types: bool = False) -> None:
                     '--class-name',
                     name.lstrip('_'),
                     '--disable-timestamp',
-                    '--use-default',
                 ]
                 self._logger.debug(' '.join(args))
                 subprocess.run(args, check=True)