Fix multiple codegen regressions (#776)

Author: droserasprout

CHANGELOG.md

@@ -10,6 +10,8 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic
 
 - ci: Fixed dipdup package metadata.
 - cli: Generate base template from replay only when --base flag is set.
+- cli: Remove cached jsonschemas when calling init --force.
+- codegen: Filter jsonschemas by prefixes supported by code generator.
 - index: Remove Python limitation on large int<->str conversions.
 - index: Fixed crash when parsing typed transactions with empty parameter.
 - package: Don't create empty pyproject.toml during init.

pdm.lock

@@ -18,13 +18,13 @@ babel==2.12.1
 bitarray==2.8.0
 black==23.7.0
 certifi==2023.7.22
-chardet==5.1.0
+chardet==5.2.0
 charset-normalizer==3.2.0
 click==8.1.6
 coverage==7.2.7
 cytoolz==0.12.2
-datamodel-code-generator==0.21.2
-dnspython==2.4.0
+datamodel-code-generator==0.21.3
+dnspython==2.4.1
 docker==6.1.3
 docutils==0.20.1
 email-validator==2.0.0.post2
@@ -39,9 +39,7 @@ eth-utils==2.2.0
 execnet==2.0.2
 frozenlist==1.4.0
 genson==1.2.2
-h11==0.14.0
 hexbytes==0.3.1
-httpcore==0.17.3
 idna==3.4
 imagesize==1.4.1
 inflect==5.6.2
@@ -58,15 +56,15 @@ msgpack==1.0.5
 multidict==6.0.4
 mypy==1.4.1
 mypy-extensions==1.0.0
-numpy==1.25.1
+numpy==1.25.2
 openapi-schema-validator==0.4.4
 openapi-spec-validator==0.5.2
 orjson==3.9.2
 packaging==23.1
 parsimonious==0.9.0
 pathable==0.4.3
-pathspec==0.11.1
-platformdirs==3.9.1
+pathspec==0.11.2
+platformdirs==3.10.0
 pluggy==1.2.0
 pprofile==2.1.0
 prance==23.6.21.0

requirements.dev.txt

@@ -16,12 +16,12 @@ attrs==23.1.0
 bitarray==2.8.0
 black==23.7.0
 certifi==2023.7.22
-chardet==5.1.0
+chardet==5.2.0
 charset-normalizer==3.2.0
 click==8.1.6
 cytoolz==0.12.2
-datamodel-code-generator==0.21.2
-dnspython==2.4.0
+datamodel-code-generator==0.21.3
+dnspython==2.4.1
 email-validator==2.0.0.post2
 eth-abi==4.1.0
 eth-account==0.9.0
@@ -33,9 +33,7 @@ eth-typing==3.4.0
 eth-utils==2.2.0
 frozenlist==1.4.0
 genson==1.2.2
-h11==0.14.0
 hexbytes==0.3.1
-httpcore==0.17.3
 idna==3.4
 inflect==5.6.2
 iso8601==1.1.0
@@ -49,15 +47,15 @@ MarkupSafe==2.1.3
 msgpack==1.0.5
 multidict==6.0.4
 mypy-extensions==1.0.0
-numpy==1.25.1
+numpy==1.25.2
 openapi-schema-validator==0.4.4
 openapi-spec-validator==0.5.2
 orjson==3.9.2
 packaging==23.1
 parsimonious==0.9.0
 pathable==0.4.3
-pathspec==0.11.1
-platformdirs==3.9.1
+pathspec==0.11.2
+platformdirs==3.10.0
 prance==23.6.21.0
 prometheus-client==0.17.1
 protobuf==4.23.4

requirements.txt

@@ -3,6 +3,7 @@
 from abc import ABC
 from abc import abstractmethod
 from pathlib import Path
+from shutil import rmtree
 from shutil import which
 from typing import Any
 from typing import Awaitable
@@ -56,7 +57,7 @@ async def init(
             render_base(replay, force)
 
         await self.generate_abi()
-        await self.generate_schemas()
+        await self.generate_schemas(force)
         await self._generate_types(force)
 
         await self._generate_models()
@@ -71,7 +72,7 @@ async def generate_abi(self) -> None:
         ...
 
     @abstractmethod
-    async def generate_schemas(self) -> None:
+    async def generate_schemas(self, force: bool = False) -> None:
         ...
 
     @abstractmethod
@@ -206,6 +207,10 @@ async def _generate_models(self) -> None:
         content_path = Path(__file__).parent.parent / 'templates' / 'models.py'
         write(path, content_path.read_text())
 
+    def _cleanup_schemas(self) -> None:
+        rmtree(self._package.schemas)
+        self._package.schemas.mkdir()
+
 
 async def generate_environments(config: DipDupConfig, package: DipDupPackage) -> None:
     for default_env_path in package.deploy.glob(f'*{DEFAULT_ENV}'):

src/dipdup/codegen/__init__.py

@@ -111,7 +111,10 @@ async def generate_abi(self) -> None:
             elif isinstance(index_config, SubsquidOperationsIndexConfig):
                 raise NotImplementedError
 
-    async def generate_schemas(self) -> None:
+    async def generate_schemas(self, force: bool = False) -> None:
+        if force:
+            self._cleanup_schemas()
+
         events: set[str] = set()
         functions: set[str] = set()
 
@@ -180,3 +183,8 @@ def get_typeclass_name(self, schema_path: Path) -> str:
         # else:
         #     class_name = module_name
         return module_name
+
+    async def _generate_type(self, schema_path: Path, force: bool) -> None:
+        if 'evm_events' not in schema_path.parts:
+            return
+        await super()._generate_type(schema_path, force)

src/dipdup/codegen/evm_subsquid.py

@@ -101,9 +101,11 @@ def __init__(
     async def generate_abi(self) -> None:
         pass
 
-    async def generate_schemas(self) -> None:
+    async def generate_schemas(self, force: bool = False) -> None:
         """Fetch JSONSchemas for all contracts used in config"""
         self._logger.info('Fetching contract schemas')
+        if force:
+            self._cleanup_schemas()
 
         unused_operation_templates = [
             t for t in self._config.templates.values() if isinstance(t, TzktOperationsIndexConfig)
@@ -171,15 +173,26 @@ async def generate_system_hooks(self) -> None:
 
     def get_typeclass_name(self, schema_path: Path) -> str:
         module_name = schema_path.stem
-        if schema_path.name == 'tezos_storage.json':
+        if module_name == 'tezos_storage':
             class_name = f'{schema_path.parent.name}_storage'
         elif schema_path.parent.name == 'tezos_parameters':
             class_name = f'{module_name}_parameter'
         elif schema_path.parent.name == 'tezos_events':
             class_name = f'{module_name}_payload'
         else:
             class_name = module_name
-        return class_name
+        return snake_to_pascal(class_name)
+
+    async def _generate_type(self, schema_path: Path, force: bool) -> None:
+        markers = {
+            'tezos_storage.json',
+            'tezos_parameters',
+            'tezos_events',
+            'tezos_big_maps',
+        }
+        if not set(schema_path.parts).intersection(markers):
+            return
+        await super()._generate_type(schema_path, force)
 
     async def _get_schema(
         self,
@@ -378,7 +391,7 @@ def get_parameter_type(package: DipDupPackage, typename: str, entrypoint: str) -
 
 def get_event_payload_type(package: DipDupPackage, typename: str, tag: str) -> TypeClass:
     tag = pascal_to_snake(tag.replace('.', '_'))
-    module_name = f'tezos_event.{tag}'
+    module_name = f'tezos_events.{tag}'
     cls_name = snake_to_pascal(f'{tag}_payload')
     return package.get_type(typename, module_name, cls_name)
 

src/dipdup/codegen/tezos_tzkt.py

@@ -46,9 +46,10 @@ def prepare_event_handler_args(
         typename=handler_config.contract.module_name,
         tag=handler_config.tag,
     )
+    if not matched_event.payload:
+        raise FrameworkException('Event is typed, but payload is empty')
+
     with suppress(InvalidDataError):
-        if not matched_event.payload:
-            raise FrameworkException('Event payload is empty')
         typed_payload = parse_object(type_, matched_event.payload)
         return TzktEvent(
             data=matched_event,
@@ -97,6 +98,6 @@ def match_events(
     # NOTE: We don't care about `merge_subscriptions` here implying that all events will be processed
     # NOTE: Maybe "unfiltered" indexes will cover that case?
     for address in {event.contract_address for event in events}:
-        _logger.warning('Some events were not matched; fallback handler is missing for `{}`', address)
+        _logger.warning('Some events were not matched; fallback handler is missing for `%s`', address)
 
     return matched_handlers

src/dipdup/indexes/tezos_tzkt_events/matcher.py

@@ -61,7 +61,6 @@ class DipDupPackage:
     def __init__(self, root: Path) -> None:
         self.root = root
         self.name = root.name
-        self.loader = root / f'{self.name}.py'
 
         # NOTE: Package sections with .keep markers
         self.abi = root / 'abi'
@@ -108,7 +107,6 @@ def skel(self) -> dict[Path, str | None]:
             self.models: '**/*.py',
             self.sql: '**/*.sql',
             self.types: '**/*.py',
-            self.loader: '',
             # NOTE: Python metadata
             Path(PEP_561_MARKER): None,
             Path(PACKAGE_MARKER): None,