Fix creation of missing project package (#140)

Author: droserasprout

CHANGELOG.md

@@ -15,6 +15,7 @@
 * Removed unnecessary calls to TzKT API during the partial sync.
 * Fixed removal of PostgreSQL extensions (`timescaledb`, `pgcrypto`) by function `truncate_database` triggered on reindex.
 * Fixed updating relation between index and head in DB.
+* Fixed creation of missing project package
 
 ## 3.0.1 - 2021-09-24
 

src/dipdup/cli.py

@@ -92,6 +92,7 @@ def init_sentry(config: DipDupConfig) -> None:
 @click.pass_context
 @cli_wrapper
 async def cli(ctx, config: List[str], env_file: List[str], logging_config: str):
+    # NOTE: Config from cwd, fallback to builtin
     try:
         path = join(os.getcwd(), logging_config)
         _logging_config = LoggingConfig.load(path)
@@ -111,6 +112,8 @@ async def cli(ctx, config: List[str], env_file: List[str], logging_config: str):
     _config = DipDupConfig.load(config)
     init_sentry(_config)
 
+    await DipDupCodeGenerator(_config, {}).create_package()
+
     if _config.spec_version not in spec_version_mapping:
         raise ConfigurationError(f'Unknown `spec_version`, correct ones: {", ".join(spec_version_mapping)}')
     if _config.spec_version != __spec_version__ and ctx.invoked_subcommand != 'migrate':
@@ -148,7 +151,7 @@ async def run(ctx, reindex: bool, oneshot: bool) -> None:
 @cli_wrapper
 async def init(ctx, overwrite_types: bool):
     config: DipDupConfig = ctx.obj.config
-    config.pre_initialize()
+    config.initialize(skip_imports=True)
     dipdup = DipDup(config)
     await dipdup.init(overwrite_types)
 
@@ -158,7 +161,7 @@ async def init(ctx, overwrite_types: bool):
 @cli_wrapper
 async def migrate(ctx):
     config: DipDupConfig = ctx.obj.config
-    config.pre_initialize()
+    config.initialize(skip_imports=True)
     migrations = DipDupMigrationManager(config, ctx.obj.config_paths)
     await migrations.migrate()
 

src/dipdup/config.py

@@ -888,6 +888,7 @@ def __post_init_post_parse__(self):
         self._default_hooks: bool = False
         self._links_resolved: Set[str] = set()
         self._imports_resolved: Set[str] = set()
+        self._package_path: Optional[str] = None
 
     @property
     def environment(self) -> Dict[str, str]:
@@ -899,8 +900,17 @@ def filenames(self) -> List[str]:
 
     @property
     def package_path(self) -> str:
-        package = importlib.import_module(self.package)
-        return dirname(package.__file__)
+        if not self._package_path:
+            package = importlib.import_module(self.package)
+            self._package_path = dirname(package.__file__)
+
+        return self._package_path
+
+    @package_path.setter
+    def package_path(self, value: str):
+        if self._package_path:
+            raise ConfigInitializationException
+        self._package_path = value
 
     @classmethod
     def load(
@@ -972,14 +982,14 @@ def get_tzkt_datasource(self, name: str) -> TzktDatasourceConfig:
             raise ConfigurationError('`datasource` field must refer to TzKT datasource')
         return datasource
 
-    def pre_initialize(self) -> None:
+    def initialize(self, skip_imports: bool = False) -> None:
         self._set_names()
         self._resolve_templates()
         self._resolve_links()
         self._validate()
 
-    def initialize(self) -> None:
-        self.pre_initialize()
+        if skip_imports:
+            return
 
         for index_config in self.indexes.values():
             if index_config.name in self._imports_resolved:
@@ -991,12 +1001,12 @@ def initialize(self) -> None:
                 raise ConfigInitializationException
 
             elif isinstance(index_config, OperationIndexConfig):
-                self._load_operation_index_types(index_config)
-                self._load_index_callbacks(index_config)
+                self._import_operation_index_types(index_config)
+                self._import_index_callbacks(index_config)
 
             elif isinstance(index_config, BigMapIndexConfig):
-                self._load_big_map_index_types(index_config)
-                self._load_index_callbacks(index_config)
+                self._import_big_map_index_types(index_config)
+                self._import_index_callbacks(index_config)
 
             else:
                 raise NotImplementedError(f'Index kind `{index_config.kind}` is not supported')
@@ -1151,7 +1161,7 @@ def _set_names(self) -> None:
             for name, config in named_configs.items():
                 config.name = name
 
-    def _load_operation_index_types(self, index_config: OperationIndexConfig) -> None:
+    def _import_operation_index_types(self, index_config: OperationIndexConfig) -> None:
         for handler_config in index_config.handlers:
             for pattern_config in handler_config.pattern:
                 if isinstance(pattern_config, OperationHandlerTransactionPatternConfig):
@@ -1165,11 +1175,11 @@ def _load_operation_index_types(self, index_config: OperationIndexConfig) -> Non
                 else:
                     raise NotImplementedError
 
-    def _load_index_callbacks(self, index_config: ResolvedIndexConfigT) -> None:
+    def _import_index_callbacks(self, index_config: ResolvedIndexConfigT) -> None:
         for handler_config in index_config.handlers:
             handler_config.initialize_callback_fn(self.package)
 
-    def _load_big_map_index_types(self, index_config: BigMapIndexConfig) -> None:
+    def _import_big_map_index_types(self, index_config: BigMapIndexConfig) -> None:
         for big_map_handler_config in index_config.handlers:
             big_map_handler_config.initialize_big_map_type(self.package)
 

src/dipdup/dipdup.py

@@ -97,7 +97,7 @@ async def _fetch_contracts(self) -> None:
             if contract.name not in self._ctx.config.contracts:
                 contract_config = ContractConfig(address=contract.address, typename=contract.typename)
                 self._ctx.config.contracts[contract.name] = contract_config
-        self._ctx.config.pre_initialize()
+        self._ctx.config.initialize(skip_imports=True)
 
     async def _subscribe_to_datasource_events(self) -> None:
         for datasource in self._ctx.datasources.values():

tests/integration_tests/test_codegen.py

@@ -21,7 +21,7 @@ async def test_codegen(self):
             with self.subTest(name):
                 config_path = join(dirname(__file__), name)
                 config = DipDupConfig.load([config_path])
-                config.pre_initialize()
+                config.initialize(skip_imports=True)
                 config.package = 'tmp_test_dipdup'
 
                 if config.package in sys.modules:

tests/integration_tests/test_hasura.py

@@ -18,7 +18,7 @@ class HasuraTest(IsolatedAsyncioTestCase):
     async def test_configure_hasura(self):
         config_path = join(dirname(__file__), 'hic_et_nunc.yml')
         config = DipDupConfig.load([config_path])
-        config.pre_initialize()
+        config.initialize(skip_imports=True)
 
         async with AsyncExitStack() as stack:
             postgres_container = PostgresContainer()