Fix ReindexingReason parsing, fix FrozenInstanceError, update help messages (#310)

droserasprout · web-flow · commit 2294b8c13190 · 2022-04-21T02:25:10.000Z
* Fix `ReindexingReason` parsing, update help messages

* Fix `FrozenInstanceError`
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,17 @@
 # Changelog
 
+## [unreleased]
+
+### Fixed
+
+* context: Fixed reporting incorrect reindexing reason.
+* exceptions: Fixed crash with `FrozenInstanceError` when exception is raised from callback.
+
+### Improved
+
+* codegen: Refined `on_rollback` hook template.
+* exceptions: Updated help messages for known exceptions.
+
 ## 5.0.1 - 2022-04-12
 
 ### Fixed
diff --git a/src/demo_hic_et_nunc/hooks/on_rollback.py b/src/demo_hic_et_nunc/hooks/on_rollback.py
@@ -1,13 +1,18 @@
 from dipdup.context import HookContext
-from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import IndexDatasource
 from dipdup.enums import ReindexingReason
 
 
 async def on_rollback(
     ctx: HookContext,
-    datasource: Datasource,
+    datasource: IndexDatasource,
     from_level: int,
     to_level: int,
 ) -> None:
     await ctx.execute_sql('on_rollback')
-    await ctx.reindex(ReindexingReason.rollback)
+    await ctx.reindex(
+        ReindexingReason.rollback,
+        datasource=datasource.name,
+        from_level=from_level,
+        to_level=to_level,
+    )
diff --git a/src/demo_quipuswap/hooks/on_rollback.py b/src/demo_quipuswap/hooks/on_rollback.py
@@ -1,13 +1,18 @@
 from dipdup.context import HookContext
-from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import IndexDatasource
 from dipdup.enums import ReindexingReason
 
 
 async def on_rollback(
     ctx: HookContext,
-    datasource: Datasource,
+    datasource: IndexDatasource,
     from_level: int,
     to_level: int,
 ) -> None:
     await ctx.execute_sql('on_rollback')
-    await ctx.reindex(ReindexingReason.rollback)
+    await ctx.reindex(
+        ReindexingReason.rollback,
+        datasource=datasource.name,
+        from_level=from_level,
+        to_level=to_level,
+    )
diff --git a/src/demo_registrydao/hooks/on_rollback.py b/src/demo_registrydao/hooks/on_rollback.py
@@ -1,13 +1,18 @@
 from dipdup.context import HookContext
-from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import IndexDatasource
 from dipdup.enums import ReindexingReason
 
 
 async def on_rollback(
     ctx: HookContext,
-    datasource: Datasource,
+    datasource: IndexDatasource,
     from_level: int,
     to_level: int,
 ) -> None:
     await ctx.execute_sql('on_rollback')
-    await ctx.reindex(ReindexingReason.rollback)
+    await ctx.reindex(
+        ReindexingReason.rollback,
+        datasource=datasource.name,
+        from_level=from_level,
+        to_level=to_level,
+    )
diff --git a/src/demo_tezos_domains/hooks/on_rollback.py b/src/demo_tezos_domains/hooks/on_rollback.py
@@ -1,13 +1,18 @@
 from dipdup.context import HookContext
-from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import IndexDatasource
 from dipdup.enums import ReindexingReason
 
 
 async def on_rollback(
     ctx: HookContext,
-    datasource: Datasource,
+    datasource: IndexDatasource,
     from_level: int,
     to_level: int,
 ) -> None:
     await ctx.execute_sql('on_rollback')
-    await ctx.reindex(ReindexingReason.rollback)
+    await ctx.reindex(
+        ReindexingReason.rollback,
+        datasource=datasource.name,
+        from_level=from_level,
+        to_level=to_level,
+    )
diff --git a/src/demo_tezos_domains_big_map/hooks/on_rollback.py b/src/demo_tezos_domains_big_map/hooks/on_rollback.py
@@ -1,13 +1,18 @@
 from dipdup.context import HookContext
-from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import IndexDatasource
 from dipdup.enums import ReindexingReason
 
 
 async def on_rollback(
     ctx: HookContext,
-    datasource: Datasource,
+    datasource: IndexDatasource,
     from_level: int,
     to_level: int,
 ) -> None:
     await ctx.execute_sql('on_rollback')
-    await ctx.reindex(ReindexingReason.rollback)
+    await ctx.reindex(
+        ReindexingReason.rollback,
+        datasource=datasource.name,
+        from_level=from_level,
+        to_level=to_level,
+    )
diff --git a/src/demo_tzbtc/hooks/on_rollback.py b/src/demo_tzbtc/hooks/on_rollback.py
@@ -1,13 +1,18 @@
 from dipdup.context import HookContext
-from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import IndexDatasource
 from dipdup.enums import ReindexingReason
 
 
 async def on_rollback(
     ctx: HookContext,
-    datasource: Datasource,
+    datasource: IndexDatasource,
     from_level: int,
     to_level: int,
 ) -> None:
     await ctx.execute_sql('on_rollback')
-    await ctx.reindex(ReindexingReason.rollback)
+    await ctx.reindex(
+        ReindexingReason.rollback,
+        datasource=datasource.name,
+        from_level=from_level,
+        to_level=to_level,
+    )
diff --git a/src/demo_tzcolors/hooks/on_rollback.py b/src/demo_tzcolors/hooks/on_rollback.py
@@ -1,13 +1,18 @@
 from dipdup.context import HookContext
-from dipdup.datasources.datasource import Datasource
+from dipdup.datasources.datasource import IndexDatasource
 from dipdup.enums import ReindexingReason
 
 
 async def on_rollback(
     ctx: HookContext,
-    datasource: Datasource,
+    datasource: IndexDatasource,
     from_level: int,
     to_level: int,
 ) -> None:
     await ctx.execute_sql('on_rollback')
-    await ctx.reindex(ReindexingReason.rollback)
+    await ctx.reindex(
+        ReindexingReason.rollback,
+        datasource=datasource.name,
+        from_level=from_level,
+        to_level=to_level,
+    )
diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py
@@ -376,7 +376,12 @@ async def _generate_callback(self, callback_config: CallbackMixin, sql: bool = F
                 code.append(f"await ctx.execute_sql('{original_callback}')")
                 if callback == 'on_rollback':
                     imports.add('from dipdup.enums import ReindexingReason')
-                    code.append('await ctx.reindex(ReindexingReason.rollback)')
+                    code.append('await ctx.reindex(')
+                    code.append('    ReindexingReason.rollback,')
+                    code.append('    datasource=datasource.name,')
+                    code.append('    from_level=from_level,')
+                    code.append('    to_level=to_level,')
+                    code.append(')')
             else:
                 code.append('...')
 
diff --git a/src/dipdup/config.py b/src/dipdup/config.py
@@ -1069,7 +1069,7 @@ def iter_imports(self, package: str) -> Iterator[Tuple[str, str]]:
     'on_rollback': HookConfig(
         callback='on_rollback',
         args={
-            'datasource': 'dipdup.datasources.datasource.Datasource',
+            'datasource': 'dipdup.datasources.datasource.IndexDatasource',
             'from_level': 'int',
             'to_level': 'int',
         },
diff --git a/src/dipdup/context.py b/src/dipdup/context.py
@@ -165,16 +165,18 @@ async def restart(self) -> None:
 
     async def reindex(self, reason: Optional[Union[str, ReindexingReason]] = None, **context) -> None:
         """Drop the whole database and restart with the same CLI arguments"""
-        if not reason:
+        if reason is None:
             reason = ReindexingReason.manual
-        elif isinstance(reason, str):
+        # NOTE: Do not check for `str`!
+        elif not isinstance(reason, ReindexingReason):
             context['message'] = reason
             reason = ReindexingReason.manual
 
         action = self.config.advanced.reindex.get(reason, ReindexingAction.exception)
-        self.logger.warning('Reindexing initialized, reason: %s, action: %s', reason.value, action.value)
+        self.logger.warning('Reindexing requested: reason `%s`, action `%s`', reason.value, action.value)
 
         if action == ReindexingAction.ignore:
+            # NOTE: Recalculate hashes on the next run
             if reason == ReindexingReason.schema_modified:
                 await Schema.filter(name=self.config.schema_name).update(hash='')
             elif reason == ReindexingReason.config_modified:
@@ -406,17 +408,18 @@ async def fire_handler(
         *args,
         **kwargs: Any,
     ) -> None:
+        module = f'{self._package}.handlers.{name}'
         handler_config = self._get_handler(name, index)
         new_ctx = HandlerContext(
             datasources=ctx.datasources,
             config=ctx.config,
             callbacks=ctx.callbacks,
-            logger=FormattedLogger(f'dipdup.handlers.{name}', fmt),
+            logger=FormattedLogger(module, fmt),
             handler_config=handler_config,
             datasource=datasource,
         )
         # NOTE: Handlers are not atomic, levels are. Do not open transaction here.
-        with self._callback_wrapper('handler', name):
+        with self._callback_wrapper(module):
             await handler_config.callback_fn(new_ctx, *args, **kwargs)
 
     async def fire_hook(
@@ -428,21 +431,21 @@ async def fire_hook(
         *args,
         **kwargs: Any,
     ) -> None:
+        module = f'{self._package}.hooks.{name}'
         hook_config = self._get_hook(name)
         new_ctx = HookContext(
             datasources=ctx.datasources,
             config=ctx.config,
             callbacks=ctx.callbacks,
-            logger=FormattedLogger(f'dipdup.hooks.{name}', fmt),
+            logger=FormattedLogger(module, fmt),
             hook_config=hook_config,
         )
 
         self._verify_arguments(new_ctx, *args, **kwargs)
 
         async def _wrapper():
             async with AsyncExitStack() as stack:
-
-                stack.enter_context(self._callback_wrapper('hook', name))
+                stack.enter_context(self._callback_wrapper(module))
                 if hook_config.atomic:
                     await stack.enter_async_context(in_global_transaction())
 
@@ -469,16 +472,16 @@ async def execute_sql(self, ctx: 'DipDupContext', name: str) -> None:
         await execute_sql_scripts(connection, sql_path)
 
     @contextmanager
-    def _callback_wrapper(self, kind: str, name: str) -> Iterator[None]:
+    def _callback_wrapper(self, module: str) -> Iterator[None]:
         try:
             with ExitStack() as stack:
                 if Metrics.enabled:
-                    stack.enter_context(Metrics.measure_callback_duration(name))
+                    stack.enter_context(Metrics.measure_callback_duration(module))
                 yield
         except Exception as e:
             if isinstance(e, ReindexingRequiredError):
                 raise
-            raise CallbackError(kind, name) from e
+            raise CallbackError(module, e) from e
 
     @classmethod
     def _verify_arguments(cls, ctx: HookContext, *args, **kwargs) -> None:
diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py
@@ -484,12 +484,15 @@ async def _initialize_datasources(self) -> None:
             if not db_head:
                 continue
 
+            # NOTE: Ensure that no reorgs happened while we were offline
             actual_head = await datasource.get_block(db_head.level)
             if db_head.hash != actual_head.hash:
                 await self._ctx.reindex(
                     ReindexingReason.rollback,
-                    hash=db_head.hash,
-                    actual_hash=actual_head.hash,
+                    datasource=datasource.name,
+                    level=db_head.level,
+                    stored_block_hash=db_head.hash,
+                    actual_block_hash=actual_head.hash,
                 )
 
     async def _set_up_index_dispatcher(
diff --git a/src/dipdup/exceptions.py b/src/dipdup/exceptions.py
diff --git a/src/dipdup/index.py b/src/dipdup/index.py
diff --git a/tests/test_dipdup/test_config/test_custom_config.py b/tests/test_dipdup/test_config/test_custom_config.py
