Notify about required reindexing, wrap tortoise exceptions (#100)

Author: droserasprout

src/dipdup/__init__.py

@@ -5,3 +5,8 @@
     '1.0': '>=1.0.0, <=1.1.2',
     '1.1': '>=2.0.0',
 }
+spec_reindex_mapping = {
+    '0.1': False,
+    '1.0': False,
+    '1.1': True,
+}

src/dipdup/cli.py

@@ -13,7 +13,7 @@
 from fcache.cache import FileCache  # type: ignore
 from sentry_sdk.integrations.aiohttp import AioHttpIntegration
 
-from dipdup import __spec_version__, __version__, spec_version_mapping
+from dipdup import __spec_version__, __version__, spec_version_mapping, spec_reindex_mapping
 from dipdup.codegen import DEFAULT_DOCKER_ENV_FILE, DEFAULT_DOCKER_IMAGE, DEFAULT_DOCKER_TAG, DipDupCodeGenerator
 from dipdup.config import DipDupConfig, LoggingConfig, PostgresDatabaseConfig
 from dipdup.dipdup import DipDup
@@ -71,7 +71,8 @@ async def cli(ctx, config: List[str], env_file: List[str], logging_config: str):
     if _config.spec_version not in spec_version_mapping:
         raise ConfigurationError('Unknown `spec_version`, correct ones: {}')
     if _config.spec_version != __spec_version__ and ctx.invoked_subcommand != 'migrate':
-        raise MigrationRequiredError(None, _config.spec_version, __spec_version__)
+        reindex = spec_reindex_mapping[__spec_version__]
+        raise MigrationRequiredError(None, _config.spec_version, __spec_version__, reindex)
 
     if _config.sentry:
         sentry_sdk.init(

src/dipdup/dipdup.py

@@ -32,7 +32,7 @@
 from dipdup.datasources.coinbase.datasource import CoinbaseDatasource
 from dipdup.datasources.datasource import IndexDatasource
 from dipdup.datasources.tzkt.datasource import TzktDatasource
-from dipdup.exceptions import ConfigurationError
+from dipdup.exceptions import ConfigurationError, ReindexingRequiredError
 from dipdup.hasura import HasuraGateway
 from dipdup.index import BigMapIndex, Index, OperationIndex
 from dipdup.models import BigMapData, HeadBlockData, IndexType, OperationData, State
@@ -287,16 +287,20 @@ async def _initialize_database(self, reindex: bool = False) -> None:
 
         self._logger.info('Checking database schema')
         connection_name, connection = next(iter(Tortoise._connections.items()))
-        schema_sql = get_schema_sql(connection, False)
-
-        # NOTE: Column order could differ in two generated schemas for the same models, drop commas and sort strings to eliminate this
-        processed_schema_sql = '\n'.join(sorted(schema_sql.replace(',', '').split('\n'))).encode()
-        schema_hash = hashlib.sha256(processed_schema_sql).hexdigest()
 
         try:
             schema_state = await State.get_or_none(index_type=IndexType.schema, index_name=connection_name)
         except OperationalError:
             schema_state = None
+        # TODO: Process exception in Tortoise
+        except KeyError as e:
+            raise ReindexingRequiredError(None) from e
+
+        schema_sql = get_schema_sql(connection, False)
+
+        # NOTE: Column order could differ in two generated schemas for the same models, drop commas and sort strings to eliminate this
+        processed_schema_sql = '\n'.join(sorted(schema_sql.replace(',', '').split('\n'))).encode()
+        schema_hash = hashlib.sha256(processed_schema_sql).hexdigest()
 
         # NOTE: `State.index_hash` field contains schema hash when `index_type` is `IndexType.schema`
         if schema_state is None:

src/dipdup/exceptions.py

@@ -7,6 +7,8 @@
 
 from dipdup import spec_version_mapping
 
+_tab = '\n\n' + ('_' * 80) + '\n\n'
+
 _migration_required_message = """Project migration required!
 
 {version_table}
@@ -17,6 +19,14 @@
 See https://baking-bad.org/blog/ for additional release information.
 """
 
+_reindexing_required_message = """Reindexing required!
+
+Recent changes in the framework have made it necessary to reindex the project.
+
+  1. Optionally backup a database
+  2. Run `dipdup run --reindex` 
+"""
+
 _handler_import_message = """Failed to import `{obj}` from `{module}`.
 
 Reasons in order of possibility:
@@ -52,8 +62,6 @@
 {error_context}
 """
 
-_tab = '\n\n' + ('_' * 80) + '\n\n'
-
 
 class DipDupError(ABC, Exception):
     exit_code = 1
@@ -86,12 +94,13 @@ def format_help(self) -> str:
 
 
 class MigrationRequiredError(DipDupError):
-    """Project and DipDup spec versions don't match """
+    """Project and DipDup spec versions don't match"""
 
-    def __init__(self, ctx, from_: str, to: str) -> None:
+    def __init__(self, ctx, from_: str, to: str, reindex: bool = False) -> None:
         super().__init__(ctx)
         self.from_ = from_
         self.to = to
+        self.reindex = reindex
 
     def format_help(self) -> str:
         version_table = tabulate(
@@ -101,7 +110,16 @@ def format_help(self) -> str:
             ],
             headers=['', 'spec_version', 'DipDup version'],
         )
-        return _migration_required_message.format(version_table=version_table)
+        message = _migration_required_message.format(version_table=version_table)
+        if self.reindex:
+            message += _tab +_reindexing_required_message
+        return message
+
+
+class ReindexingRequiredError(DipDupError):
+    """Performed migration requires reindexing"""
+    def format_help(self) -> str:
+        return _reindexing_required_message
 
 
 class HandlerImportError(DipDupError):