Bugfixes: faster sync, TimescaleDB compatibility (#139)

Author: droserasprout

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 3.0.2 - 2021-09-25
+
+### Added
+
+* Human-readable `CHANGELOG.md` 🕺
+
+### Changed
+
+* Reindex on schema hash mismatch was disabled until TimescaleDB issues won't be resolved.
+
+### Fixed
+
+* Removed unnecessary calls to TzKT API during the partial sync.
+* Fixed removal of PostgreSQL extensions (`timescaledb`, `pgcrypto`) by function `truncate_database` triggered on reindex.
+* Fixed updating relation between index and head in DB.
+
 ## 3.0.1 - 2021-09-24
 
 ### Added

src/dipdup/dipdup.py

@@ -262,6 +262,7 @@ async def _create_datasources(self) -> None:
 
     async def _initialize_schema(self) -> None:
         self._logger.info('Initializing database schema')
+        # TODO: Incorrect for sqlite, fix on the next major release
         schema_name = 'public'
         conn = get_connection(None)
 
@@ -293,7 +294,9 @@ async def _initialize_schema(self) -> None:
                 raise ReindexingRequiredError from e
 
         elif self._schema.hash != schema_hash:
-            await self._ctx.reindex(reason='schema hash mismatch')
+            # FIXME: It seems like this check is broken in some cases
+            # await self._ctx.reindex(reason='schema hash mismatch')
+            self._logger.error('Schema hash mismatch, reindex may be required')
 
         await self._ctx.fire_hook('on_restart')
 

src/dipdup/index.py

@@ -21,7 +21,7 @@
 from dipdup.context import DipDupContext
 from dipdup.datasources.tzkt.datasource import BigMapFetcher, OperationFetcher, TzktDatasource
 from dipdup.exceptions import ConfigInitializationException, InvalidDataError
-from dipdup.models import BigMapData, BigMapDiff, HeadBlockData
+from dipdup.models import BigMapData, BigMapDiff, Head, HeadBlockData
 from dipdup.models import Index as IndexState
 from dipdup.models import IndexStatus, OperationData, Origination, Transaction
 from dipdup.utils import FormattedLogger
@@ -111,7 +111,15 @@ async def _process_queue(self) -> None:
         ...
 
     async def _enter_sync_state(self, last_level: int) -> Optional[int]:
-        first_level = self.state.level
+        if self.state.status == IndexStatus.ONESHOT:
+            return None
+        # FIXME: I'm not sure if this is a good way to check if index is in sync
+        # TODO: Move to model class
+        elif self.state.status == IndexStatus.REALTIME and isinstance(self.state.head, Head):
+            self.state.head.level
+        else:
+            first_level = self.state.level
+
         if first_level == last_level:
             return None
         if first_level > last_level:

src/dipdup/models.py

@@ -314,7 +314,10 @@ async def update_status(
                 raise DipDupException('Index level is higher than desired level')
             self.level = level  # type: ignore
 
-        self.head = head
+        if head:
+            self.head = head
+            self.head_id = head.pk if head else None
+
         self.status = status
         await self.save()
 

src/dipdup/utils/database.py

@@ -128,6 +128,8 @@ async def create_schema(conn: BaseDBAsyncClient, name: str) -> None:
         raise NotImplementedError
 
     await conn.execute_script(f'CREATE SCHEMA IF NOT EXISTS {name}')
+    # FIXME: Oh...
+    await conn.execute_script(_truncate_schema_sql)
 
 
 async def generate_schema(conn: BaseDBAsyncClient, name: str) -> None:

src/dipdup/utils/truncate_schema.sql

@@ -39,7 +39,7 @@ BEGIN
             EXECUTE rec.name;
         EXCEPTION
             WHEN others THEN END;
-        END LOOP;
+    END LOOP;
 
     FOR rec IN SELECT
             'DROP FUNCTION ' || quote_ident(ns.nspname) || '.'
@@ -52,16 +52,25 @@ BEGIN
         ON
             (pg_proc.pronamespace = ns.oid)
         WHERE
-            ns.nspname =
-            schema_name
+            ns.nspname = schema_name AND
+            pg_catalog.pg_function_is_visible(pg_proc.oid)
         ORDER BY
             proname
     LOOP
         BEGIN
             EXECUTE rec.name;
         EXCEPTION
             WHEN others THEN END;
-        END LOOP;
+    END LOOP;
+
+    -- BEGIN
+    --     CREATE EXTENSION IF NOT EXISTS pgcrypto;
+    --     CREATE EXTENSION IF NOT EXISTS timescaledb;
+    -- EXCEPTION
+    --     WHEN OTHERS THEN
+    --         NULL;
+    -- END;
+    
     RETURN;
 END;
 $$ LANGUAGE plpgsql;