Update demo configs, fix schema export, check for early_realtime in big_map index (#212)

* Update demo configs, raise exception on useless combo

* Bump

* Fix schema export

* Fix registrydao demo and test

Author: droserasprout

poetry.lock

@@ -18,6 +18,9 @@ datasources:
     kind: tzkt
     url: ${TZKT_URL:-https://api.tzkt.io}
 
+advanced:
+  early_realtime: True
+
 indexes:
   hen_mainnet:
     kind: operation

src/demo_hic_et_nunc/dipdup.yml

@@ -24,6 +24,9 @@ datasources:
     kind: tzkt
     url: https://api.tzkt.io
 
+advanced:
+  early_realtime: True
+
 templates:
   quipuswap_fa12:
     kind: operation

src/demo_quipuswap/dipdup.yml

@@ -7,14 +7,17 @@ database:
 
 contracts:
   registry:
-    address: KT1JytEHBVBg27n9w85tNhSWUpbTvETD6rx6
+    address: KT19CF3KKrvdW77ttFomCuin2k4uAVkryYqh
     typename: registry
 
 datasources:
   tzkt:
     kind: tzkt
     url: ${TZKT_URL:-https://api.tzkt.io}
 
+advanced:
+  early_realtime: True
+
 templates:
 
   registry_dao:

src/demo_registrydao/dipdup.yml

@@ -15,6 +15,9 @@ datasources:
     kind: tzkt
     url: ${TZKT_URL:-https://api.tzkt.io}
 
+advanced:
+  early_realtime: True
+
 templates:
   tezos_domains:
     kind: operation

src/demo_tezos_domains/dipdup.yml

@@ -15,6 +15,9 @@ datasources:
     kind: tzkt
     url: https://api.tzkt.io
 
+advanced:
+  early_realtime: True
+
 indexes:
   tzbtc_holders_mainnet:
     kind: operation

src/demo_tzbtc/dipdup.yml

@@ -18,6 +18,9 @@ datasources:
     kind: tzkt
     url: ${TZKT_URL:-https://api.tzkt.io}
 
+advanced:
+  early_realtime: True
+
 templates:
 
   tzcolors_auction:

src/demo_tzcolors/dipdup.yml

@@ -427,6 +427,11 @@ async def schema_export(ctx):
     async with tortoise_wrapper(url, models):
         conn = get_connection(None)
         output = get_schema_sql(conn, False) + '\n'
-        for file in iter_files(join(config.package_path, 'sql', 'on_reindex')):
-            output += file.read() + '\n'
+        dipdup_sql_path = join(dirname(__file__), 'sql', 'on_reindex')
+        project_sql_path = join(config.package_path, 'sql', 'on_reindex')
+
+        for sql_path in (dipdup_sql_path, project_sql_path):
+            for file in iter_files(sql_path):
+                output += file.read() + '\n'
+
         echo(output)

src/dipdup/cli.py

@@ -40,6 +40,7 @@
 from dipdup.datasources.tzkt.datasource import TzktDatasource
 from dipdup.datasources.tzkt.models import deserialize_storage
 from dipdup.exceptions import ConfigInitializationException
+from dipdup.exceptions import ConfigurationError
 from dipdup.exceptions import InvalidDataError
 from dipdup.exceptions import ReindexingReason
 from dipdup.models import BigMapAction
@@ -608,6 +609,10 @@ async def _synchronize_full(self, first_level: int, last_level: int, cache: bool
             await self._process_level_big_maps(big_maps)
 
     async def _synchronize_level(self, last_level: int, cache: bool = False) -> None:
+        # NOTE: Checking late because feature flags could be modified after loading config
+        if not self._ctx.config.advanced.early_realtime:
+            raise ConfigurationError('`skip_history` requires `early_realtime` feature flag to be enabled')
+
         big_map_addresses = await self._get_big_map_addresses()
         big_map_paths = await self._get_big_map_paths()
         big_map_ids: Tuple[Tuple[int, str], ...] = ()

src/dipdup/index.py

@@ -7,7 +7,7 @@ database:
 
 contracts:
   registry:
-    address: KT1QMdCTqzmY4QKHntV1nZEinLPU1GbxUFQu
+    address: KT19CF3KKrvdW77ttFomCuin2k4uAVkryYqh
     typename: registry
 
 datasources: