Skip to content

Commit 507a10e

Browse files
Bugfixes: faster sync, TimescaleDB compatibility (#139)
1 parent 53e8bcb commit 507a10e

File tree

6 files changed

+49
-8
lines changed

6 files changed

+49
-8
lines changed

CHANGELOG.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
11
# Changelog
22

3+
## 3.0.2 - 2021-09-25
4+
5+
### Added
6+
7+
* Human-readable `CHANGELOG.md` 🕺
8+
9+
### Changed
10+
11+
* Reindex on schema hash mismatch was disabled until TimescaleDB issues won't be resolved.
12+
13+
### Fixed
14+
15+
* Removed unnecessary calls to TzKT API during the partial sync.
16+
* Fixed removal of PostgreSQL extensions (`timescaledb`, `pgcrypto`) by function `truncate_database` triggered on reindex.
17+
* Fixed updating relation between index and head in DB.
18+
319
## 3.0.1 - 2021-09-24
420

521
### Added

src/dipdup/dipdup.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,7 @@ async def _create_datasources(self) -> None:
262262

263263
async def _initialize_schema(self) -> None:
264264
self._logger.info('Initializing database schema')
265+
# TODO: Incorrect for sqlite, fix on the next major release
265266
schema_name = 'public'
266267
conn = get_connection(None)
267268

@@ -293,7 +294,9 @@ async def _initialize_schema(self) -> None:
293294
raise ReindexingRequiredError from e
294295

295296
elif self._schema.hash != schema_hash:
296-
await self._ctx.reindex(reason='schema hash mismatch')
297+
# FIXME: It seems like this check is broken in some cases
298+
# await self._ctx.reindex(reason='schema hash mismatch')
299+
self._logger.error('Schema hash mismatch, reindex may be required')
297300

298301
await self._ctx.fire_hook('on_restart')
299302

src/dipdup/index.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from dipdup.context import DipDupContext
2222
from dipdup.datasources.tzkt.datasource import BigMapFetcher, OperationFetcher, TzktDatasource
2323
from dipdup.exceptions import ConfigInitializationException, InvalidDataError
24-
from dipdup.models import BigMapData, BigMapDiff, HeadBlockData
24+
from dipdup.models import BigMapData, BigMapDiff, Head, HeadBlockData
2525
from dipdup.models import Index as IndexState
2626
from dipdup.models import IndexStatus, OperationData, Origination, Transaction
2727
from dipdup.utils import FormattedLogger
@@ -111,7 +111,15 @@ async def _process_queue(self) -> None:
111111
...
112112

113113
async def _enter_sync_state(self, last_level: int) -> Optional[int]:
114-
first_level = self.state.level
114+
if self.state.status == IndexStatus.ONESHOT:
115+
return None
116+
# FIXME: I'm not sure if this is a good way to check if index is in sync
117+
# TODO: Move to model class
118+
elif self.state.status == IndexStatus.REALTIME and isinstance(self.state.head, Head):
119+
self.state.head.level
120+
else:
121+
first_level = self.state.level
122+
115123
if first_level == last_level:
116124
return None
117125
if first_level > last_level:

src/dipdup/models.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,10 @@ async def update_status(
314314
raise DipDupException('Index level is higher than desired level')
315315
self.level = level # type: ignore
316316

317-
self.head = head
317+
if head:
318+
self.head = head
319+
self.head_id = head.pk if head else None
320+
318321
self.status = status
319322
await self.save()
320323

src/dipdup/utils/database.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ async def create_schema(conn: BaseDBAsyncClient, name: str) -> None:
128128
raise NotImplementedError
129129

130130
await conn.execute_script(f'CREATE SCHEMA IF NOT EXISTS {name}')
131+
# FIXME: Oh...
132+
await conn.execute_script(_truncate_schema_sql)
131133

132134

133135
async def generate_schema(conn: BaseDBAsyncClient, name: str) -> None:

src/dipdup/utils/truncate_schema.sql

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ BEGIN
3939
EXECUTE rec.name;
4040
EXCEPTION
4141
WHEN others THEN END;
42-
END LOOP;
42+
END LOOP;
4343

4444
FOR rec IN SELECT
4545
'DROP FUNCTION ' || quote_ident(ns.nspname) || '.'
@@ -52,16 +52,25 @@ BEGIN
5252
ON
5353
(pg_proc.pronamespace = ns.oid)
5454
WHERE
55-
ns.nspname =
56-
schema_name
55+
ns.nspname = schema_name AND
56+
pg_catalog.pg_function_is_visible(pg_proc.oid)
5757
ORDER BY
5858
proname
5959
LOOP
6060
BEGIN
6161
EXECUTE rec.name;
6262
EXCEPTION
6363
WHEN others THEN END;
64-
END LOOP;
64+
END LOOP;
65+
66+
-- BEGIN
67+
-- CREATE EXTENSION IF NOT EXISTS pgcrypto;
68+
-- CREATE EXTENSION IF NOT EXISTS timescaledb;
69+
-- EXCEPTION
70+
-- WHEN OTHERS THEN
71+
-- NULL;
72+
-- END;
73+
6574
RETURN;
6675
END;
6776
$$ LANGUAGE plpgsql;

0 commit comments

Comments
 (0)