tortoise-orm 0.19 bump (#273)

droserasprout · web-flow · commit 1fe125fd8ae6 · 2022-05-24T09:11:06.000Z
* tortoise-orm 0.19 bump

* Cleanup

* Bump tortoise

* DB utils, typing
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,10 @@
 * jobs: Fixed jobs with `daemon` schedule never start.
 * jobs: Fixed failed jobs not throwing exceptions into the main loop.
 
+### Other
+
+* database: Tortoise ORM updated to `0.19.1`.
+
 ## 5.1.1 - 2022-05-13
 
 ### Fixed
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -41,7 +41,7 @@ python-dotenv = "^0.19.0"
 sentry-sdk = "^1.4.3"
 sqlparse = "^0.4.2"
 tabulate = "^0.8.9"
-tortoise-orm = "0.17.8"
+tortoise-orm = "0.19.1"
 typing-inspect = "^0.6.0"
 
 [tool.poetry.dev-dependencies]
diff --git a/src/demo_hic_et_nunc/models.py b/src/demo_hic_et_nunc/models.py
@@ -1,5 +1,6 @@
 from enum import IntEnum
 
+from tortoise import ForeignKeyFieldInstance
 from tortoise import Model
 from tortoise import fields
 
@@ -16,15 +17,15 @@ class Holder(Model):
 
 class Token(Model):
     id = fields.BigIntField(pk=True)
-    creator = fields.ForeignKeyField('models.Holder', 'tokens')
+    creator: ForeignKeyFieldInstance[Holder] = fields.ForeignKeyField('models.Holder', 'tokens')
     supply = fields.BigIntField()
     level = fields.BigIntField()
     timestamp = fields.DatetimeField()
 
 
 class Swap(Model):
     id = fields.BigIntField(pk=True)
-    creator = fields.ForeignKeyField('models.Holder', 'swaps')
+    creator: ForeignKeyFieldInstance[Holder] = fields.ForeignKeyField('models.Holder', 'swaps')
     price = fields.BigIntField()
     amount = fields.BigIntField()
     amount_left = fields.BigIntField()
@@ -35,9 +36,9 @@ class Swap(Model):
 
 class Trade(Model):
     id = fields.BigIntField(pk=True)
-    swap = fields.ForeignKeyField('models.Swap', 'trades')
-    seller = fields.ForeignKeyField('models.Holder', 'sales')
-    buyer = fields.ForeignKeyField('models.Holder', 'purchases')
+    swap: ForeignKeyFieldInstance[Swap] = fields.ForeignKeyField('models.Swap', 'trades')
+    seller: ForeignKeyFieldInstance[Holder] = fields.ForeignKeyField('models.Holder', 'sales')
+    buyer: ForeignKeyFieldInstance[Holder] = fields.ForeignKeyField('models.Holder', 'purchases')
     amount = fields.BigIntField()
     level = fields.BigIntField()
     timestamp = fields.DatetimeField()
diff --git a/src/demo_registrydao/models.py b/src/demo_registrydao/models.py
@@ -1,3 +1,4 @@
+from tortoise import ForeignKeyFieldInstance
 from tortoise import Model
 from tortoise import fields
 
@@ -16,7 +17,7 @@ class Meta:
 
 class Proposal(Model):
     id = fields.IntField(pk=True)
-    dao = fields.ForeignKeyField('models.DAO', 'proposals')
+    dao: ForeignKeyFieldInstance[DAO] = fields.ForeignKeyField('models.DAO', 'proposals')
     # upvotes = fields.IntField(default=0)
     # downvotes = fields.IntField(default=0)
     # start_date = fields.DatetimeField()
@@ -29,7 +30,7 @@ class Meta:
 
 class Vote(Model):
     id = fields.IntField(pk=True)
-    proposal = fields.ForeignKeyField('models.Proposal', 'votes')
+    proposal: ForeignKeyFieldInstance[Proposal] = fields.ForeignKeyField('models.Proposal', 'votes')
     amount = fields.IntField()
 
     class Meta:
diff --git a/src/demo_tezos_domains/models.py b/src/demo_tezos_domains/models.py
@@ -1,3 +1,4 @@
+from tortoise import ForeignKeyFieldInstance
 from tortoise import Model
 from tortoise import fields
 
@@ -9,13 +10,13 @@ class TLD(Model):
 
 class Domain(Model):
     id = fields.CharField(max_length=255, pk=True)
-    tld = fields.ForeignKeyField('models.TLD', 'domains')
+    tld: ForeignKeyFieldInstance[TLD] = fields.ForeignKeyField('models.TLD', 'domains')
     expiry = fields.DatetimeField(null=True)
     owner = fields.CharField(max_length=36)
     token_id = fields.BigIntField(null=True)
 
 
 class Record(Model):
     id = fields.CharField(max_length=255, pk=True)
-    domain = fields.ForeignKeyField('models.Domain', 'records')
+    domain: ForeignKeyFieldInstance[Domain] = fields.ForeignKeyField('models.Domain', 'records')
     address = fields.CharField(max_length=36, null=True)
diff --git a/src/demo_tezos_domains_big_map/models.py b/src/demo_tezos_domains_big_map/models.py
@@ -1,3 +1,4 @@
+from tortoise import ForeignKeyFieldInstance
 from tortoise import Model
 from tortoise import fields
 
@@ -14,12 +15,12 @@ class Expiry(Model):
 
 class Domain(Model):
     id = fields.CharField(max_length=255, pk=True)
-    tld = fields.ForeignKeyField('models.TLD', 'domains')
+    tld: ForeignKeyFieldInstance[TLD] = fields.ForeignKeyField('models.TLD', 'domains')
     owner = fields.CharField(max_length=36)
     token_id = fields.BigIntField(null=True)
 
 
 class Record(Model):
     id = fields.CharField(max_length=255, pk=True)
-    domain = fields.ForeignKeyField('models.Domain', 'records')
+    domain: ForeignKeyFieldInstance[Domain] = fields.ForeignKeyField('models.Domain', 'records')
     address = fields.CharField(max_length=36, null=True)
diff --git a/src/demo_tzcolors/models.py b/src/demo_tzcolors/models.py
@@ -1,5 +1,6 @@
 from enum import IntEnum
 
+from tortoise import ForeignKeyFieldInstance
 from tortoise import Model
 from tortoise import fields
 
@@ -22,18 +23,18 @@ class Token(Model):
     amount = fields.BigIntField()
     level = fields.BigIntField()
     timestamp = fields.DatetimeField()
-    holder = fields.ForeignKeyField('models.Address', 'tokens')
+    holder: ForeignKeyFieldInstance[Address] = fields.ForeignKeyField('models.Address', 'tokens')
 
     class Meta:
         table = 'tokens'
 
 
 class Auction(Model):
     id = fields.BigIntField(pk=True)
-    token = fields.ForeignKeyField('models.Token', 'auctions')
+    token: ForeignKeyFieldInstance[Token] = fields.ForeignKeyField('models.Token', 'auctions')
     bid_amount = fields.BigIntField()
-    bidder = fields.ForeignKeyField('models.Address', 'winning_auctions')
-    seller = fields.ForeignKeyField('models.Address', 'created_auctions')
+    bidder: ForeignKeyFieldInstance[Address] = fields.ForeignKeyField('models.Address', 'winning_auctions')
+    seller: ForeignKeyFieldInstance[Address] = fields.ForeignKeyField('models.Address', 'created_auctions')
     end_timestamp = fields.DatetimeField()
     status = fields.IntEnumField(AuctionStatus)
     level = fields.BigIntField()
@@ -45,8 +46,8 @@ class Meta:
 
 class Bid(Model):
     id = fields.BigIntField(pk=True)
-    auction = fields.ForeignKeyField('models.Auction', 'bids')
+    auction: ForeignKeyFieldInstance[Auction] = fields.ForeignKeyField('models.Auction', 'bids')
     bid_amount = fields.BigIntField()
-    bidder = fields.ForeignKeyField('models.Address', 'bids')
+    bidder: ForeignKeyFieldInstance[Address] = fields.ForeignKeyField('models.Address', 'bids')
     level = fields.BigIntField()
     timestamp = fields.DatetimeField()
diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py
@@ -26,7 +26,6 @@
 from sentry_sdk.integrations.logging import LoggingIntegration
 from tabulate import tabulate
 from tortoise import Tortoise
-from tortoise.transactions import get_connection
 from tortoise.utils import get_schema_sql
 
 from dipdup import __spec_version__
@@ -48,6 +47,7 @@
 from dipdup.models import Schema
 from dipdup.utils import iter_files
 from dipdup.utils.database import generate_schema
+from dipdup.utils.database import get_connection
 from dipdup.utils.database import set_decimal_context
 from dipdup.utils.database import tortoise_wrapper
 from dipdup.utils.database import wipe_schema
@@ -406,7 +406,7 @@ async def schema_wipe(ctx, immune: bool, force: bool):
     _logger.info('Wiping schema `%s`', url)
 
     async with tortoise_wrapper(url, models):
-        conn = get_connection(None)
+        conn = get_connection()
         if isinstance(config.database, PostgresDatabaseConfig):
             await wipe_schema(
                 conn=conn,
@@ -437,7 +437,7 @@ async def schema_init(ctx):
         await dipdup._initialize_schema()
 
         # NOTE: It's not necessary a reindex, but it's safe to execute built-in scripts to (re)create views.
-        conn = get_connection(None)
+        conn = get_connection()
         await generate_schema(conn, config.database.schema_name)
 
     _logger.info('Schema initialized')
@@ -452,7 +452,7 @@ async def schema_export(ctx):
     models = f'{config.package}.models'
 
     async with tortoise_wrapper(url, models):
-        conn = get_connection(None)
+        conn = get_connection()
         output = get_schema_sql(conn, False) + '\n'
         dipdup_sql_path = join(dirname(__file__), 'sql', 'on_reindex')
         project_sql_path = join(config.package_path, 'sql', 'on_reindex')
diff --git a/src/dipdup/config.py b/src/dipdup/config.py
@@ -1037,7 +1037,7 @@ class JobConfig(NameMixin):
     args: Dict[str, Any] = field(default_factory=dict)
 
     def __post_init_post_parse__(self):
-        schedules_enabled = sum(map(lambda x: int(bool(x)), (self.crontab, self.interval, self.daemon)))
+        schedules_enabled = sum(int(bool(x)) for x in (self.crontab, self.interval, self.daemon))
         if schedules_enabled > 1:
             raise ConfigurationError('Only one of `crontab`, `interval` of `daemon` can be specified')
         elif not schedules_enabled:
diff --git a/src/dipdup/context.py b/src/dipdup/context.py
@@ -23,7 +23,7 @@
 
 from tortoise import Tortoise
 from tortoise.exceptions import OperationalError
-from tortoise.transactions import get_connection
+from tortoise.transactions import _get_connection
 
 from dipdup.config import BigMapIndexConfig
 from dipdup.config import ContractConfig
@@ -192,7 +192,7 @@ async def reindex(self, reason: Optional[Union[str, ReindexingReason]] = None, *
             raise ReindexingRequiredError(schema.reindex, context)
 
         elif action == ReindexingAction.wipe:
-            conn = get_connection(None)
+            conn = _get_connection(None)
             if isinstance(self.config.database, PostgresDatabaseConfig):
                 await wipe_schema(conn, self.config.database.schema_name, self.config.database.immune_tables)
             else:
@@ -474,7 +474,7 @@ async def execute_sql(self, ctx: 'DipDupContext', name: str) -> None:
             raise InitializationRequiredError(f'Missing SQL directory for hook `{name}`')
 
         # NOTE: SQL hooks are executed on default connection
-        connection = get_connection(None)
+        connection = _get_connection(None)
         await execute_sql_scripts(connection, sql_path)
 
     @contextmanager
diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py
@@ -17,7 +17,6 @@
 
 from prometheus_client import start_http_server  # type: ignore
 from tortoise.exceptions import OperationalError
-from tortoise.transactions import get_connection
 
 from dipdup.codegen import DipDupCodeGenerator
 from dipdup.config import ContractConfig
@@ -61,6 +60,7 @@
 from dipdup.utils import is_importable
 from dipdup.utils import slowdown
 from dipdup.utils.database import generate_schema
+from dipdup.utils.database import get_connection
 from dipdup.utils.database import get_schema_hash
 from dipdup.utils.database import prepare_models
 from dipdup.utils.database import tortoise_wrapper
@@ -430,7 +430,7 @@ async def _create_datasources(self) -> None:
     async def _initialize_schema(self) -> None:
         self._logger.info('Initializing database schema')
         schema_name = self._config.schema_name
-        conn = get_connection(None)
+        conn = get_connection()
 
         # NOTE: Try to fetch existing schema
         try:
diff --git a/src/dipdup/hasura.py b/src/dipdup/hasura.py
@@ -24,7 +24,6 @@
 from aiohttp import ServerDisconnectedError
 from pydantic.dataclasses import dataclass
 from tortoise import fields
-from tortoise.transactions import get_connection
 
 from dipdup.config import DEFAULT_POSTGRES_SCHEMA
 from dipdup.config import HasuraConfig
@@ -36,6 +35,7 @@
 from dipdup.models import Schema
 from dipdup.utils import iter_files
 from dipdup.utils import pascal_to_snake
+from dipdup.utils.database import get_connection
 from dipdup.utils.database import iter_models
 
 _get_fields_query = '''
@@ -227,10 +227,11 @@ async def _replace_metadata(self, metadata: Dict[str, Any]) -> None:
         await self._hasura_request(endpoint, json)
 
     async def _get_views(self) -> List[str]:
+        conn = get_connection()
         views = [
             row[0]
             for row in (
-                await get_connection(None).execute_query(
+                await conn.execute_query(
                     f"SELECT table_name FROM information_schema.views WHERE table_schema = '{self._database_config.schema_name}' UNION "
                     f"SELECT matviewname as table_name FROM pg_matviews WHERE schemaname = '{self._database_config.schema_name}'"
                 )
@@ -377,7 +378,7 @@ async def _get_fields(self, name: str = 'query_root') -> List[Field]:
         for field_json in fields_json:
             # NOTE: Exclude autogenerated aggregate and pk fields
             ignore_postfixes = ('_aggregate', '_by_pk', 'Aggregate', 'ByPk')
-            if any(map(lambda postfix: field_json['name'].endswith(postfix), ignore_postfixes)):  # noqa: C417
+            if any(field_json['name'].endswith(p) for p in ignore_postfixes):
                 continue
 
             # NOTE: Exclude relations. Not reliable enough but ok for now.
@@ -438,7 +439,7 @@ def _format_rest_query(self, name: str, table: str, filter: str, fields: Iterabl
             name = humps.camelize(name)
             filter = humps.camelize(filter)
             table = humps.camelize(table)
-            map(lambda f: f.camelize(), fields)  # noqa: C417
+            fields = [f.camelize() for f in fields]
 
         try:
             filter_field = next(f for f in fields if f.name == filter)
diff --git a/src/dipdup/index.py b/src/dipdup/index.py
@@ -514,7 +514,7 @@ async def _match_operation_subgroup(self, operation_subgroup: OperationSubgroup)
                     matched_operations.clear()
                     pattern_idx = 0
 
-            if len(matched_operations) >= sum(map(lambda x: 0 if x.optional else 1, handler_config.pattern)):
+            if len(matched_operations) >= sum(0 if x.optional else 1 for x in handler_config.pattern):
                 self._logger.info('%s: `%s` handler matched!', operation_subgroup.hash, handler_config.callback)
 
                 args = await self._prepare_handler_args(handler_config, matched_operations)
diff --git a/src/dipdup/models.py b/src/dipdup/models.py
@@ -233,7 +233,7 @@ class Index(Model):
 
     config_hash = fields.CharField(256)
     template = fields.CharField(256, null=True)
-    template_values = fields.JSONField(null=True)
+    template_values: Dict[str, Any] = fields.JSONField(null=True)
 
     level = fields.IntField(default=0)
 
diff --git a/src/dipdup/utils/database.py b/src/dipdup/utils/database.py
diff --git a/tests/test_dipdup/test_dipdup.py b/tests/test_dipdup/test_dipdup.py
diff --git a/tests/test_dipdup/test_utils.py b/tests/test_dipdup/test_utils.py
