Skip to content

Commit e34c2c0

Browse files
henadzithenadzit
authored
Use pypika's SqlContext to improve performance (#1837)
* Updates required for SqlContext * Use pypika-tortoise=0.5.0
1 parent 46e3aef commit e34c2c0

File tree

13 files changed

+74
-56
lines changed

13 files changed

+74
-56
lines changed

‎CHANGELOG.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ Fixed
1818
Changed
1919
^^^^^^^
2020
- Optimize field conversion to database format to speed up `create` and `bulk_create` (#1840)
21+
- Improved query performance by optimizing SQL generation (#1837)
2122

2223
0.23.0
2324
------

‎poetry.lock‎

Lines changed: 12 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ classifiers = [
3737

3838
[tool.poetry.dependencies]
3939
python = "^3.8"
40-
pypika-tortoise = "^0.4.0"
40+
pypika-tortoise = "^0.5.0"
4141
iso8601 = "^2.1.0"
4242
aiosqlite = ">=0.16.0, <0.21.0"
4343
pytz = "*"

‎tests/test_q.py‎

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import operator
22
from unittest import TestCase as _TestCase
33

4+
from pypika_tortoise.context import DEFAULT_SQL_CONTEXT
5+
46
from tests.testmodels import CharFields, IntFields
57
from tortoise.contrib.test import TestCase
68
from tortoise.exceptions import OperationalError
@@ -134,58 +136,64 @@ def setUp(self) -> None:
134136
def test_q_basic(self):
135137
q = Q(id=8)
136138
r = q.resolve(self.int_fields_context)
137-
self.assertEqual(r.where_criterion.get_sql(), '"id"=8')
139+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id"=8')
138140

139141
def test_q_basic_and(self):
140142
q = Q(join_type="AND", id=8)
141143
r = q.resolve(self.int_fields_context)
142-
self.assertEqual(r.where_criterion.get_sql(), '"id"=8')
144+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id"=8')
143145

144146
def test_q_basic_or(self):
145147
q = Q(join_type="OR", id=8)
146148
r = q.resolve(self.int_fields_context)
147-
self.assertEqual(r.where_criterion.get_sql(), '"id"=8')
149+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id"=8')
148150

149151
def test_q_multiple_and(self):
150152
q = Q(join_type="AND", id__gt=8, id__lt=10)
151153
r = q.resolve(self.int_fields_context)
152-
self.assertEqual(r.where_criterion.get_sql(), '"id">8 AND "id"<10')
154+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">8 AND "id"<10')
153155

154156
def test_q_multiple_or(self):
155157
q = Q(join_type="OR", id__gt=8, id__lt=10)
156158
r = q.resolve(self.int_fields_context)
157-
self.assertEqual(r.where_criterion.get_sql(), '"id">8 OR "id"<10')
159+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">8 OR "id"<10')
158160

159161
def test_q_multiple_and2(self):
160162
q = Q(join_type="AND", id=8, intnum=80)
161163
r = q.resolve(self.int_fields_context)
162-
self.assertEqual(r.where_criterion.get_sql(), '"id"=8 AND "intnum"=80')
164+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id"=8 AND "intnum"=80')
163165

164166
def test_q_multiple_or2(self):
165167
q = Q(join_type="OR", id=8, intnum=80)
166168
r = q.resolve(self.int_fields_context)
167-
self.assertEqual(r.where_criterion.get_sql(), '"id"=8 OR "intnum"=80')
169+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id"=8 OR "intnum"=80')
168170

169171
def test_q_complex_int(self):
170172
q = Q(Q(intnum=80), Q(id__lt=5, id__gt=50, join_type="OR"), join_type="AND")
171173
r = q.resolve(self.int_fields_context)
172-
self.assertEqual(r.where_criterion.get_sql(), '"intnum"=80 AND ("id"<5 OR "id">50)')
174+
self.assertEqual(
175+
r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"intnum"=80 AND ("id"<5 OR "id">50)'
176+
)
173177

174178
def test_q_complex_int2(self):
175179
q = Q(Q(intnum="80"), Q(Q(id__lt="5"), Q(id__gt="50"), join_type="OR"), join_type="AND")
176180
r = q.resolve(self.int_fields_context)
177-
self.assertEqual(r.where_criterion.get_sql(), '"intnum"=80 AND ("id"<5 OR "id">50)')
181+
self.assertEqual(
182+
r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"intnum"=80 AND ("id"<5 OR "id">50)'
183+
)
178184

179185
def test_q_complex_int3(self):
180186
q = Q(Q(id__lt=5, id__gt=50, join_type="OR"), join_type="AND", intnum=80)
181187
r = q.resolve(self.int_fields_context)
182-
self.assertEqual(r.where_criterion.get_sql(), '"intnum"=80 AND ("id"<5 OR "id">50)')
188+
self.assertEqual(
189+
r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"intnum"=80 AND ("id"<5 OR "id">50)'
190+
)
183191

184192
def test_q_complex_char(self):
185193
q = Q(Q(char_null=80), ~Q(char__lt=5, char__gt=50, join_type="OR"), join_type="AND")
186194
r = q.resolve(self.char_fields_context)
187195
self.assertEqual(
188-
r.where_criterion.get_sql(),
196+
r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT),
189197
"\"char_null\"='80' AND NOT (\"char\"<'5' OR \"char\">'50')",
190198
)
191199

@@ -197,47 +205,47 @@ def test_q_complex_char2(self):
197205
)
198206
r = q.resolve(self.char_fields_context)
199207
self.assertEqual(
200-
r.where_criterion.get_sql(),
208+
r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT),
201209
"\"char_null\"='80' AND NOT (\"char\"<'5' OR \"char\">'50')",
202210
)
203211

204212
def test_q_complex_char3(self):
205213
q = Q(~Q(char__lt=5, char__gt=50, join_type="OR"), join_type="AND", char_null=80)
206214
r = q.resolve(self.char_fields_context)
207215
self.assertEqual(
208-
r.where_criterion.get_sql(),
216+
r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT),
209217
"\"char_null\"='80' AND NOT (\"char\"<'5' OR \"char\">'50')",
210218
)
211219

212220
def test_q_with_blank_and(self):
213221
q = Q(Q(id__gt=5), Q(), join_type=Q.AND)
214222
r = q.resolve(self.char_fields_context)
215-
self.assertEqual(r.where_criterion.get_sql(), '"id">5')
223+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">5')
216224

217225
def test_q_with_blank_or(self):
218226
q = Q(Q(id__gt=5), Q(), join_type=Q.OR)
219227
r = q.resolve(self.char_fields_context)
220-
self.assertEqual(r.where_criterion.get_sql(), '"id">5')
228+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">5')
221229

222230
def test_q_with_blank_and2(self):
223231
q = Q(id__gt=5) & Q()
224232
r = q.resolve(self.char_fields_context)
225-
self.assertEqual(r.where_criterion.get_sql(), '"id">5')
233+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">5')
226234

227235
def test_q_with_blank_or2(self):
228236
q = Q(id__gt=5) | Q()
229237
r = q.resolve(self.char_fields_context)
230-
self.assertEqual(r.where_criterion.get_sql(), '"id">5')
238+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">5')
231239

232240
def test_q_with_blank_and3(self):
233241
q = Q() & Q(id__gt=5)
234242
r = q.resolve(self.char_fields_context)
235-
self.assertEqual(r.where_criterion.get_sql(), '"id">5')
243+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">5')
236244

237245
def test_q_with_blank_or3(self):
238246
q = Q() | Q(id__gt=5)
239247
r = q.resolve(self.char_fields_context)
240-
self.assertEqual(r.where_criterion.get_sql(), '"id">5')
248+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">5')
241249

242250
def test_annotations_resolved(self):
243251
q = Q(id__gt=5) | Q(annotated__lt=5)
@@ -255,4 +263,4 @@ def test_annotations_resolved(self):
255263
},
256264
)
257265
)
258-
self.assertEqual(r.where_criterion.get_sql(), '"id">5 OR "intnum"<5')
266+
self.assertEqual(r.where_criterion.get_sql(DEFAULT_SQL_CONTEXT), '"id">5 OR "intnum"<5')

‎tortoise/backends/mysql/executor.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import enum
22

3-
from pypika_tortoise import functions
3+
from pypika_tortoise import functions, SqlContext
44
from pypika_tortoise.enums import SqlTypes
55
from pypika_tortoise.functions import Cast, Coalesce
66
from pypika_tortoise.terms import BasicCriterion, Criterion
@@ -43,8 +43,8 @@ class StrWrapper(ValueWrapper):
4343
Naive str wrapper that doesn't use the monkey-patched pypika ValueWrapper for MySQL
4444
"""
4545

46-
def get_value_sql(self, **kwargs) -> str:
47-
quote_char = kwargs.get("secondary_quote_char") or ""
46+
def get_value_sql(self, ctx: SqlContext) -> str:
47+
quote_char = ctx.secondary_quote_char or ""
4848
value = self.value.replace(quote_char, quote_char * 2)
4949
return format_quotes(value, quote_char)
5050

‎tortoise/backends/psycopg/client.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from __future__ import annotations
2+
13
import asyncio
24
import typing
35
from contextlib import _AsyncGeneratorContextManager
@@ -8,6 +10,7 @@
810
import psycopg.pq
911
import psycopg.rows
1012
import psycopg_pool
13+
from pypika_tortoise import SqlContext
1114
from pypika_tortoise.dialects.postgresql import PostgreSQLQuery, PostgreSQLQueryBuilder
1215
from pypika_tortoise.terms import Parameterizer
1316

@@ -41,11 +44,12 @@ class PsycopgSQLQueryBuilder(PostgreSQLQueryBuilder):
4144
Psycopg opted to use a custom parameter placeholder, so we need to override the default
4245
"""
4346

44-
def get_parameterized_sql(self, **kwargs) -> typing.Tuple[str, list]:
45-
parameterizer = kwargs.pop(
46-
"parameterizer", Parameterizer(placeholder_factory=lambda _: "%s")
47-
)
48-
return super().get_parameterized_sql(parameterizer=parameterizer, **kwargs)
47+
def get_parameterized_sql(self, ctx: SqlContext | None = None) -> typing.Tuple[str, list]:
48+
if not ctx:
49+
ctx = self.QUERY_CLS.SQL_CONTEXT
50+
if not ctx.parameterizer:
51+
ctx = ctx.copy(parameterizer=Parameterizer(placeholder_factory=lambda _: "%s"))
52+
return super().get_parameterized_sql(ctx)
4953

5054

5155
class PsycopgClient(postgres_client.BasePostgresClient):

‎tortoise/contrib/mysql/search.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from enum import Enum
22
from typing import Any, Optional
33

4+
from pypika_tortoise import SqlContext
45
from pypika_tortoise.enums import Comparator
56
from pypika_tortoise.terms import BasicCriterion
67
from pypika_tortoise.terms import Function as PypikaFunction
@@ -28,7 +29,7 @@ def __init__(self, expr: Term, mode: Optional[Mode] = None) -> None:
2829
super(Against, self).__init__("AGAINST", expr)
2930
self.mode = mode
3031

31-
def get_special_params_sql(self, **kwargs: Any) -> Any:
32+
def get_special_params_sql(self, ctx: SqlContext) -> Any:
3233
if not self.mode:
3334
return ""
3435
return self.mode.value

‎tortoise/contrib/test/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ async def truncate_all_models() -> None:
104104
# TODO: This is a naive implementation: Will fail to clear M2M and non-cascade foreign keys
105105
for app in Tortoise.apps.values():
106106
for model in app.values():
107-
quote_char = model._meta.db.query_class._builder().QUOTE_CHAR
107+
quote_char = model._meta.db.query_class.SQL_CONTEXT.quote_char
108108
await model._meta.db.execute_script(
109109
f"DELETE FROM {quote_char}{model._meta.db_table}{quote_char}" # nosec
110110
)

‎tortoise/expressions.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
from pypika_tortoise import Case as PypikaCase
1010
from pypika_tortoise import Field as PypikaField
11-
from pypika_tortoise import Table
11+
from pypika_tortoise import SqlContext, Table
1212
from pypika_tortoise.functions import AggregateFunction, DistinctOptionFunction
1313
from pypika_tortoise.terms import ArithmeticExpression, Criterion
1414
from pypika_tortoise.terms import Function as PypikaFunction
@@ -203,10 +203,10 @@ def __init__(self, query: "AwaitableQuery") -> None:
203203
super().__init__()
204204
self.query = query
205205

206-
def get_sql(self, **kwargs: Any) -> str:
206+
def get_sql(self, ctx: SqlContext) -> str:
207207
self.query._choose_db_if_not_chosen()
208208
self.query._make_query()
209-
return self.query.query.get_parameterized_sql(**kwargs)[0]
209+
return self.query.query.get_parameterized_sql(ctx)[0]
210210

211211
def as_(self, alias: str) -> "Selectable": # type: ignore
212212
self.query._choose_db_if_not_chosen()
@@ -219,9 +219,9 @@ def __init__(self, sql: str) -> None:
219219
super().__init__()
220220
self.sql = sql
221221

222-
def get_sql(self, with_alias: bool = False, **kwargs: Any) -> str:
223-
if with_alias:
224-
return format_alias_sql(sql=self.sql, alias=self.alias, **kwargs)
222+
def get_sql(self, ctx: SqlContext) -> str:
223+
if ctx.with_alias:
224+
return format_alias_sql(sql=self.sql, alias=self.alias, ctx=ctx)
225225
return self.sql
226226

227227

‎tortoise/filters.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
TypedDict,
1414
)
1515

16-
from pypika_tortoise import Table
16+
from pypika_tortoise import SqlContext, Table
1717
from pypika_tortoise.enums import DatePart, Matching, SqlTypes
1818
from pypika_tortoise.functions import Cast, Extract, Upper
1919
from pypika_tortoise.terms import (
@@ -43,9 +43,9 @@ def __init__(self, left, right, alias=None, escape=" ESCAPE '\\'") -> None:
4343
super().__init__(Matching.like, left, right, alias=alias)
4444
self.escape = escape
4545

46-
def get_sql(self, quote_char='"', with_alias=False, **kwargs) -> str:
47-
sql = super().get_sql(quote_char=quote_char, with_alias=False, **kwargs) + str(self.escape)
48-
if with_alias and self.alias: # pragma: nocoverage
46+
def get_sql(self, ctx: SqlContext):
47+
sql = super().get_sql(ctx.copy(with_alias=False)) + str(self.escape)
48+
if ctx.with_alias and self.alias: # pragma: nocoverage
4949
return '{sql} "{alias}"'.format(sql=sql, alias=self.alias)
5050
return sql
5151

0 commit comments

Comments
 (0)