Skip to content

Commit 4f91ca4

Browse files
michiyamichiya
committed
fix issue avidal#3 and add support for new offsetting syntax
1 parent 5a9ee4e commit 4f91ca4

File tree

1 file changed

+101
-126
lines changed

1 file changed

+101
-126
lines changed

‎sql_server/pyodbc/compiler.py‎

Lines changed: 101 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -3,104 +3,92 @@
33

44
from sql_server.pyodbc.compat import zip_longest
55

6-
REV_ODIR = {
7-
'ASC': 'DESC',
8-
'DESC': 'ASC'
9-
}
10-
11-
# Strategies for handling limit+offset emulation:
12-
USE_ROW_NUMBER = 0 # For SQL Server >= 2005
13-
146

157
class SQLCompiler(compiler.SQLCompiler):
16-
8+
179
def resolve_columns(self, row, fields=()):
1810
index_start = len(list(self.query.extra_select.keys()))
1911
values = [self.query.convert_values(v, None, connection=self.connection) for v in row[:index_start]]
2012
for value, field in zip_longest(row[index_start:], fields):
21-
values.append(self.query.convert_values(value, field, connection=self.connection))
13+
if field:
14+
value = self.query.convert_values(value, field, connection=self.connection)
15+
values.append(value)
2216
return tuple(values)
2317

24-
def modify_query(self, strategy, ordering, out_cols):
25-
"""
26-
Helper method, called from _as_sql()
27-
28-
Sets the value of the self._ord and self.default_reverse_ordering
29-
attributes.
30-
Can modify the values of the out_cols list argument and the
31-
self.query.ordering_aliases attribute.
32-
"""
33-
self.default_reverse_ordering = False
34-
self._ord = []
35-
cnt = 0
36-
extra_select_aliases = [k.strip('[]') for k in self.query.extra_select.keys()]
37-
for ord_spec_item in ordering:
38-
if ord_spec_item.endswith(' ASC') or ord_spec_item.endswith(' DESC'):
39-
parts = ord_spec_item.split()
40-
col, odir = ' '.join(parts[:-1]), parts[-1]
41-
if col not in self.query.ordering_aliases and col.strip('[]') not in extra_select_aliases:
42-
if col.isdigit():
43-
cnt += 1
44-
n = int(col)-1
45-
alias = 'OrdAlias%d' % cnt
46-
# ordering by aliases defined in the same query is not available ...
47-
self._ord.append((out_cols[n], odir))
48-
out_cols[n] = '%s AS [%s]' % (out_cols[n], alias)
49-
else:
50-
self._ord.append((col, odir))
51-
else:
52-
self._ord.append((col, odir))
53-
54-
if not self._ord and 'RAND()' in ordering:
55-
self._ord.append(('RAND()',''))
56-
57-
def _as_sql(self, strategy):
18+
def as_sql(self, with_limits=True, with_col_aliases=False):
5819
"""
59-
Helper method, called from as_sql()
60-
Similar to django/db/models/sql/query.py:Query.as_sql() but without
61-
the ordering and limits code.
20+
Creates the SQL for this query. Returns the SQL string and list of
21+
parameters.
6222
63-
Returns SQL that hasn't an order-by clause.
23+
If 'with_limits' is False, any limit/offset information is not included
24+
in the query.
6425
"""
65-
# get_columns needs to be called before get_ordering to populate
66-
# _select_alias.
67-
out_cols = self.get_columns(True)
68-
ordering, ordering_group_by = self.get_ordering()
69-
if not ordering:
70-
meta = self.query.get_meta()
71-
qn = self.quote_name_unless_alias
72-
# Special case: pk not in out_cols, use random ordering.
73-
#
74-
if '%s.%s' % (qn(meta.db_table), qn(meta.pk.db_column or meta.pk.column)) not in self.get_columns():
75-
ordering = ['RAND()']
76-
# XXX: Maybe use group_by field for ordering?
77-
#if self.group_by:
78-
#ordering = ['%s.%s ASC' % (qn(self.group_by[0][0]),qn(self.group_by[0][1]))]
79-
else:
80-
ordering = ['%s.%s ASC' % (qn(meta.db_table), qn(meta.pk.db_column or meta.pk.column))]
26+
if with_limits and self.query.low_mark == self.query.high_mark:
27+
return '', ()
8128

82-
self.modify_query(strategy, ordering, out_cols)
29+
self.pre_sql_setup()
8330

84-
order = ', '.join(['%s %s' % pair for pair in self._ord])
85-
self.query.ordering_aliases.append('(ROW_NUMBER() OVER (ORDER BY %s)) AS [rn]' % order)
31+
# The do_offset flag indicates whether we need to construct
32+
# the SQL needed to use limit/offset w/SQL Server.
33+
high_mark = self.query.high_mark
34+
low_mark = self.query.low_mark
35+
do_limit = with_limits and high_mark is not None
36+
do_offset = with_limits and low_mark != 0
37+
# SQL Server 2012 or newer supports OFFSET/FETCH clause
38+
supports_offset_clause = self.connection.ops.sql_server_ver >= 2012
39+
40+
# After executing the query, we must get rid of any joins the query
41+
# setup created. So, take note of alias counts before the query ran.
42+
# However we do not want to get rid of stuff done in pre_sql_setup(),
43+
# as the pre_sql_setup will modify query state in a way that forbids
44+
# another run of it.
45+
if self.connection._DJANGO_VERSION >= 14:
46+
self.refcounts_before = self.query.alias_refcount.copy()
47+
out_cols = self.get_columns(with_col_aliases)
48+
ordering, ordering_group_by, offset_params = \
49+
self._get_ordering(out_cols, supports_offset_clause or not do_offset)
8650

8751
# This must come after 'select' and 'ordering' -- see docstring of
8852
# get_from_clause() for details.
8953
from_, f_params = self.get_from_clause()
9054

9155
qn = self.quote_name_unless_alias
92-
where, w_params = self.query.where.as_sql(qn, self.connection)
93-
having, h_params = self.query.having.as_sql(qn, self.connection)
56+
57+
where, w_params = self.query.where.as_sql(qn=qn, connection=self.connection)
58+
having, h_params = self.query.having.as_sql(qn=qn, connection=self.connection)
9459
params = []
9560
for val in self.query.extra_select.values():
9661
params.extend(val[1])
9762

9863
result = ['SELECT']
64+
9965
if self.query.distinct:
10066
result.append('DISTINCT')
10167

68+
if do_offset:
69+
if not ordering:
70+
meta = self.query.get_meta()
71+
qn = self.quote_name_unless_alias
72+
pk = '%s.%s' % (qn(meta.db_table), qn(meta.pk.db_column or meta.pk.column))
73+
# Special case: pk not in out_cols, use random ordering.
74+
if not pk in out_cols:
75+
ordering = [self.connection.ops.random_function_sql()]
76+
# XXX: Maybe use group_by field for ordering?
77+
#if self.group_by:
78+
#ordering = ['%s.%s ASC' % (qn(self.group_by[0][0]),qn(self.group_by[0][1]))]
79+
else:
80+
ordering = ['%s ASC' % pk]
81+
if not supports_offset_clause:
82+
order = ', '.join(ordering)
83+
self.query.ordering_aliases.append('(ROW_NUMBER() OVER (ORDER BY %s)) AS [rn]' % order)
84+
ordering = self.connection.ops.force_no_ordering()
85+
elif do_limit:
86+
result.append('TOP %d' % high_mark)
87+
10288
result.append(', '.join(out_cols + self.query.ordering_aliases))
10389

90+
params.extend(offset_params)
91+
10492
result.append('FROM')
10593
result.extend(from_)
10694
params.extend(f_params)
@@ -113,8 +101,7 @@ def _as_sql(self, strategy):
113101
grouping, gb_params = self.get_grouping(ordering_group_by)
114102
else:
115103
grouping, gb_params = self.get_grouping()
116-
if grouping:
117-
if ordering:
104+
if grouping and ordering:
118105
# If the backend can't group by PK (i.e., any database
119106
# other than MySQL), then any fields mentioned in the
120107
# ordering clause needs to be in the group by clause.
@@ -123,7 +110,8 @@ def _as_sql(self, strategy):
123110
if col not in grouping:
124111
grouping.append(str(col))
125112
gb_params.extend(col_params)
126-
else:
113+
if grouping:
114+
if not ordering:
127115
ordering = self.connection.ops.force_no_ordering()
128116
result.append('GROUP BY %s' % ', '.join(grouping))
129117
params.extend(gb_params)
@@ -132,67 +120,54 @@ def _as_sql(self, strategy):
132120
result.append('HAVING %s' % having)
133121
params.extend(h_params)
134122

135-
return ' '.join(result), tuple(params)
136-
137-
def as_sql(self, with_limits=True, with_col_aliases=False):
138-
"""
139-
Creates the SQL for this query. Returns the SQL string and list of
140-
parameters.
141-
142-
If 'with_limits' is False, any limit/offset information is not included
143-
in the query.
144-
"""
145-
if with_limits and self.query.low_mark == self.query.high_mark:
146-
return '', ()
147-
148-
# The do_offset flag indicates whether we need to construct
149-
# the SQL needed to use limit/offset w/SQL Server.
150-
do_offset = with_limits and (self.query.high_mark is not None or self.query.low_mark != 0)
151-
152-
# no row ordering or row offsetting is assumed to be required
153-
# if the result type is specified as SINGLE
154-
if hasattr(self, 'result_type') and self.result_type == SINGLE:
155-
do_offset = False
156-
157-
# If no offsets, just return the result of the base class
158-
# `as_sql`.
159-
if not do_offset:
160-
return super(SQLCompiler, self).as_sql(with_limits=False,
161-
with_col_aliases=with_col_aliases)
162-
# Shortcut for the corner case when high_mark value is 0:
163-
if self.query.high_mark == 0:
164-
return "", ()
165-
166-
self.pre_sql_setup()
167-
168-
# SQL Server 2005 or newer
169-
sql, params = self._as_sql(USE_ROW_NUMBER)
170-
171-
# Construct the final SQL clause, using the initial select SQL
172-
# obtained above.
173-
result = ['SELECT * FROM (%s) AS X' % sql]
123+
if ordering and not with_col_aliases:
124+
result.append('ORDER BY %s' % ', '.join(ordering))
125+
if do_offset and supports_offset_clause:
126+
result.append('OFFSET %d ROWS' % low_mark)
127+
if do_limit:
128+
result.append('FETCH FIRST %d ROWS ONLY' % (high_mark - low_mark))
129+
130+
if do_offset and not supports_offset_clause:
131+
# Construct the final SQL clause, using the initial select SQL
132+
# obtained above.
133+
result = ['SELECT * FROM (%s) AS X WHERE X.rn' % ' '.join(result)]
134+
# Place WHERE condition on `rn` for the desired range.
135+
if do_limit:
136+
result.append('BETWEEN %d AND %d' % (low_mark+1, high_mark))
137+
else:
138+
result.append('>= %d' % (low_mark+1))
139+
result.append('ORDER BY X.rn')
174140

175-
# Place WHERE condition on `rn` for the desired range.
176-
if self.query.high_mark is None:
177-
self.query.high_mark = 9223372036854775807
178-
result.append('WHERE X.rn BETWEEN %d AND %d' % (self.query.low_mark+1, self.query.high_mark))
141+
# Finally do cleanup - get rid of the joins we created above.
142+
if self.connection._DJANGO_VERSION >= 14:
143+
self.query.reset_refcounts(self.refcounts_before)
179144

180-
return ' '.join(result), params
145+
return ' '.join(result), tuple(params)
181146

182-
def get_ordering(self):
147+
def _get_ordering(self, out_cols, allow_aliases=True):
183148
# SQL Server doesn't support grouping by column number
184-
ordering, ordering_group_by = super(SQLCompiler, self).get_ordering()
149+
ordering, ordering_group_by = self.get_ordering()
185150
grouping = []
186-
for t in ordering_group_by:
151+
for group_by in ordering_group_by:
187152
try:
188-
int(t[0])
153+
col_index = int(group_by[0]) - 1
154+
grouping.append((out_cols[col_index], group_by[1]))
189155
except ValueError:
190-
grouping.append(t)
191-
return ordering, grouping
192-
193-
def execute_sql(self, result_type=MULTI):
194-
self.result_type = result_type
195-
return super(SQLCompiler, self).execute_sql(result_type)
156+
grouping.append(group_by)
157+
# value_expression in OVER clause cannot refer to
158+
# expressions or aliases in the select list. See:
159+
# http://msdn.microsoft.com/en-us/library/ms189461.aspx
160+
offset_params = []
161+
if not allow_aliases:
162+
keys = self.query.extra.keys()
163+
for i in range(len(ordering)):
164+
order_col, order_dir = ordering[i].split()
165+
order_col = order_col.strip('[]')
166+
if order_col in keys:
167+
ex = self.query.extra[order_col]
168+
ordering[i] = '%s %s' % (ex[0], order_dir)
169+
offset_params.extend(ex[1])
170+
return ordering, grouping, offset_params
196171

197172
class SQLInsertCompiler(compiler.SQLInsertCompiler, SQLCompiler):
198173

0 commit comments

Comments
 (0)