Merge remote-tracking branch 'upstream/master'

Author: mulhollandms

.github/workflows/main.yml

@@ -41,11 +41,14 @@ jobs:
         tox_env:
           - "py36-django22"
           - "py36-django30"
+          - "py36-django31"
 
           - "py37-django22"
           - "py37-django30"
+          - "py37-django31"
 
           - "py38-django30"
+          - "py38-django31"
 
         include:
           - python: "3.6"
@@ -54,15 +57,24 @@ jobs:
           - python: "3.6"
             tox_env: "py36-django30"
 
+          - python: "3.6"
+            tox_env: "py36-django31"
+
           - python: "3.7"
             tox_env: "py37-django22"
 
           - python: "3.7"
             tox_env: "py37-django30"
 
+          - python: "3.7"
+            tox_env: "py37-django31"
+
           - python: "3.8"
             tox_env: "py38-django30"
 
+          - python: "3.8"
+            tox_env: "py38-django31"
+
 
     steps:
       - uses: actions/checkout@v2

.travis.yml

@@ -41,19 +41,25 @@ matrix:
 
     - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django22 }
     - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django30 }
+    - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django31 }
 
     - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django22 }
     - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django30 }
+    - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django31 }
 
     - { before_install: *linux_before_install, python: "3.8", os: linux, env: TOX_ENV=py38-django30 }
+    - { before_install: *linux_before_install, python: "3.8", os: linux, env: TOX_ENV=py38-django31 }
 
     - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django22 }
     - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django30 }
+    - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django31 }
 
     - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django22 }
     - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django30 }
+    - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django31 }
 
     - { before_install: *win_before_install, language: sh, python: "3.8", os: windows, env: TOX_ENV=py38-django30 }
+    - { before_install: *win_before_install, language: sh, python: "3.8", os: windows, env: TOX_ENV=py38-django31 }
 
 
 

sql_server/pyodbc/base.py

@@ -12,28 +12,28 @@
 except ImportError as e:
     raise ImproperlyConfigured("Error loading pyodbc module: %s" % e)
 
-from django.utils.version import get_version_tuple # noqa
+from django.utils.version import get_version_tuple  # noqa
 
 pyodbc_ver = get_version_tuple(Database.version)
 if pyodbc_ver < (3, 0):
     raise ImproperlyConfigured("pyodbc 3.0 or newer is required; you have %s" % Database.version)
 
-from django.conf import settings # noqa
-from django.db import NotSupportedError # noqa
-from django.db.backends.base.base import BaseDatabaseWrapper # noqa
-from django.utils.encoding import smart_str # noqa
-from django.utils.functional import cached_property # noqa
+from django.conf import settings  # noqa
+from django.db import NotSupportedError  # noqa
+from django.db.backends.base.base import BaseDatabaseWrapper  # noqa
+from django.utils.encoding import smart_str  # noqa
+from django.utils.functional import cached_property  # noqa
 
 if hasattr(settings, 'DATABASE_CONNECTION_POOLING'):
     if not settings.DATABASE_CONNECTION_POOLING:
         Database.pooling = False
 
-from .client import DatabaseClient # noqa
-from .creation import DatabaseCreation # noqa
-from .features import DatabaseFeatures # noqa
-from .introspection import DatabaseIntrospection # noqa
-from .operations import DatabaseOperations # noqa
-from .schema import DatabaseSchemaEditor # noqa
+from .client import DatabaseClient  # noqa
+from .creation import DatabaseCreation  # noqa
+from .features import DatabaseFeatures  # noqa
+from .introspection import DatabaseIntrospection  # noqa
+from .operations import DatabaseOperations  # noqa
+from .schema import DatabaseSchemaEditor  # noqa
 
 EDITION_AZURE_SQL_DB = 5
 
@@ -95,6 +95,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         'TextField': 'nvarchar(max)',
         'TimeField': 'time',
         'UUIDField': 'char(32)',
+        'JSONField': 'nvarchar(max)',
     }
     data_type_check_constraints = {
         'PositiveIntegerField': '[%(column)s] >= 0',

sql_server/pyodbc/creation.py

@@ -1,10 +1,17 @@
 import binascii
 import os
 
+import django
 from django.db.backends.base.creation import BaseDatabaseCreation
 
 
 class DatabaseCreation(BaseDatabaseCreation):
+    @property
+    def cursor(self):
+        if django.VERSION >= (3, 1):
+            return self.connection._nodb_cursor
+
+        return self.connection._nodb_connection.cursor
 
     def _destroy_test_db(self, test_database_name, verbosity):
         """
@@ -14,7 +21,7 @@ def _destroy_test_db(self, test_database_name, verbosity):
         # ourselves. Connect to the previous database (not the test database)
         # to do so, because it's not allowed to delete a database while being
         # connected to it.
-        with self.connection._nodb_connection.cursor() as cursor:
+        with self.cursor() as cursor:
             to_azure_sql_db = self.connection.to_azure_sql_db
             if not to_azure_sql_db:
                 cursor.execute("ALTER DATABASE %s SET SINGLE_USER WITH ROLLBACK IMMEDIATE"
@@ -36,7 +43,7 @@ def enable_clr(self):
         This function will not fail if current user doesn't have
         permissions to enable clr, and clr is already enabled
         """
-        with self._nodb_connection.cursor() as cursor:
+        with self.cursor() as cursor:
             # check whether clr is enabled
             cursor.execute('''
             SELECT value FROM sys.configurations
@@ -86,7 +93,7 @@ def install_regex_clr(self, database_name):
 
         self.enable_clr()
 
-        with self._nodb_connection.cursor() as cursor:
+        with self.cursor() as cursor:
             for s in sql:
                 cursor.execute(s)
 

sql_server/pyodbc/features.py

@@ -22,6 +22,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     requires_literal_defaults = True
     requires_sqlparse_for_splitting = False
     supports_boolean_expr_in_select_clause = False
+    supports_deferrable_unique_constraints = False
     supports_ignore_conflicts = False
     supports_index_on_text_field = False
     supports_paramstyle_pyformat = False
@@ -33,6 +34,8 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     supports_timezones = False
     supports_transactions = True
     uses_savepoints = True
+    supports_order_by_nulls_modifier = False
+    supports_order_by_is_nulls = False
 
     @cached_property
     def has_bulk_insert(self):

sql_server/pyodbc/functions.py

@@ -12,6 +12,34 @@ class TryCast(Cast):
     function = 'TRY_CAST'
 
 
+def sqlserver_as_sql(self, compiler, connection, template=None, **extra_context):
+    template = template or self.template
+    if connection.features.supports_order_by_nulls_modifier:
+        if self.nulls_last:
+            template = '%s NULLS LAST' % template
+        elif self.nulls_first:
+            template = '%s NULLS FIRST' % template
+    else:
+        if self.nulls_last and not (
+            self.descending and connection.features.order_by_nulls_first
+        ) and connection.features.supports_order_by_is_nulls:
+            template = '%%(expression)s IS NULL, %s' % template
+        elif self.nulls_first and not (
+            not self.descending and connection.features.order_by_nulls_first
+        ) and connection.features.supports_order_by_is_nulls:
+            template = '%%(expression)s IS NOT NULL, %s' % template
+    connection.ops.check_expression_support(self)
+    expression_sql, params = compiler.compile(self.expression)
+    placeholders = {
+        'expression': expression_sql,
+        'ordering': 'DESC' if self.descending else 'ASC',
+        **extra_context,
+    }
+    template = template or self.template
+    params *= template.count('%(expression)s')
+    return (template % placeholders).rstrip(), params
+
+
 def sqlserver_atan2(self, compiler, connection, **extra_context):
     return self.as_sql(compiler, connection, function='ATN2', **extra_context)
 
@@ -85,3 +113,4 @@ def sqlserver_orderby(self, compiler, connection):
     Exists.as_microsoft = sqlserver_exists
 
 OrderBy.as_microsoft = sqlserver_orderby
+OrderBy.as_sql = sqlserver_as_sql

sql_server/pyodbc/operations.py

@@ -1,9 +1,13 @@
 import datetime
 import uuid
 import warnings
+import django
 
 from django.conf import settings
 from django.db.backends.base.operations import BaseDatabaseOperations
+from django.db.models import Exists, ExpressionWrapper
+from django.db.models.expressions import RawSQL
+from django.db.models.sql.where import WhereNode
 from django.utils import timezone
 from django.utils.encoding import force_str
 
@@ -310,7 +314,7 @@ def savepoint_rollback_sql(self, sid):
         """
         return "ROLLBACK TRANSACTION %s" % sid
 
-    def sql_flush(self, style, tables, sequences, allow_cascade=False):
+    def sql_flush(self, style, tables, *, reset_sequences=False, allow_cascade=False):
         """
         Returns a list of SQL statements required to remove all data from
         the given database tables (without actually removing the tables
@@ -325,14 +329,21 @@ def sql_flush(self, style, tables, sequences, allow_cascade=False):
         The `allow_cascade` argument determines whether truncation may cascade
         to tables with foreign keys pointing the tables being truncated.
         """
-        if tables:
-            # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY
-            # So must use the much slower DELETE
-            from django.db import connections
-            cursor = connections[self.connection.alias].cursor()
-            # Try to minimize the risks of the braindeaded inconsistency in
-            # DBCC CHEKIDENT(table, RESEED, n) behavior.
-            seqs = []
+        if not tables:
+            return []
+
+        # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY
+        # So must use the much slower DELETE
+        from django.db import connections
+        cursor = connections[self.connection.alias].cursor()
+        # Try to minimize the risks of the braindeaded inconsistency in
+        # DBCC CHEKIDENT(table, RESEED, n) behavior.
+        seqs = []
+        if reset_sequences:
+            sequences = [
+                sequence
+                for sequence in self.connection.introspection.sequence_list()
+            ]
             for seq in sequences:
                 cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"]))
                 rowcnt = cursor.fetchone()[0]
@@ -343,37 +354,36 @@ def sql_flush(self, style, tables, sequences, allow_cascade=False):
                     elem['start_id'] = 1
                 elem.update(seq)
                 seqs.append(elem)
-            COLUMNS = "TABLE_NAME, CONSTRAINT_NAME"
-            WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')"
-            cursor.execute(
-                "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE))
-            fks = cursor.fetchall()
-            sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' %
-                        (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]
-            sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'),
-                                            style.SQL_FIELD(self.quote_name(table))) for table in tables])
-
-            if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014:
-                warnings.warn("Resetting identity columns is not supported "
-                              "on this versios of Azure SQL Database.",
-                              RuntimeWarning)
-            else:
-                # Then reset the counters on each table.
-                sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % (
-                    style.SQL_KEYWORD('DBCC'),
-                    style.SQL_KEYWORD('CHECKIDENT'),
-                    style.SQL_FIELD(self.quote_name(seq["table"])),
-                    style.SQL_KEYWORD('RESEED'),
-                    style.SQL_FIELD('%d' % seq['start_id']),
-                    style.SQL_KEYWORD('WITH'),
-                    style.SQL_KEYWORD('NO_INFOMSGS'),
-                ) for seq in seqs])
-
-            sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' %
-                             (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks])
-            return sql_list
+
+        COLUMNS = "TABLE_NAME, CONSTRAINT_NAME"
+        WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')"
+        cursor.execute(
+            "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE))
+        fks = cursor.fetchall()
+        sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' %
+                    (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]
+        sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'),
+                                        style.SQL_FIELD(self.quote_name(table))) for table in tables])
+
+        if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014:
+            warnings.warn("Resetting identity columns is not supported "
+                          "on this versios of Azure SQL Database.",
+                          RuntimeWarning)
         else:
-            return []
+            # Then reset the counters on each table.
+            sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % (
+                style.SQL_KEYWORD('DBCC'),
+                style.SQL_KEYWORD('CHECKIDENT'),
+                style.SQL_FIELD(self.quote_name(seq["table"])),
+                style.SQL_KEYWORD('RESEED'),
+                style.SQL_FIELD('%d' % seq['start_id']),
+                style.SQL_KEYWORD('WITH'),
+                style.SQL_KEYWORD('NO_INFOMSGS'),
+            ) for seq in seqs])
+
+        sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' %
+                         (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks])
+        return sql_list
 
     def start_transaction_sql(self):
         """
@@ -440,3 +450,18 @@ def time_trunc_sql(self, lookup_type, field_name):
         elif lookup_type == 'second':
             sql = "CONVERT(time, SUBSTRING(CONVERT(varchar, %s, 114), 0, 9))" % field_name
         return sql
+
+    def conditional_expression_supported_in_where_clause(self, expression):
+        """
+        Following "Moved conditional expression wrapping to the Exact lookup" in django 3.1
+        https://github.com/django/django/commit/37e6c5b79bd0529a3c85b8c478e4002fd33a2a1d
+        """
+        if django.VERSION >= (3, 1):
+            if isinstance(expression, (Exists, WhereNode)):
+                return True
+            if isinstance(expression, ExpressionWrapper) and expression.conditional:
+                return self.conditional_expression_supported_in_where_clause(expression.expression)
+            if isinstance(expression, RawSQL) and expression.conditional:
+                return True
+            return False
+        return True

sql_server/pyodbc/schema.py

@@ -694,13 +694,15 @@ def create_unique_name(*args, **kwargs):
                 name=name,
                 columns=columns,
                 condition=' WHERE ' + condition,
+                deferrable=''
             ) if self.connection.features.supports_partial_indexes else None
         else:
             return Statement(
                 self.sql_create_unique,
                 table=table,
                 name=name,
                 columns=columns,
+                deferrable=''
             )
 
     def _create_index_sql(self, model, fields, *, name=None, suffix='', using='',

tox.ini

@@ -2,6 +2,7 @@
 envlist =
        {py36,py37}-django22,
        {py36,py37,py38}-django30,
+       {py36,py37,py38}-django31,
 
 [testenv]
 passenv =
@@ -19,4 +20,5 @@ commands =
 deps =
     django22: django==2.2.*
     django30: django>=3.0a1,<3.1
+    django31: django>=3.1,<3.2
     dj-database-url==0.5.0