Fix Django3.1 related issues

absci · davidjb · commit 205c93b4d15b · 2021-03-03T12:26:17.000+10:00
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -6,7 +6,7 @@ variables:
   agent.preferPowerShellOnContainers: true
 
 schedules:
-- cron: "0 0 * * *"
+- cron: "0 9 * * *"
   displayName: Daily midnight build
   branches:
     include:
@@ -20,12 +20,26 @@ jobs:
 
     strategy:
       matrix:
+        Python 3.8 - Django 3.1:
+          python.version: '3.8'
+          tox.env: 'py38-django31'
+        Python 3.7 - Django 3.1:
+          python.version: '3.7'
+          tox.env: 'py37-django31'
+        Python 3.6 - Django 3.1:
+          python.version: '3.6'
+          tox.env: 'py36-django31'
+
+        Python 3.8 - Django 3.0:
+          python.version: '3.8'
+          tox.env: 'py38-django30'
         Python 3.7 - Django 3.0:
           python.version: '3.7'
           tox.env: 'py37-django30'
         Python 3.6 - Django 3.0:
           python.version: '3.6'
           tox.env: 'py36-django30'
+
         Python 3.7 - Django 2.2:
           python.version: '3.7'
           tox.env: 'py37-django22'
@@ -62,12 +76,26 @@ jobs:
 
     strategy:
       matrix:
+        Python 3.8 - Django 3.1:
+          python.version: '3.8'
+          tox.env: 'py38-django31'
+        Python 3.7 - Django 3.1:
+          python.version: '3.7'
+          tox.env: 'py37-django31'
+        Python 3.6 - Django 3.1:
+          python.version: '3.6'
+          tox.env: 'py36-django31'
+
+        Python 3.8 - Django 3.0:
+          python.version: '3.8'
+          tox.env: 'py38-django30'
         Python 3.7 - Django 3.0:
           python.version: '3.7'
           tox.env: 'py37-django30'
         Python 3.6 - Django 3.0:
           python.version: '3.6'
           tox.env: 'py36-django30'
+
         Python 3.7 - Django 2.2:
           python.version: '3.7'
           tox.env: 'py37-django22'
@@ -87,7 +115,7 @@ jobs:
           curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
           curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
           sudo apt-get update
-          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 g++ unixodbc-dev
+          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 g++ unixodbc-dev libmemcached-dev
         displayName: Install SQL Server
 
       - script: |
diff --git a/mssql/creation.py b/mssql/creation.py
@@ -5,10 +5,17 @@
 import os
 
 from django.db.backends.base.creation import BaseDatabaseCreation
+from django import VERSION as django_version
 
 
 class DatabaseCreation(BaseDatabaseCreation):
 
+    def cursor(self):
+        if django_version >= (3, 1):
+            return self.connection._nodb_cursor()
+
+        return self.connection._nodb_connection.cursor()
+
     def _destroy_test_db(self, test_database_name, verbosity):
         """
         Internal implementation - remove the test db tables.
@@ -17,7 +24,7 @@ def _destroy_test_db(self, test_database_name, verbosity):
         # ourselves. Connect to the previous database (not the test database)
         # to do so, because it's not allowed to delete a database while being
         # connected to it.
-        with self.connection._nodb_connection.cursor() as cursor:
+        with self.cursor() as cursor:
             to_azure_sql_db = self.connection.to_azure_sql_db
             if not to_azure_sql_db:
                 cursor.execute("ALTER DATABASE %s SET SINGLE_USER WITH ROLLBACK IMMEDIATE"
@@ -39,7 +46,7 @@ def enable_clr(self):
         This function will not fail if current user doesn't have
         permissions to enable clr, and clr is already enabled
         """
-        with self._nodb_connection.cursor() as cursor:
+        with self.cursor() as cursor:
             # check whether clr is enabled
             cursor.execute('''
             SELECT value FROM sys.configurations
@@ -89,7 +96,7 @@ def install_regex_clr(self, database_name):
 
         self.enable_clr()
 
-        with self._nodb_connection.cursor() as cursor:
+        with self.self.cursor() as cursor:
             for s in sql:
                 cursor.execute(s)
 
diff --git a/mssql/features.py b/mssql/features.py
@@ -24,8 +24,10 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     requires_literal_defaults = True
     requires_sqlparse_for_splitting = False
     supports_boolean_expr_in_select_clause = False
+    supports_deferrable_unique_constraints = False
     supports_ignore_conflicts = False
     supports_index_on_text_field = False
+    supports_json_field = False
     supports_paramstyle_pyformat = False
     supports_regex_backreferencing = True
     supports_sequence_reset = False
diff --git a/mssql/operations.py b/mssql/operations.py
@@ -7,9 +7,11 @@
 
 from django.conf import settings
 from django.db.backends.base.operations import BaseDatabaseOperations
+from django.db.models.expressions import Exists, ExpressionWrapper, RawSQL
+from django.db.models.sql.where import WhereNode
 from django.utils import timezone
 from django.utils.encoding import force_str
-
+from django import VERSION as django_version
 import pytz
 
 
@@ -315,7 +317,33 @@ def savepoint_rollback_sql(self, sid):
         """
         return "ROLLBACK TRANSACTION %s" % sid
 
-    def sql_flush(self, style, tables, sequences, allow_cascade=False):
+    def _build_sequences(self, sequences, cursor):
+        seqs = []
+        for seq in sequences:
+            cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"]))
+            rowcnt = cursor.fetchone()[0]
+            elem = {}
+            if rowcnt:
+                elem['start_id'] = 0
+            else:
+                elem['start_id'] = 1
+            elem.update(seq)
+            seqs.append(elem)
+        return seqs
+
+    def _sql_flush_new(self, style, tables, *, reset_sequences=False, allow_cascade=False):
+        if reset_sequences:
+            return [
+                sequence
+                for sequence in self.connection.introspection.sequence_list()
+            ]
+
+        return []
+
+    def _sql_flush_old(self, style, tables, sequences, allow_cascade=False):
+        return sequences
+
+    def sql_flush(self, style, tables, *args, **kwargs):
         """
         Returns a list of SQL statements required to remove all data from
         the given database tables (without actually removing the tables
@@ -330,56 +358,50 @@ def sql_flush(self, style, tables, sequences, allow_cascade=False):
         The `allow_cascade` argument determines whether truncation may cascade
         to tables with foreign keys pointing the tables being truncated.
         """
-        if tables:
-            # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY
-            # So must use the much slower DELETE
-            from django.db import connections
-            cursor = connections[self.connection.alias].cursor()
-            # Try to minimize the risks of the braindeaded inconsistency in
-            # DBCC CHEKIDENT(table, RESEED, n) behavior.
-            seqs = []
-            for seq in sequences:
-                cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"]))
-                rowcnt = cursor.fetchone()[0]
-                elem = {}
-                if rowcnt:
-                    elem['start_id'] = 0
-                else:
-                    elem['start_id'] = 1
-                elem.update(seq)
-                seqs.append(elem)
-            COLUMNS = "TABLE_NAME, CONSTRAINT_NAME"
-            WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')"
-            cursor.execute(
-                "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE))
-            fks = cursor.fetchall()
-            sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' %
-                        (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]
-            sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'),
-                                            style.SQL_FIELD(self.quote_name(table))) for table in tables])
-
-            if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014:
-                warnings.warn("Resetting identity columns is not supported "
-                              "on this versios of Azure SQL Database.",
-                              RuntimeWarning)
-            else:
-                # Then reset the counters on each table.
-                sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % (
-                    style.SQL_KEYWORD('DBCC'),
-                    style.SQL_KEYWORD('CHECKIDENT'),
-                    style.SQL_FIELD(self.quote_name(seq["table"])),
-                    style.SQL_KEYWORD('RESEED'),
-                    style.SQL_FIELD('%d' % seq['start_id']),
-                    style.SQL_KEYWORD('WITH'),
-                    style.SQL_KEYWORD('NO_INFOMSGS'),
-                ) for seq in seqs])
-
-            sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' %
-                             (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks])
-            return sql_list
-        else:
+
+        if not tables:
             return []
 
+        if django_version >= (3, 1):
+            sequences = self._sql_flush_new(style, tables, *args, **kwargs)
+        else:
+            sequences = self._sql_flush_old(style, tables, *args, **kwargs)
+
+        from django.db import connections
+        cursor = connections[self.connection.alias].cursor()
+
+        seqs = self._build_sequences(sequences, cursor)
+
+        COLUMNS = "TABLE_NAME, CONSTRAINT_NAME"
+        WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')"
+        cursor.execute(
+            "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE))
+        fks = cursor.fetchall()
+        sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' %
+                    (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]
+        sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'),
+                                        style.SQL_FIELD(self.quote_name(table))) for table in tables])
+
+        if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014:
+            warnings.warn("Resetting identity columns is not supported "
+                          "on this versios of Azure SQL Database.",
+                          RuntimeWarning)
+        else:
+            # Then reset the counters on each table.
+            sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % (
+                style.SQL_KEYWORD('DBCC'),
+                style.SQL_KEYWORD('CHECKIDENT'),
+                style.SQL_FIELD(self.quote_name(seq["table"])),
+                style.SQL_KEYWORD('RESEED'),
+                style.SQL_FIELD('%d' % seq['start_id']),
+                style.SQL_KEYWORD('WITH'),
+                style.SQL_KEYWORD('NO_INFOMSGS'),
+            ) for seq in seqs])
+
+        sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' %
+                         (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks])
+        return sql_list
+
     def start_transaction_sql(self):
         """
         Returns the SQL statement required to start a transaction.
@@ -445,3 +467,16 @@ def time_trunc_sql(self, lookup_type, field_name):
         elif lookup_type == 'second':
             sql = "CONVERT(time, SUBSTRING(CONVERT(varchar, %s, 114), 0, 9))" % field_name
         return sql
+
+    def conditional_expression_supported_in_where_clause(self, expression):
+        """
+        Following "Moved conditional expression wrapping to the Exact lookup" in django 3.1
+        https://github.com/django/django/commit/37e6c5b79bd0529a3c85b8c478e4002fd33a2a1d
+        """
+        if isinstance(expression, (Exists, WhereNode)):
+            return True
+        if isinstance(expression, ExpressionWrapper) and expression.conditional:
+            return self.conditional_expression_supported_in_where_clause(expression.expression)
+        if isinstance(expression, RawSQL) and expression.conditional:
+            return True
+        return False
diff --git a/mssql/schema.py b/mssql/schema.py
@@ -16,6 +16,7 @@
     Statement as DjStatement,
     Table,
 )
+from django import VERSION as django_version
 from django.db.models import Index
 from django.db.models.fields import AutoField, BigAutoField
 from django.db.transaction import TransactionManagementError
@@ -687,7 +688,10 @@ def add_field(self, model, field):
         if self.connection.features.connection_persists_old_columns:
             self.connection.close()
 
-    def _create_unique_sql(self, model, columns, name=None, condition=None):
+    def _create_unique_sql(self, model, columns, name=None, condition=None, deferrable=None):
+        if (deferrable and not self.connection.features.supports_deferrable_unique_constraints):
+            return None
+
         def create_unique_name(*args, **kwargs):
             return self.quote_name(self._create_index_name(*args, **kwargs))
 
@@ -697,20 +701,26 @@ def create_unique_name(*args, **kwargs):
         else:
             name = self.quote_name(name)
         columns = Columns(table, columns, self.quote_name)
+        statement_args = {
+            "deferrable": self._deferrable_constraint_sql(deferrable)
+        } if django_version >= (3, 1) else {}
+
         if condition:
             return Statement(
                 self.sql_create_unique_index,
                 table=table,
                 name=name,
                 columns=columns,
                 condition=' WHERE ' + condition,
+                **statement_args
             ) if self.connection.features.supports_partial_indexes else None
         else:
             return Statement(
                 self.sql_create_unique,
                 table=table,
                 name=name,
                 columns=columns,
+                **statement_args
             )
 
     def _create_index_sql(self, model, fields, *, name=None, suffix='', using='',
diff --git a/testapp/settings.py b/testapp/settings.py
@@ -32,9 +32,8 @@
 
 SECRET_KEY = "django_tests_secret_key"
 
-# Use a fast hasher to speed up tests.
 PASSWORD_HASHERS = [
-    'django.contrib.auth.hashers.MD5PasswordHasher',
+    'django.contrib.auth.hashers.PBKDF2PasswordHasher',
 ]
 
 ENABLE_REGEX_TESTS = False
@@ -151,6 +150,7 @@
                   'expressions.tests.FTimeDeltaTests.test_date_subquery_subtraction',
                   'expressions.tests.FTimeDeltaTests.test_datetime_subquery_subtraction',
                   'expressions.tests.FTimeDeltaTests.test_time_subquery_subtraction',
+                  'expressions.tests.BasicExpressionsTests.test_filtering_on_q_that_is_boolean',
                   'migrations.test_operations.OperationTests.test_alter_field_reloads_state_on_fk_with_to_field_target_type_change',
                   'migrations.test_operations.OperationTests.test_autofield__bigautofield_foreignfield_growth',
                   'migrations.test_operations.OperationTests.test_smallfield_autofield_foreignfield_growth',
@@ -159,7 +159,35 @@
                   'schema.tests.SchemaTests.test_alter_autofield_pk_to_bigautofield_pk_sequence_owner',
                   'schema.tests.SchemaTests.test_alter_autofield_pk_to_smallautofield_pk_sequence_owner',
                   'schema.tests.SchemaTests.test_alter_primary_key_quoted_db_table',
-                  'schema.tests.SchemaTests.test_alter_smallint_pk_to_smallautofield_pk'
+                  'schema.tests.SchemaTests.test_alter_smallint_pk_to_smallautofield_pk',
+
+                  'annotations.tests.NonAggregateAnnotationTestCase.test_combined_expression_annotation_with_aggregation',
+                  'bulk_create.tests.BulkCreateTests.test_bulk_insert_nullable_fields',
+                  'db_functions.comparison.test_cast.CastTests.test_cast_to_integer',
+                  'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_func',
+                  'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_iso_weekday_func',
+                  'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func',
+                  'constraints.tests.CheckConstraintTests.test_database_constraint_expression',
+                  'constraints.tests.CheckConstraintTests.test_database_constraint_expressionwrapper',
+                  'constraints.tests.UniqueConstraintTests.test_database_constraint_with_condition',
+                  'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_iso_weekday_func',
+                  'datetimes.tests.DateTimesTests.test_datetimes_ambiguous_and_invalid_times',
+                  'expressions.tests.ExpressionOperatorTests.test_lefthand_bitwise_xor',
+                  'expressions.tests.ExpressionOperatorTests.test_lefthand_bitwise_xor_null',
+                  'inspectdb.tests.InspectDBTestCase.test_number_field_types',
+                  'inspectdb.tests.InspectDBTestCase.test_json_field',
+                  'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_backend_range_save',
+                  'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_coercing',
+                  'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_documented_range',
+                  'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_types',
+                  'ordering.tests.OrderingTests.test_default_ordering_by_f_expression',
+                  'ordering.tests.OrderingTests.test_order_by_nulls_first',
+                  'ordering.tests.OrderingTests.test_order_by_nulls_last',
+                  'queries.test_bulk_update.BulkUpdateTests.test_json_field',
+                  'queries.test_qs_combinators.QuerySetSetOperationTests.test_ordering_by_f_expression_and_alias',
+                  'queries.test_db_returning.ReturningValuesTests.test_insert_returning_multiple',
+                  'dbshell.tests.DbshellCommandTestCase.test_command_missing',
+                  'schema.tests.SchemaTests.test_char_field_pk_to_auto_field'
                   ]
 
 REGEX_TESTS = ['lookup.tests.LookupTests.test_regex',
diff --git a/tox.ini b/tox.ini
