Add TimeZone support (microsoft#140)

* Added timezone support and skipped unit test that uses cursor

* Removed unneeded import

Author: jmah8

mssql/base.py

@@ -8,6 +8,7 @@
 import re
 import time
 import struct
+import datetime
 
 from django.core.exceptions import ImproperlyConfigured
 
@@ -35,7 +36,7 @@
 from .client import DatabaseClient  # noqa
 from .creation import DatabaseCreation  # noqa
 from .features import DatabaseFeatures  # noqa
-from .introspection import DatabaseIntrospection  # noqa
+from .introspection import DatabaseIntrospection, SQL_TIMESTAMP_WITH_TIMEZONE  # noqa
 from .operations import DatabaseOperations  # noqa
 from .schema import DatabaseSchemaEditor  # noqa
 
@@ -80,6 +81,13 @@ def encode_value(v):
     return v
 
 
+def handle_datetimeoffset(dto_value):
+    # Decode bytes returned from SQL Server
+    # source: https://github.com/mkleehammer/pyodbc/wiki/Using-an-Output-Converter-function
+    tup = struct.unpack("<6hI2h", dto_value)  # e.g., (2017, 3, 16, 10, 35, 18, 500000000)
+    return datetime.datetime(tup[0], tup[1], tup[2], tup[3], tup[4], tup[5], tup[6] // 1000)
+
+
 class DatabaseWrapper(BaseDatabaseWrapper):
     vendor = 'microsoft'
     display_name = 'SQL Server'
@@ -95,7 +103,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         'BooleanField': 'bit',
         'CharField': 'nvarchar(%(max_length)s)',
         'DateField': 'date',
-        'DateTimeField': 'datetime2',
+        'DateTimeField': 'datetimeoffset',
         'DecimalField': 'numeric(%(max_digits)s, %(decimal_places)s)',
         'DurationField': 'bigint',
         'FileField': 'nvarchar(%(max_length)s)',
@@ -364,6 +372,9 @@ def get_new_connection(self, conn_params):
                 if not need_to_retry:
                     raise
 
+        # Handling values from DATETIMEOFFSET columns
+        # source: https://github.com/mkleehammer/pyodbc/wiki/Using-an-Output-Converter-function
+        conn.add_output_converter(SQL_TIMESTAMP_WITH_TIMEZONE, handle_datetimeoffset)
         conn.timeout = query_timeout
         if setencoding:
             for entry in setencoding:

mssql/features.py

@@ -46,7 +46,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     supports_subqueries_in_group_by = False
     supports_tablespaces = True
     supports_temporal_subtraction = True
-    supports_timezones = False
+    supports_timezones = True
     supports_transactions = True
     uses_savepoints = True
     has_bulk_insert = True

mssql/introspection.py

@@ -12,6 +12,7 @@
 
 SQL_AUTOFIELD = -777555
 SQL_BIGAUTOFIELD = -777444
+SQL_TIMESTAMP_WITH_TIMEZONE = -155
 
 
 def get_schema_name():
@@ -41,7 +42,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
         Database.SQL_TINYINT: 'SmallIntegerField',
         Database.SQL_TYPE_DATE: 'DateField',
         Database.SQL_TYPE_TIME: 'TimeField',
-        Database.SQL_TYPE_TIMESTAMP: 'DateTimeField',
+        SQL_TIMESTAMP_WITH_TIMEZONE: 'DateTimeField',
         Database.SQL_VARBINARY: 'BinaryField',
         Database.SQL_VARCHAR: 'TextField',
         Database.SQL_WCHAR: 'CharField',

mssql/operations.py

@@ -4,6 +4,7 @@
 import datetime
 import uuid
 import warnings
+import sys
 
 from django.conf import settings
 from django.db.backends.base.operations import BaseDatabaseOperations
@@ -26,7 +27,7 @@ def max_in_list_size(self):
         return 2048
 
     def _convert_field_to_tz(self, field_name, tzname):
-        if settings.USE_TZ and not tzname == 'UTC':
+        if tzname and settings.USE_TZ and self.connection.timezone_name != tzname:
             offset = self._get_utcoffset(tzname)
             field_name = 'DATEADD(second, %d, %s)' % (offset, field_name)
         return field_name
@@ -107,7 +108,7 @@ def combine_expression(self, connector, sub_expressions):
 
     def convert_datetimefield_value(self, value, expression, connection):
         if value is not None:
-            if settings.USE_TZ:
+            if settings.USE_TZ and not timezone.is_aware(value):
                 value = timezone.make_aware(value, self.connection.timezone)
         return value
 
@@ -129,6 +130,8 @@ def date_extract_sql(self, lookup_type, field_name):
             return "DATEPART(weekday, %s)" % field_name
         elif lookup_type == 'week':
             return "DATEPART(iso_week, %s)" % field_name
+        elif lookup_type == 'iso_week_day':
+            return "DATEPART(weekday, DATEADD(day, -1, %s))" % field_name
         elif lookup_type == 'iso_year':
             return "YEAR(DATEADD(day, 26 - DATEPART(isoww, %s), %s))" % (field_name, field_name)
         else:
@@ -144,7 +147,8 @@ def date_interval_sql(self, timedelta):
             sql = 'DATEADD(microsecond, %d%%s, CAST(%s AS datetime2))' % (timedelta.microseconds, sql)
         return sql
 
-    def date_trunc_sql(self, lookup_type, field_name, tzname=''):
+    def date_trunc_sql(self, lookup_type, field_name, tzname=None):
+        field_name = self._convert_field_to_tz(field_name, tzname)
         CONVERT_YEAR = 'CONVERT(varchar, DATEPART(year, %s))' % field_name
         CONVERT_QUARTER = 'CONVERT(varchar, 1+((DATEPART(quarter, %s)-1)*3))' % field_name
         CONVERT_MONTH = 'CONVERT(varchar, DATEPART(month, %s))' % field_name
@@ -480,9 +484,22 @@ def adapt_datetimefield_value(self, value):
         """
         if value is None:
             return None
-        if settings.USE_TZ and timezone.is_aware(value):
-            # pyodbc donesn't support datetimeoffset
-            value = value.astimezone(self.connection.timezone).replace(tzinfo=None)
+
+        # Expression values are adapted by the database.
+        if hasattr(value, 'resolve_expression'):
+            return value
+
+        if timezone.is_aware(value):
+            if settings.USE_TZ:
+                # When support for time zones is enabled, Django stores datetime information
+                # in UTC in the database and uses time-zone-aware objects internally
+                # source: https://docs.djangoproject.com/en/dev/topics/i18n/timezones/#overview
+                value = value.astimezone(timezone.utc)
+            else:
+                # When USE_TZ is False, settings.TIME_ZONE is the time zone in
+                # which Django will store all datetimes
+                # source: https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-TIME_ZONE
+                value = timezone.make_naive(value, self.connection.timezone)
         return value
 
     def time_trunc_sql(self, lookup_type, field_name, tzname=''):

test.sh

@@ -105,6 +105,7 @@ coverage run tests/runtests.py --settings=testapp.settings --noinput \
     select_related_onetoone \
     select_related_regress \
     serializers \
+    timezones \
     transaction_hooks \
     transactions \
     update \

testapp/migrations/0020_timezone.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.0.4 on 2022-05-18 20:55
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('testapp', '0019_customer_name_customer_address'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TimeZone',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('date', models.DateTimeField(default=django.utils.timezone.now)),
+            ],
+        ),
+    ]

testapp/models.py

@@ -219,3 +219,6 @@ class Customer_address(models.Model):
     Customer_address = models.CharField(max_length=100)
     class Meta:
         ordering = ['Customer_address']
+
+class TimeZone(models.Model):
+    date = models.DateTimeField(default=timezone.now)

testapp/settings.py

@@ -272,7 +272,19 @@
     'backends.tests.BackendTestCase.test_queries_logger',
     'migrations.test_operations.OperationTests.test_alter_field_pk_mti_fk',
     'migrations.test_operations.OperationTests.test_run_sql_add_missing_semicolon_on_collect_sql',
-    'migrations.test_operations.OperationTests.test_alter_field_pk_mti_and_fk_to_base'
+    'migrations.test_operations.OperationTests.test_alter_field_pk_mti_and_fk_to_base',
+
+    # Timezone
+    'timezones.tests.NewDatabaseTests.test_cursor_explicit_time_zone',
+    # Skipped next tests because pyodbc drops timezone https://github.com/mkleehammer/pyodbc/issues/810
+    #LegacyDatabaseTests
+    'timezones.tests.LegacyDatabaseTests.test_cursor_execute_accepts_naive_datetime',
+    'timezones.tests.LegacyDatabaseTests.test_cursor_execute_returns_naive_datetime',
+    # NewDatabaseTests
+    'timezones.tests.NewDatabaseTests.test_cursor_execute_accepts_naive_datetime',
+    'timezones.tests.NewDatabaseTests.test_cursor_execute_returns_naive_datetime',
+    'timezones.tests.NewDatabaseTests.test_cursor_execute_accepts_aware_datetime',
+    'timezones.tests.NewDatabaseTests.test_cursor_execute_returns_aware_datetime',
 ]
 
 REGEX_TESTS = [

testapp/tests/test_timezones.py

@@ -0,0 +1,23 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the BSD license.
+
+import datetime
+
+from django.test import TestCase
+
+from ..models import TimeZone
+
+class TestDateTimeField(TestCase):
+
+    def test_iso_week_day(self):
+        days = {
+            1: TimeZone.objects.create(date=datetime.datetime(2022, 5, 16)),
+            2: TimeZone.objects.create(date=datetime.datetime(2022, 5, 17)),
+            3: TimeZone.objects.create(date=datetime.datetime(2022, 5, 18)),
+            4: TimeZone.objects.create(date=datetime.datetime(2022, 5, 19)),
+            5: TimeZone.objects.create(date=datetime.datetime(2022, 5, 20)),
+            6: TimeZone.objects.create(date=datetime.datetime(2022, 5, 21)),
+            7: TimeZone.objects.create(date=datetime.datetime(2022, 5, 22)),
+        }
+        for k, v in days.items():
+            self.assertSequenceEqual(TimeZone.objects.filter(date__iso_week_day=k), [v])