diff --git a/django_mongodb_backend/base.py b/django_mongodb_backend/base.py index abfed412..a70c7fbd 100644 --- a/django_mongodb_backend/base.py +++ b/django_mongodb_backend/base.py @@ -40,8 +40,8 @@ def __exit__(self, exception_type, exception_value, exception_traceback): class DatabaseWrapper(BaseDatabaseWrapper): data_types = { - "AutoField": "int", - "BigAutoField": "long", + "AutoField": "", # Not supported + "BigAutoField": "", # Not supported "BinaryField": "binData", "BooleanField": "bool", "CharField": "string", @@ -52,16 +52,15 @@ class DatabaseWrapper(BaseDatabaseWrapper): "FileField": "string", "FilePathField": "string", "FloatField": "double", - "IntegerField": "int", + "IntegerField": "long", "BigIntegerField": "long", "GenericIPAddressField": "string", "JSONField": "object", - "OneToOneField": "int", - "PositiveBigIntegerField": "int", + "PositiveBigIntegerField": "long", "PositiveIntegerField": "long", "PositiveSmallIntegerField": "int", "SlugField": "string", - "SmallAutoField": "int", + "SmallAutoField": "", # Not supported "SmallIntegerField": "int", "TextField": "string", "TimeField": "date", diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index eb098a78..7a27e168 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -94,6 +94,8 @@ class DatabaseFeatures(GISFeatures, BaseDatabaseFeatures): # Value.as_mql() doesn't call output_field.get_db_prep_save(): # https://github.com/mongodb/django-mongodb-backend/issues/282 "model_fields.test_jsonfield.TestSaveLoad.test_bulk_update_custom_get_prep_value", + # To debug: https://github.com/mongodb/django-mongodb-backend/issues/362 + "constraints.tests.UniqueConstraintTests.test_validate_case_when", } # $bitAnd, #bitOr, and $bitXor are new in MongoDB 6.3. _django_test_expected_failures_bitwise = { diff --git a/django_mongodb_backend/fields/duration.py b/django_mongodb_backend/fields/duration.py index cd0fd551..f6ec1873 100644 --- a/django_mongodb_backend/fields/duration.py +++ b/django_mongodb_backend/fields/duration.py @@ -1,3 +1,4 @@ +from bson import Int64 from django.db.models.fields import DurationField _get_db_prep_value = DurationField.get_db_prep_value @@ -8,6 +9,8 @@ def get_db_prep_value(self, value, connection, prepared=False): value = _get_db_prep_value(self, value, connection, prepared) if connection.vendor == "mongodb" and value is not None: value //= 1000 + # Store value as Int64 (long). + value = Int64(value) return value diff --git a/django_mongodb_backend/operations.py b/django_mongodb_backend/operations.py index 435c5bb8..4b494c3d 100644 --- a/django_mongodb_backend/operations.py +++ b/django_mongodb_backend/operations.py @@ -4,7 +4,7 @@ import uuid from decimal import Decimal -from bson.decimal128 import Decimal128 +from bson import Decimal128, Int64 from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.db import DataError @@ -66,6 +66,14 @@ def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None): return None return Decimal128(value) + def adapt_integerfield_value(self, value, internal_type): + """Store non-SmallIntegerField variants as Int64 (long).""" + if value is None: + return None + if "Small" not in internal_type: + return Int64(value) + return value + def adapt_json_value(self, value, encoder): if encoder is None: return value @@ -131,8 +139,16 @@ def get_db_converters(self, expression): converters.append(self.convert_timefield_value) elif internal_type == "UUIDField": converters.append(self.convert_uuidfield_value) + elif "IntegerField" in internal_type and "Small" not in internal_type: + converters.append(self.convert_integerfield_value) return converters + def convert_integerfield_value(self, value, expression, connection): + if value is not None: + # from Int64 to int + value = int(value) + return value + def convert_datefield_value(self, value, expression, connection): if value is not None: value = value.date() diff --git a/docs/source/ref/models/fields.rst b/docs/source/ref/models/fields.rst index eca30425..874ca258 100644 --- a/docs/source/ref/models/fields.rst +++ b/docs/source/ref/models/fields.rst @@ -27,6 +27,17 @@ A few notes about some of the other fields: prohibit inserting values outside of the supported range and unique constraints don't work for values outside of the 32-bit range of the BSON ``int`` type. +- :class:`~django.db.models.IntegerField`, + :class:`~django.db.models.BigIntegerField` and + :class:`~django.db.models.PositiveSmallIntegerField`, and + :class:`~django.db.models.PositiveBigIntegerField` support 64 bit values + (ranges ``(-9223372036854775808, 9223372036854775807)`` and + ``(0, 9223372036854775807)``, respectively), validated by forms and model + validation. If you're inserting data outside of the ORM, you must cast all + values to :class:`bson.int64.Int64`, otherwise values less then 32 bits will + be stored as ``int`` and won't be validated by unique constraints. +- Similarly, all :class:`~django.db.models.DurationField` values are stored as + :class:`bson.int64.Int64`. MongoDB-specific model fields ============================= diff --git a/docs/source/releases/5.2.x.rst b/docs/source/releases/5.2.x.rst index 21b8a93e..058b660f 100644 --- a/docs/source/releases/5.2.x.rst +++ b/docs/source/releases/5.2.x.rst @@ -37,6 +37,17 @@ Bug fixes databases. - :meth:`QuerySet.explain() ` now :ref:`returns a string that can be parsed as JSON `. +- Fixed unique constraint generation for :class:`~django.db.models.IntegerField` + and :class:`~django.db.models.PositiveBigIntegerField`, which incorrectly + allowed duplicate values larger than 32 bits. Existing unique constraints + must be recreated to use ``$type: long`` instead of ``int``. +- :class:`~django.db.models.IntegerField`, + :class:`~django.db.models.BigIntegerField` (as well as the + ``Positive`` versions of these fields), and + :class:`~django.db.models.DurationField` values are now sent to MongoDB as + :class:`bson.int64.Int64`, which fixes unique constraints on values less than + 32 bits for ``BigIntegerField``, ``PositiveIntegerField``, and + ``DurationField``. Existing data must be converted to ``Int64``. Performance improvements ------------------------ diff --git a/tests/model_fields_/models.py b/tests/model_fields_/models.py index 0a001067..21e0af24 100644 --- a/tests/model_fields_/models.py +++ b/tests/model_fields_/models.py @@ -15,7 +15,12 @@ class UniqueIntegers(models.Model): small = models.SmallIntegerField(unique=True, blank=True, null=True) + plain = models.IntegerField(unique=True, blank=True, null=True) + big = models.BigIntegerField(unique=True, blank=True, null=True) positive_small = models.PositiveSmallIntegerField(unique=True, blank=True, null=True) + positive = models.PositiveIntegerField(unique=True, blank=True, null=True) + positive_big = models.PositiveBigIntegerField(unique=True, blank=True, null=True) + duration = models.DurationField(unique=True, blank=True, null=True) # ObjectIdField diff --git a/tests/model_fields_/test_durationfield.py b/tests/model_fields_/test_durationfield.py new file mode 100644 index 00000000..aefddeb9 --- /dev/null +++ b/tests/model_fields_/test_durationfield.py @@ -0,0 +1,27 @@ +from datetime import timedelta + +from django.db import IntegrityError +from django.test import TestCase + +from .models import UniqueIntegers + + +class UniqueTests(TestCase): + def test_small_value(self): + """ + Duplicate values < 32 bits are prohibited. This confirms DurationField + values are cast to Int64 so MongoDB stores them as long. Otherwise, the + partialFilterExpression: {$type: long} unique constraint doesn't work. + """ + UniqueIntegers.objects.create(duration=timedelta(1)) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(duration=timedelta(1)) + + def test_large_value(self): + """ + Duplicate values > 32 bits are prohibited. This confirms DurationField + uses the long db_type() rather than the 32 bit int type. + """ + UniqueIntegers.objects.create(duration=timedelta(1000000)) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(duration=timedelta(1000000)) diff --git a/tests/model_fields_/test_integerfield.py b/tests/model_fields_/test_integerfield.py index bd3ec491..e7b26b7e 100644 --- a/tests/model_fields_/test_integerfield.py +++ b/tests/model_fields_/test_integerfield.py @@ -73,3 +73,62 @@ def test_validate_min_value(self): msg = "{'positive_small': ['Ensure this value is greater than or equal to 0.']" with self.assertRaisesMessage(ValidationError, msg): UniqueIntegers(positive_small=self.min_value - 1).full_clean() + + +class SmallUniqueTests(TestCase): + """ + Duplicate values < 32 bits are prohibited. This confirms integer field + values are cast to Int64 so MongoDB stores it as long. Otherwise, the + partialFilterExpression: {$type: long} unique constraint doesn't work. + """ + + test_value = 123 + + def test_integerfield(self): + UniqueIntegers.objects.create(plain=self.test_value) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(plain=self.test_value) + + def test_bigintegerfield(self): + UniqueIntegers.objects.create(big=self.test_value) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(big=self.test_value) + + def test_positiveintegerfield(self): + UniqueIntegers.objects.create(positive=self.test_value) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(positive=self.test_value) + + def test_positivebigintegerfield(self): + UniqueIntegers.objects.create(positive_big=self.test_value) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(positive_big=self.test_value) + + +class LargeUniqueTests(TestCase): + """ + Duplicate values > 32 bits are prohibited. This confirms each field uses + the long db_type() rather than the 32 bit int type. + """ + + test_value = 2**63 - 1 + + def test_integerfield(self): + UniqueIntegers.objects.create(plain=self.test_value) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(plain=self.test_value) + + def test_bigintegerfield(self): + UniqueIntegers.objects.create(big=self.test_value) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(big=self.test_value) + + def test_positiveintegerfield(self): + UniqueIntegers.objects.create(positive=self.test_value) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(positive=self.test_value) + + def test_positivebigintegerfield(self): + UniqueIntegers.objects.create(positive_big=self.test_value) + with self.assertRaises(IntegrityError): + UniqueIntegers.objects.create(positive_big=self.test_value)