From f11a3324db8fdc9f7814a684d84eb8a9fa8fe92b Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sun, 20 Apr 2025 22:00:47 -0400 Subject: [PATCH 1/9] INTPYTHON-599 Make a field's custom lookups available in embedded model queries --- .../fields/embedded_model.py | 7 +++ docs/source/releases/5.1.x.rst | 10 +++++ tests/model_fields_/models.py | 2 + tests/model_fields_/test_embedded_model.py | 43 +++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/django_mongodb_backend/fields/embedded_model.py b/django_mongodb_backend/fields/embedded_model.py index d9dd5b6cf..543c2db06 100644 --- a/django_mongodb_backend/fields/embedded_model.py +++ b/django_mongodb_backend/fields/embedded_model.py @@ -155,6 +155,9 @@ def __init__(self, key_name, ref_field, *args, **kwargs): self.key_name = str(key_name) self.ref_field = ref_field + def get_lookup(self, name): + return self.ref_field.get_lookup(name) + def get_transform(self, name): """ Validate that `name` is either a field of an embedded model or a @@ -204,6 +207,10 @@ def as_mql(self, compiler, connection): result = build_json_mql_path(result, json_key_transforms) return result + @property + def output_field(self): + return self.ref_field + class KeyTransformFactory: def __init__(self, key_name, ref_field): diff --git a/docs/source/releases/5.1.x.rst b/docs/source/releases/5.1.x.rst index 30ac695df..58b153bb9 100644 --- a/docs/source/releases/5.1.x.rst +++ b/docs/source/releases/5.1.x.rst @@ -2,6 +2,16 @@ Django MongoDB Backend 5.1.x ============================ +5.1.0 beta 3 +============ + +*Unreleased* + +- Added support for a field's custom lookups in ``EmbeddedModelField``, e.g. + ``ArrayField``’s ``contains``, ``contained__by``, etc. + +.. _django-mongodb-backend-5.1.0-beta-2: + 5.1.0 beta 2 ============ diff --git a/tests/model_fields_/models.py b/tests/model_fields_/models.py index b25b94a1c..ad573323b 100644 --- a/tests/model_fields_/models.py +++ b/tests/model_fields_/models.py @@ -110,12 +110,14 @@ class Address(EmbeddedModel): city = models.CharField(max_length=20) state = models.CharField(max_length=2) zip_code = models.IntegerField(db_index=True) + tags = ArrayField(models.CharField(max_length=100), null=True, blank=True) class Author(EmbeddedModel): name = models.CharField(max_length=10) age = models.IntegerField() address = EmbeddedModelField(Address) + skills = ArrayField(models.CharField(max_length=100), null=True, blank=True) class Book(models.Model): diff --git a/tests/model_fields_/test_embedded_model.py b/tests/model_fields_/test_embedded_model.py index eee0dd1a9..6466857bd 100644 --- a/tests/model_fields_/test_embedded_model.py +++ b/tests/model_fields_/test_embedded_model.py @@ -186,6 +186,49 @@ def test_nested(self): self.assertCountEqual(Book.objects.filter(author__address__city="NYC"), [obj]) +class ArrayFieldTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.book = Book.objects.create( + author=Author( + name="Shakespeare", + age=55, + skills=["writing", "editing"], + address=Address(city="NYC", state="NY", tags=["home", "shipping"]), + ), + ) + + def test_contains(self): + self.assertCountEqual(Book.objects.filter(author__skills__contains=["nonexistent"]), []) + self.assertCountEqual( + Book.objects.filter(author__skills__contains=["writing"]), [self.book] + ) + # Nested + self.assertCountEqual( + Book.objects.filter(author__address__tags__contains=["nonexistent"]), [] + ) + self.assertCountEqual( + Book.objects.filter(author__address__tags__contains=["home"]), [self.book] + ) + + def test_contained_by(self): + self.assertCountEqual( + Book.objects.filter(author__skills__contained_by=["writing", "publishing"]), [] + ) + self.assertCountEqual( + Book.objects.filter(author__skills__contained_by=["writing", "editing", "publishing"]), + [self.book], + ) + # Nested + self.assertCountEqual( + Book.objects.filter(author__address__tags__contained_by=["home", "work"]), [] + ) + self.assertCountEqual( + Book.objects.filter(author__address__tags__contained_by=["home", "work", "shipping"]), + [self.book], + ) + + class InvalidLookupTests(SimpleTestCase): def test_invalid_field(self): msg = "Author has no field named 'first_name'" From 6437d3f812f4e0cb629972da179eb1b7035033ae Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 26 Apr 2025 19:50:44 -0400 Subject: [PATCH 2/9] INTPYTHON-599 Make a field's custom transforms available in embedded model queries --- .../fields/embedded_model.py | 53 ++++++------------- docs/source/releases/5.1.x.rst | 5 +- tests/model_fields_/test_embedded_model.py | 7 +++ 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/django_mongodb_backend/fields/embedded_model.py b/django_mongodb_backend/fields/embedded_model.py index 543c2db06..cf45eac8a 100644 --- a/django_mongodb_backend/fields/embedded_model.py +++ b/django_mongodb_backend/fields/embedded_model.py @@ -7,7 +7,6 @@ from django.db.models.lookups import Transform from .. import forms -from .json import build_json_mql_path class EmbeddedModelField(models.Field): @@ -163,49 +162,29 @@ def get_transform(self, name): Validate that `name` is either a field of an embedded model or a lookup on an embedded model's field. """ - result = None - if isinstance(self.ref_field, EmbeddedModelField): - opts = self.ref_field.embedded_model._meta - new_field = opts.get_field(name) - result = KeyTransformFactory(name, new_field) + if transform := self.ref_field.get_transform(name): + return transform + suggested_lookups = difflib.get_close_matches(name, self.ref_field.get_lookups()) + if suggested_lookups: + suggested_lookups = " or ".join(suggested_lookups) + suggestion = f", perhaps you meant {suggested_lookups}?" else: - if self.ref_field.get_transform(name) is None: - suggested_lookups = difflib.get_close_matches(name, self.ref_field.get_lookups()) - if suggested_lookups: - suggested_lookups = " or ".join(suggested_lookups) - suggestion = f", perhaps you meant {suggested_lookups}?" - else: - suggestion = "." - raise FieldDoesNotExist( - f"Unsupported lookup '{name}' for " - f"{self.ref_field.__class__.__name__} '{self.ref_field.name}'" - f"{suggestion}" - ) - result = KeyTransformFactory(name, self.ref_field) - return result + suggestion = "." + raise FieldDoesNotExist( + f"Unsupported lookup '{name}' for " + f"{self.ref_field.__class__.__name__} '{self.ref_field.name}'" + f"{suggestion}" + ) - def preprocess_lhs(self, compiler, connection): + def as_mql(self, compiler, connection): previous = self - embedded_key_transforms = [] - json_key_transforms = [] + key_transforms = [] while isinstance(previous, KeyTransform): - if isinstance(previous.ref_field, EmbeddedModelField): - embedded_key_transforms.insert(0, previous.key_name) - else: - json_key_transforms.insert(0, previous.key_name) + key_transforms.insert(0, previous.key_name) previous = previous.lhs mql = previous.as_mql(compiler, connection) - # The first json_key_transform is the field name. - embedded_key_transforms.append(json_key_transforms.pop(0)) - return mql, embedded_key_transforms, json_key_transforms - - def as_mql(self, compiler, connection): - mql, key_transforms, json_key_transforms = self.preprocess_lhs(compiler, connection) transforms = ".".join(key_transforms) - result = f"{mql}.{transforms}" - if json_key_transforms: - result = build_json_mql_path(result, json_key_transforms) - return result + return f"{mql}.{transforms}" @property def output_field(self): diff --git a/docs/source/releases/5.1.x.rst b/docs/source/releases/5.1.x.rst index 58b153bb9..dde2e03a2 100644 --- a/docs/source/releases/5.1.x.rst +++ b/docs/source/releases/5.1.x.rst @@ -7,8 +7,9 @@ Django MongoDB Backend 5.1.x *Unreleased* -- Added support for a field's custom lookups in ``EmbeddedModelField``, e.g. - ``ArrayField``’s ``contains``, ``contained__by``, etc. +- Added support for a field's custom lookups and transforms in + ``EmbeddedModelField``, e.g. ``ArrayField``’s ``contains``, + ``contained__by``, ``len``, etc. .. _django-mongodb-backend-5.1.0-beta-2: diff --git a/tests/model_fields_/test_embedded_model.py b/tests/model_fields_/test_embedded_model.py index 6466857bd..700a3cf1c 100644 --- a/tests/model_fields_/test_embedded_model.py +++ b/tests/model_fields_/test_embedded_model.py @@ -228,6 +228,13 @@ def test_contained_by(self): [self.book], ) + def test_len(self): + self.assertCountEqual(Book.objects.filter(author__skills__len=1), []) + self.assertCountEqual(Book.objects.filter(author__skills__len=2), [self.book]) + # Nested + self.assertCountEqual(Book.objects.filter(author__address__tags__len=1), []) + self.assertCountEqual(Book.objects.filter(author__address__tags__len=2), [self.book]) + class InvalidLookupTests(SimpleTestCase): def test_invalid_field(self): From 87fa60c22c88646dfc8daa2c62281396deb983be Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Thu, 1 May 2025 22:08:34 -0400 Subject: [PATCH 3/9] Fix Trunc database function with tzinfo parameter --- django_mongodb_backend/features.py | 4 ---- django_mongodb_backend/functions.py | 32 ++++++++++++++++++++++++++++ django_mongodb_backend/operations.py | 14 +++++++++--- docs/source/releases/5.1.x.rst | 2 ++ docs/source/topics/known-issues.rst | 4 ---- 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index a286a2cbf..793590901 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -58,10 +58,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "model_fields.test_jsonfield.TestQuerying.test_icontains", # MongoDB gives ROUND(365, -1)=360 instead of 370 like other databases. "db_functions.math.test_round.RoundTests.test_integer_with_negative_precision", - # Truncating in another timezone doesn't work becauase MongoDB converts - # the result back to UTC. - "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_func_with_timezone", - "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_timezone_applied_before_truncation", # Unexpected alias_refcount in alias_map. "queries.tests.Queries1Tests.test_order_by_tables", # The $sum aggregation returns 0 instead of None for null. diff --git a/django_mongodb_backend/functions.py b/django_mongodb_backend/functions.py index 464dddd6f..f8546f963 100644 --- a/django_mongodb_backend/functions.py +++ b/django_mongodb_backend/functions.py @@ -1,4 +1,8 @@ +from datetime import datetime + +from django.conf import settings from django.db import NotSupportedError +from django.db.models import DateField, DateTimeField, TimeField from django.db.models.expressions import Func from django.db.models.functions.comparison import Cast, Coalesce, Greatest, Least, NullIf from django.db.models.functions.datetime import ( @@ -195,6 +199,33 @@ def trunc(self, compiler, connection): return {"$dateTrunc": lhs_mql} +def trunc_convert_value(self, value, expression, connection): + if connection.vendor == "mongodb": + # A custom TruncBase.convert_value() for MongoDB. + if value is None: + return None + convert_to_tz = settings.USE_TZ and self.get_tzname() != "UTC" + if isinstance(self.output_field, DateTimeField): + if convert_to_tz: + # Unlike other databases, MongoDB returns the value in UTC, + # so rather than setting the time zone equal to self.tzinfo, + # the value must be converted to tzinfo. + value = value.astimezone(self.tzinfo) + elif isinstance(value, datetime): + if isinstance(self.output_field, DateField): + if convert_to_tz: + value = value.astimezone(self.tzinfo) + # Truncate for Trunc(..., output_field=DateField) + value = value.date() + elif isinstance(self.output_field, TimeField): + if convert_to_tz: + value = value.astimezone(self.tzinfo) + # Truncate for Trunc(..., output_field=TimeField) + value = value.time() + return value + return self.convert_value(value, expression, connection) + + def trunc_date(self, compiler, connection): # Cast to date rather than truncate to date. lhs_mql = process_lhs(self, compiler, connection) @@ -254,6 +285,7 @@ def register_functions(): Substr.as_mql = substr Trim.as_mql = trim("trim") TruncBase.as_mql = trunc + TruncBase.convert_value = trunc_convert_value TruncDate.as_mql = trunc_date TruncTime.as_mql = trunc_time Upper.as_mql = preserve_null("toUpper") diff --git a/django_mongodb_backend/operations.py b/django_mongodb_backend/operations.py index cb1e93db0..9138f06e4 100644 --- a/django_mongodb_backend/operations.py +++ b/django_mongodb_backend/operations.py @@ -10,7 +10,7 @@ from django.db.backends.base.operations import BaseDatabaseOperations from django.db.models import TextField from django.db.models.expressions import Combinable, Expression -from django.db.models.functions import Cast +from django.db.models.functions import Cast, Trunc from django.utils import timezone from django.utils.regex_helper import _lazy_re_compile @@ -97,7 +97,11 @@ def get_db_converters(self, expression): ] ) elif internal_type == "DateField": - converters.append(self.convert_datefield_value) + # Trunc(... output_field="DateField") values must remain datetime + # until Trunc.convert_value() so they can be converted from UTC + # before truncation. + if not isinstance(expression, Trunc): + converters.append(self.convert_datefield_value) elif internal_type == "DateTimeField": if settings.USE_TZ: converters.append(self.convert_datetimefield_value) @@ -106,7 +110,11 @@ def get_db_converters(self, expression): elif internal_type == "JSONField": converters.append(self.convert_jsonfield_value) elif internal_type == "TimeField": - converters.append(self.convert_timefield_value) + # Trunc(... output_field="TimeField") values must remain datetime + # until Trunc.convert_value() so they can be converted from UTC + # before truncation. + if not isinstance(expression, Trunc): + converters.append(self.convert_timefield_value) elif internal_type == "UUIDField": converters.append(self.convert_uuidfield_value) return converters diff --git a/docs/source/releases/5.1.x.rst b/docs/source/releases/5.1.x.rst index dde2e03a2..ce3a5877b 100644 --- a/docs/source/releases/5.1.x.rst +++ b/docs/source/releases/5.1.x.rst @@ -10,6 +10,8 @@ Django MongoDB Backend 5.1.x - Added support for a field's custom lookups and transforms in ``EmbeddedModelField``, e.g. ``ArrayField``’s ``contains``, ``contained__by``, ``len``, etc. +- Fixed the results of queries that use the ``tzinfo`` parameter of the + ``Trunc`` database functions. .. _django-mongodb-backend-5.1.0-beta-2: diff --git a/docs/source/topics/known-issues.rst b/docs/source/topics/known-issues.rst index 5f5ed4a45..4848d3bad 100644 --- a/docs/source/topics/known-issues.rst +++ b/docs/source/topics/known-issues.rst @@ -74,10 +74,6 @@ Database functions :class:`~django.db.models.functions.SHA512` - :class:`~django.db.models.functions.Sign` -- The ``tzinfo`` parameter of the :class:`~django.db.models.functions.Trunc` - database functions doesn't work properly because MongoDB converts the result - back to UTC. - Transaction management ====================== From ac281218e0555512a32bb476e3bffa888b81c34f Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 2 May 2025 20:32:41 -0400 Subject: [PATCH 4/9] Document the tzinfo parameter of TruncDate/TruncTime as unsupported Add the same exception raising from TruncDate to TruncTime and add tests for both functions. --- django_mongodb_backend/functions.py | 3 +++ docs/source/topics/known-issues.rst | 5 +++++ tests/db_functions_/models.py | 5 +++++ tests/db_functions_/test_datetime.py | 26 ++++++++++++++++++++++++++ 4 files changed, 39 insertions(+) create mode 100644 tests/db_functions_/models.py create mode 100644 tests/db_functions_/test_datetime.py diff --git a/django_mongodb_backend/functions.py b/django_mongodb_backend/functions.py index f8546f963..9405e8711 100644 --- a/django_mongodb_backend/functions.py +++ b/django_mongodb_backend/functions.py @@ -248,6 +248,9 @@ def trunc_date(self, compiler, connection): def trunc_time(self, compiler, connection): + tzname = self.get_tzname() + if tzname and tzname != "UTC": + raise NotSupportedError(f"TruncTime with tzinfo ({tzname}) isn't supported on MongoDB.") lhs_mql = process_lhs(self, compiler, connection) return { "$dateFromString": { diff --git a/docs/source/topics/known-issues.rst b/docs/source/topics/known-issues.rst index 4848d3bad..e8c0a5534 100644 --- a/docs/source/topics/known-issues.rst +++ b/docs/source/topics/known-issues.rst @@ -74,6 +74,11 @@ Database functions :class:`~django.db.models.functions.SHA512` - :class:`~django.db.models.functions.Sign` +- The ``tzinfo`` parameter of the + :class:`~django.db.models.functions.TruncDate` and + :class:`~django.db.models.functions.TruncTime` database functions isn't + supported. + Transaction management ====================== diff --git a/tests/db_functions_/models.py b/tests/db_functions_/models.py new file mode 100644 index 000000000..17b9ad1a7 --- /dev/null +++ b/tests/db_functions_/models.py @@ -0,0 +1,5 @@ +from django.db import models + + +class DTModel(models.Model): + start_datetime = models.DateTimeField(null=True, blank=True) diff --git a/tests/db_functions_/test_datetime.py b/tests/db_functions_/test_datetime.py new file mode 100644 index 000000000..e0df28801 --- /dev/null +++ b/tests/db_functions_/test_datetime.py @@ -0,0 +1,26 @@ +from zoneinfo import ZoneInfo + +from django.db import NotSupportedError +from django.db.models.functions import TruncDate, TruncTime +from django.test import TestCase, override_settings + +from .models import DTModel + + +@override_settings(USE_TZ=True) +class TruncTests(TestCase): + melb = ZoneInfo("Australia/Melbourne") + + def test_truncdate_tzinfo(self): + msg = "TruncDate with tzinfo (Australia/Melbourne) isn't supported on MongoDB." + with self.assertRaisesMessage(NotSupportedError, msg): + DTModel.objects.annotate( + melb_date=TruncDate("start_datetime", tzinfo=self.melb), + ).get() + + def test_trunctime_tzinfo(self): + msg = "TruncTime with tzinfo (Australia/Melbourne) isn't supported on MongoDB." + with self.assertRaisesMessage(NotSupportedError, msg): + DTModel.objects.annotate( + melb_date=TruncTime("start_datetime", tzinfo=self.melb), + ).get() From 41cc35f39eeda34a8f522e56741b8c6f2986f10f Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Thu, 10 Apr 2025 21:23:05 -0400 Subject: [PATCH 5/9] Add support for QuerySet.dates() and datetimes() This was blocked on support for QuerySet.distinct() which was added in e04056ed9ed5000c2427ecdef0db50a1649fe3a6. --- django_mongodb_backend/compiler.py | 9 ---- django_mongodb_backend/features.py | 76 +--------------------------- docs/source/ref/models/querysets.rst | 2 - docs/source/releases/5.1.x.rst | 1 + docs/source/topics/known-issues.rst | 2 - 5 files changed, 2 insertions(+), 88 deletions(-) diff --git a/django_mongodb_backend/compiler.py b/django_mongodb_backend/compiler.py index cf666619a..1b4882eb4 100644 --- a/django_mongodb_backend/compiler.py +++ b/django_mongodb_backend/compiler.py @@ -332,15 +332,6 @@ def cursor_iter(self, cursor, chunk_size, columns): def check_query(self): """Check if the current query is supported by the database.""" - if self.query.distinct: - # This is a heuristic to detect QuerySet.datetimes() and dates(). - # "datetimefield" and "datefield" are the names of the annotations - # the methods use. A user could annotate with the same names which - # would give an incorrect error message. - if "datetimefield" in self.query.annotations: - raise NotSupportedError("QuerySet.datetimes() is not supported on MongoDB.") - if "datefield" in self.query.annotations: - raise NotSupportedError("QuerySet.dates() is not supported on MongoDB.") if self.query.extra: if any(key.startswith("_prefetch_related_") for key in self.query.extra): raise NotSupportedError("QuerySet.prefetch_related() is not supported on MongoDB.") diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 793590901..e6fc07775 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -268,81 +268,6 @@ def django_test_expected_failures(self): "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation", "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation_desc", }, - "QuerySet.dates() is not supported on MongoDB.": { - "admin_changelist.tests.ChangeListTests.test_computed_list_display_localization", - "admin_changelist.tests.ChangeListTests.test_object_tools_displayed_no_add_permission", - "admin_views.tests.AdminViewBasicTest.test_change_list_sorting_override_model_admin", - "admin_views.tests.AdminViewBasicTest.test_multiple_sort_same_field", - "admin_views.tests.AdminViewListEditable.test_inheritance", - "admin_views.tests.CSSTest.test_changelist_field_classes", - "admin_views.tests.DateHierarchyTests", - "aggregation.tests.AggregateTestCase.test_dates_with_aggregation", - "annotations.tests.AliasTests.test_dates_alias", - "aggregation_regress.tests.AggregationTests.test_more_more_more2", - "backends.tests.DateQuotingTest.test_django_date_trunc", - "dates.tests.DatesTests.test_dates_trunc_datetime_fields", - "dates.tests.DatesTests.test_related_model_traverse", - "generic_views.test_dates.ArchiveIndexViewTests.test_allow_empty_archive_view", - "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view", - "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_by_month", - "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_context_object_name", - "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_custom_sorting", - "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_custom_sorting_dec", - "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_template", - "generic_views.test_dates.ArchiveIndexViewTests.test_archive_view_template_suffix", - "generic_views.test_dates.ArchiveIndexViewTests.test_date_list_order", - "generic_views.test_dates.ArchiveIndexViewTests.test_no_duplicate_query", - "generic_views.test_dates.ArchiveIndexViewTests.test_paginated_archive_view", - "generic_views.test_dates.ArchiveIndexViewTests.test_paginated_archive_view_does_not_load_entire_table", - "generic_views.test_dates.MonthArchiveViewTests.test_custom_month_format", - "generic_views.test_dates.MonthArchiveViewTests.test_date_list_order", - "generic_views.test_dates.MonthArchiveViewTests.test_month_view", - "generic_views.test_dates.MonthArchiveViewTests.test_month_view_allow_empty", - "generic_views.test_dates.MonthArchiveViewTests.test_month_view_allow_future", - "generic_views.test_dates.MonthArchiveViewTests.test_month_view_get_month_from_request", - "generic_views.test_dates.MonthArchiveViewTests.test_month_view_paginated", - "generic_views.test_dates.MonthArchiveViewTests.test_previous_month_without_content", - "generic_views.test_dates.YearArchiveViewTests.test_date_list_order", - "generic_views.test_dates.YearArchiveViewTests.test_get_context_data_receives_extra_context", - "generic_views.test_dates.YearArchiveViewTests.test_no_duplicate_query", - "generic_views.test_dates.YearArchiveViewTests.test_year_view", - "generic_views.test_dates.YearArchiveViewTests.test_year_view_allow_future", - "generic_views.test_dates.YearArchiveViewTests.test_year_view_custom_sort_order", - "generic_views.test_dates.YearArchiveViewTests.test_year_view_empty", - "generic_views.test_dates.YearArchiveViewTests.test_year_view_make_object_list", - "generic_views.test_dates.YearArchiveViewTests.test_year_view_paginated", - "generic_views.test_dates.YearArchiveViewTests.test_year_view_two_custom_sort_orders", - "many_to_one.tests.ManyToOneTests.test_select_related", - "model_regress.tests.ModelTests.test_date_filter_null", - "reserved_names.tests.ReservedNameTests.test_dates", - "queryset_pickle.tests.PickleabilityTestCase.test_specialized_queryset", - }, - "QuerySet.datetimes() is not supported on MongoDB.": { - "admin_views.test_templatetags.DateHierarchyTests", - "admin_views.test_templatetags.AdminTemplateTagsTest.test_override_change_list_template_tags", - "admin_views.tests.AdminViewBasicTest.test_date_hierarchy_empty_queryset", - "admin_views.tests.AdminViewBasicTest.test_date_hierarchy_local_date_differ_from_utc", - "admin_views.tests.AdminViewBasicTest.test_date_hierarchy_timezone_dst", - "annotations.tests.AliasTests.test_datetimes_alias", - "datetimes.tests.DateTimesTests.test_21432", - "datetimes.tests.DateTimesTests.test_datetimes_has_lazy_iterator", - "datetimes.tests.DateTimesTests.test_datetimes_returns_available_dates_for_given_scope_and_given_field", - "datetimes.tests.DateTimesTests.test_related_model_traverse", - "generic_views.test_dates.ArchiveIndexViewTests.test_aware_datetime_archive_view", - "generic_views.test_dates.ArchiveIndexViewTests.test_datetime_archive_view", - "generic_views.test_dates.MonthArchiveViewTests.test_aware_datetime_month_view", - "generic_views.test_dates.MonthArchiveViewTests.test_datetime_month_view", - "generic_views.test_dates.YearArchiveViewTests.test_aware_datetime_year_view", - "generic_views.test_dates.YearArchiveViewTests.test_datetime_year_view", - "model_inheritance_regress.tests.ModelInheritanceTest.test_issue_7105", - "queries.tests.Queries1Tests.test_ticket7155", - "queries.tests.Queries1Tests.test_ticket7791", - "queries.tests.Queries1Tests.test_tickets_6180_6203", - "queries.tests.Queries1Tests.test_tickets_7087_12242", - "timezones.tests.LegacyDatabaseTests.test_query_datetimes", - "timezones.tests.NewDatabaseTests.test_query_datetimes", - "timezones.tests.NewDatabaseTests.test_query_datetimes_in_other_timezone", - }, "QuerySet.extra() is not supported.": { "aggregation.tests.AggregateTestCase.test_exists_extra_where_with_aggregate", "annotations.tests.NonAggregateAnnotationTestCase.test_column_field_ordering", @@ -362,6 +287,7 @@ def django_test_expected_failures(self): "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_extra_and_values_list", "queries.tests.EscapingTests.test_ticket_7302", "queries.tests.Queries1Tests.test_tickets_1878_2939", + "queries.tests.Queries1Tests.test_tickets_7087_12242", "queries.tests.Queries5Tests.test_extra_select_literal_percent_s", "queries.tests.Queries5Tests.test_ticket7256", "queries.tests.ValuesQuerysetTests.test_extra_multiple_select_params_values_order_by", diff --git a/docs/source/ref/models/querysets.rst b/docs/source/ref/models/querysets.rst index a0e32957c..b1cfbf8b7 100644 --- a/docs/source/ref/models/querysets.rst +++ b/docs/source/ref/models/querysets.rst @@ -9,8 +9,6 @@ All of Django's :doc:`QuerySet methods ` are supported, except: - :meth:`bulk_update() ` - - :meth:`dates() ` - - :meth:`datetimes() ` - :meth:`extra() ` - :meth:`prefetch_related() ` diff --git a/docs/source/releases/5.1.x.rst b/docs/source/releases/5.1.x.rst index ce3a5877b..3e164fa74 100644 --- a/docs/source/releases/5.1.x.rst +++ b/docs/source/releases/5.1.x.rst @@ -12,6 +12,7 @@ Django MongoDB Backend 5.1.x ``contained__by``, ``len``, etc. - Fixed the results of queries that use the ``tzinfo`` parameter of the ``Trunc`` database functions. +- Added support for ``QuerySet.dates()`` and ``datetimes()``. .. _django-mongodb-backend-5.1.0-beta-2: diff --git a/docs/source/topics/known-issues.rst b/docs/source/topics/known-issues.rst index e8c0a5534..0ec4fc1b8 100644 --- a/docs/source/topics/known-issues.rst +++ b/docs/source/topics/known-issues.rst @@ -31,8 +31,6 @@ Querying - The following ``QuerySet`` methods aren't supported: - :meth:`bulk_update() ` - - :meth:`dates() ` - - :meth:`datetimes() ` - :meth:`extra() ` - :meth:`prefetch_related() ` From 34abcc47164f18ea97dcc8190f73e26e3f3d0656 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 25 Apr 2025 21:46:36 -0400 Subject: [PATCH 6/9] INTPYTHON-602 Fix QuerySet results of embedded model fields that use database converters --- django_mongodb_backend/operations.py | 12 +++++++++++ docs/source/releases/5.1.x.rst | 3 +++ tests/model_fields_/models.py | 6 ++++++ tests/model_fields_/test_embedded_model.py | 25 +++++++++++++++++++++- 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/django_mongodb_backend/operations.py b/django_mongodb_backend/operations.py index 9138f06e4..bdf41bcf9 100644 --- a/django_mongodb_backend/operations.py +++ b/django_mongodb_backend/operations.py @@ -107,6 +107,8 @@ def get_db_converters(self, expression): converters.append(self.convert_datetimefield_value) elif internal_type == "DecimalField": converters.append(self.convert_decimalfield_value) + elif internal_type == "EmbeddedModelField": + converters.append(self.convert_embeddedmodelfield_value) elif internal_type == "JSONField": converters.append(self.convert_jsonfield_value) elif internal_type == "TimeField": @@ -150,6 +152,16 @@ def convert_durationfield_value(self, value, expression, connection): value = datetime.timedelta(milliseconds=int(str(value))) return value + def convert_embeddedmodelfield_value(self, value, expression, connection): + if value is not None: + # Apply database converters to each field of the embedded model. + for field in expression.output_field.embedded_model._meta.fields: + field_expr = Expression(output_field=field) + converters = connection.ops.get_db_converters(field_expr) + for converter in converters: + value[field.attname] = converter(value[field.attname], field_expr, connection) + return value + def convert_jsonfield_value(self, value, expression, connection): """ Convert dict data to a string so that JSONField.from_db_value() can diff --git a/docs/source/releases/5.1.x.rst b/docs/source/releases/5.1.x.rst index 3e164fa74..fac35eb16 100644 --- a/docs/source/releases/5.1.x.rst +++ b/docs/source/releases/5.1.x.rst @@ -13,6 +13,9 @@ Django MongoDB Backend 5.1.x - Fixed the results of queries that use the ``tzinfo`` parameter of the ``Trunc`` database functions. - Added support for ``QuerySet.dates()`` and ``datetimes()``. +- Fixed loading of ``QuerySet`` results for embedded models that have fields + that use database converters. For example, a crash for ``DecimalField``: + ``ValidationError: ['“1” value must be a decimal number.']``). .. _django-mongodb-backend-5.1.0-beta-2: diff --git a/tests/model_fields_/models.py b/tests/model_fields_/models.py index ad573323b..2470f4bb8 100644 --- a/tests/model_fields_/models.py +++ b/tests/model_fields_/models.py @@ -104,6 +104,12 @@ class Data(EmbeddedModel): auto_now = models.DateTimeField(auto_now=True) auto_now_add = models.DateTimeField(auto_now_add=True) json_value = models.JSONField() + decimal = models.DecimalField(max_digits=9, decimal_places="2", null=True, blank=True) + nested_data = EmbeddedModelField("NestedData", null=True, blank=True) + + +class NestedData(EmbeddedModel): + decimal = models.DecimalField(max_digits=9, decimal_places="2", null=True, blank=True) class Address(EmbeddedModel): diff --git a/tests/model_fields_/test_embedded_model.py b/tests/model_fields_/test_embedded_model.py index 700a3cf1c..004eae00d 100644 --- a/tests/model_fields_/test_embedded_model.py +++ b/tests/model_fields_/test_embedded_model.py @@ -24,6 +24,7 @@ Data, Holder, Library, + NestedData, ) from .utils import truncate_ms @@ -93,7 +94,16 @@ def test_pre_save(self): class QueryingTests(TestCase): @classmethod def setUpTestData(cls): - cls.objs = [Holder.objects.create(data=Data(integer=x)) for x in range(6)] + cls.objs = [ + Holder.objects.create( + data=Data( + integer=x, + decimal=f"{x}.5", + nested_data=NestedData(decimal=f"{x}.5"), + ) + ) + for x in range(6) + ] def test_exact(self): self.assertCountEqual(Holder.objects.filter(data__integer=3), [self.objs[3]]) @@ -113,6 +123,19 @@ def test_gte(self): def test_range(self): self.assertCountEqual(Holder.objects.filter(data__integer__range=(2, 4)), self.objs[2:5]) + def test_exact_decimal(self): + # EmbeddedModelField lookups call + # DatabaseOperations.adapt__field_value(). + self.assertCountEqual(Holder.objects.filter(data__decimal="3.5"), [self.objs[3]]) + + def test_lt_decimal(self): + self.assertCountEqual(Holder.objects.filter(data__decimal__lt="3"), self.objs[0:3]) + + def test_exact_decimal_nested(self): + self.assertCountEqual( + Holder.objects.filter(data__nested_data__decimal="3.5"), [self.objs[3]] + ) + def test_order_by_embedded_field(self): qs = Holder.objects.filter(data__integer__gt=3).order_by("-data__integer") self.assertSequenceEqual(qs, list(reversed(self.objs[4:]))) From cdacc51cac29ecd5e7d4e6de07765609709b6cfd Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 6 May 2025 09:37:12 -0400 Subject: [PATCH 7/9] INTPYTHON-602 Fix QuerySet results of embedded model fields that have field converters Follow up to 4633a9eefd414b83f1b20b09194d5e96375c9b4d. --- django_mongodb_backend/operations.py | 4 +++- tests/model_fields_/test_embedded_model.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/django_mongodb_backend/operations.py b/django_mongodb_backend/operations.py index bdf41bcf9..832c1db52 100644 --- a/django_mongodb_backend/operations.py +++ b/django_mongodb_backend/operations.py @@ -157,7 +157,9 @@ def convert_embeddedmodelfield_value(self, value, expression, connection): # Apply database converters to each field of the embedded model. for field in expression.output_field.embedded_model._meta.fields: field_expr = Expression(output_field=field) - converters = connection.ops.get_db_converters(field_expr) + converters = connection.ops.get_db_converters( + field_expr + ) + field_expr.get_db_converters(connection) for converter in converters: value[field.attname] = converter(value[field.attname], field_expr, connection) return value diff --git a/tests/model_fields_/test_embedded_model.py b/tests/model_fields_/test_embedded_model.py index 004eae00d..ec9f9dfc4 100644 --- a/tests/model_fields_/test_embedded_model.py +++ b/tests/model_fields_/test_embedded_model.py @@ -71,6 +71,11 @@ def test_save_load_null(self): obj = Holder.objects.get() self.assertIsNone(obj.data) + def test_save_load_json(self): + obj = Holder.objects.create(data=Data(json_value={"a": 1})) + obj.refresh_from_db() + self.assertEqual(obj.data.json_value, {"a": 1}) + def test_pre_save(self): """Field.pre_save() is called on embedded model fields.""" obj = Holder.objects.create(data=Data()) From 5726c74f4d9974ee757106f4d149f0192aecdd18 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 7 May 2025 10:34:47 -0400 Subject: [PATCH 8/9] Correct docs: QuerySet.bulk_update() is supported Added in 97769ba88e6951d8e343f8c3abeb8978d49b6f01. --- docs/source/ref/models/querysets.rst | 1 - docs/source/topics/known-issues.rst | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/source/ref/models/querysets.rst b/docs/source/ref/models/querysets.rst index b1cfbf8b7..601e79d10 100644 --- a/docs/source/ref/models/querysets.rst +++ b/docs/source/ref/models/querysets.rst @@ -8,7 +8,6 @@ Supported ``QuerySet`` methods All of Django's :doc:`QuerySet methods ` are supported, except: - - :meth:`bulk_update() ` - :meth:`extra() ` - :meth:`prefetch_related() ` diff --git a/docs/source/topics/known-issues.rst b/docs/source/topics/known-issues.rst index 0ec4fc1b8..4779eb782 100644 --- a/docs/source/topics/known-issues.rst +++ b/docs/source/topics/known-issues.rst @@ -30,7 +30,6 @@ Querying - The following ``QuerySet`` methods aren't supported: - - :meth:`bulk_update() ` - :meth:`extra() ` - :meth:`prefetch_related() ` From a7a05569e878b89f1b31be88aa0239caf4dfc9c4 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Mon, 12 May 2025 08:56:35 -0400 Subject: [PATCH 9/9] Bump version to 5.1.0b3 --- django_mongodb_backend/__init__.py | 2 +- docs/source/releases/5.1.x.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index bc2eb46f6..3e5bbaf04 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -1,4 +1,4 @@ -__version__ = "5.1.0b3.dev0" +__version__ = "5.1.0b3" # Check Django compatibility before other imports which may fail if the # wrong version of Django is installed. diff --git a/docs/source/releases/5.1.x.rst b/docs/source/releases/5.1.x.rst index fac35eb16..1bb715aa8 100644 --- a/docs/source/releases/5.1.x.rst +++ b/docs/source/releases/5.1.x.rst @@ -5,7 +5,7 @@ Django MongoDB Backend 5.1.x 5.1.0 beta 3 ============ -*Unreleased* +*May 13, 2025* - Added support for a field's custom lookups and transforms in ``EmbeddedModelField``, e.g. ``ArrayField``’s ``contains``,