diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index ca4c91999..253febd3d 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -76,14 +76,7 @@ jobs: datetimes db_functions empty - expressions.tests.BasicExpressionsTests.test_ticket_11722_iexact_lookup - expressions.tests.BasicExpressionsTests.test_ticket_16731_startswith_lookup - expressions.tests.ExpressionOperatorTests - expressions.tests.ExpressionsTests.test_insensitive_patterns_escape - expressions.tests.ExpressionsTests.test_patterns_escape - expressions.tests.FieldTransformTests.test_transform_in_values - expressions.tests.FTimeDeltaTests.test_date_minus_duration - expressions.tests.NegatedExpressionTests + expressions expressions_case defer defer_regress diff --git a/django_mongodb/base.py b/django_mongodb/base.py index 38c76c215..88363ab56 100644 --- a/django_mongodb/base.py +++ b/django_mongodb/base.py @@ -72,6 +72,16 @@ class DatabaseWrapper(BaseDatabaseWrapper): "istartswith": "LIKE UPPER(%s)", "iendswith": "LIKE UPPER(%s)", } + # As with `operators`, these patterns are used to generate SQL before MQL. + pattern_esc = "%%" + pattern_ops = { + "contains": "LIKE '%%' || {} || '%%'", + "icontains": "LIKE '%%' || UPPER({}) || '%%'", + "startswith": "LIKE {} || '%%'", + "istartswith": "LIKE UPPER({}) || '%%'", + "endswith": "LIKE '%%' || {}", + "iendswith": "LIKE '%%' || UPPER({})", + } mongo_operators = { "exact": lambda a, b: {"$eq": [a, b]}, "gt": lambda a, b: {"$gt": [a, b]}, diff --git a/django_mongodb/expressions.py b/django_mongodb/expressions.py index 9dc216643..c1fa10670 100644 --- a/django_mongodb/expressions.py +++ b/django_mongodb/expressions.py @@ -10,6 +10,7 @@ CombinedExpression, ExpressionWrapper, NegatedExpression, + Ref, Subquery, Value, When, @@ -68,6 +69,10 @@ def query(self, compiler, connection): # noqa: ARG001 raise NotSupportedError("Using a QuerySet in annotate() is not supported on MongoDB.") +def ref(self, compiler, connection): # noqa: ARG001 + return self.refs + + def subquery(self, compiler, connection): # noqa: ARG001 raise NotSupportedError(f"{self.__class__.__name__} is not supported on MongoDB.") @@ -83,6 +88,9 @@ def value(self, compiler, connection): # noqa: ARG001 elif isinstance(value, datetime.date): # Turn dates into datetimes since BSON doesn't support dates. value = datetime.datetime.combine(value, datetime.datetime.min.time()) + elif isinstance(value, datetime.time): + # Turn times into datetimes since BSON doesn't support times. + value = datetime.datetime.combine(datetime.datetime.min.date(), value) elif isinstance(value, datetime.timedelta): # DurationField stores milliseconds rather than microseconds. value /= datetime.timedelta(milliseconds=1) @@ -96,6 +104,7 @@ def register_expressions(): ExpressionWrapper.as_mql = expression_wrapper NegatedExpression.as_mql = negated_expression Query.as_mql = query + Ref.as_mql = ref Subquery.as_mql = subquery When.as_mql = when Value.as_mql = value diff --git a/django_mongodb/features.py b/django_mongodb/features.py index f015bee66..32b20e306 100644 --- a/django_mongodb/features.py +++ b/django_mongodb/features.py @@ -12,6 +12,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_json_field_contains = False # BSON Date type doesn't support microsecond precision. supports_microsecond_precision = False + supports_temporal_subtraction = True # MongoDB stores datetimes in UTC. supports_timezones = False # Not implemented: https://github.com/mongodb-labs/django-mongodb/issues/7 @@ -32,6 +33,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): "db_functions.tests.FunctionTests.test_nested_function_ordering", "db_functions.text.test_length.LengthTests.test_ordering", "db_functions.text.test_strindex.StrIndexTests.test_order_by", + "expressions.tests.BasicExpressionsTests.test_order_by_exists", + "expressions.tests.BasicExpressionsTests.test_order_by_multiline_sql", "expressions_case.tests.CaseExpressionTests.test_order_by_conditional_explicit", "lookup.tests.LookupQueryingTests.test_lookup_in_order_by", "ordering.tests.OrderingTests.test_default_ordering", @@ -86,11 +89,19 @@ class DatabaseFeatures(BaseDatabaseFeatures): "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_reverse_m2m", "annotations.tests.NonAggregateAnnotationTestCase.test_chaining_annotation_filter_with_m2m", "lookup.tests.LookupTests.test_lookup_collision", + "expressions.test_queryset_values.ValuesExpressionsTests.test_values_list_expression", + "expressions.test_queryset_values.ValuesExpressionsTests.test_values_list_expression_flat", "expressions_case.tests.CaseExpressionTests.test_join_promotion", "expressions_case.tests.CaseExpressionTests.test_join_promotion_multiple_annotations", "ordering.tests.OrderingTests.test_order_by_grandparent_fk_with_expression_in_default_ordering", "ordering.tests.OrderingTests.test_order_by_parent_fk_with_expression_in_default_ordering", "ordering.tests.OrderingTests.test_order_by_ptr_field_with_default_ordering_by_expression", + # 'Col' object has no attribute 'utcoffset' + "expressions.tests.IterableLookupInnerExpressionsTests.test_expressions_in_lookups_join_choice", + "expressions.tests.IterableLookupInnerExpressionsTests.test_in_lookup_allows_F_expressions_and_expressions_for_datetimes", + # pymongo.errors.OperationFailure: $multiply only supports numeric + # types, not date. (should be wrapped in DatabaseError). + "expressions.tests.FTimeDeltaTests.test_invalid_operator", } # $bitAnd, #bitOr, and $bitXor are new in MongoDB 6.3. _django_test_expected_failures_bitwise = { @@ -114,6 +125,10 @@ def django_test_expected_failures(self): "Insert expressions aren't supported.": { "bulk_create.tests.BulkCreateTests.test_bulk_insert_now", "bulk_create.tests.BulkCreateTests.test_bulk_insert_expressions", + "expressions.tests.BasicExpressionsTests.test_new_object_create", + "expressions.tests.BasicExpressionsTests.test_new_object_save", + "expressions.tests.BasicExpressionsTests.test_object_create_with_aggregate", + "expressions.tests.BasicExpressionsTests.test_object_create_with_f_expression_in_subquery", # PI() "db_functions.math.test_round.RoundTests.test_decimal_with_precision", "db_functions.math.test_round.RoundTests.test_float_with_precision", @@ -138,6 +153,21 @@ def django_test_expected_failures(self): "db_functions.text.test_replace.ReplaceTests.test_update", "db_functions.text.test_substr.SubstrTests.test_basic", "db_functions.text.test_upper.UpperTests.test_basic", + "expressions.tests.BasicExpressionsTests.test_arithmetic", + "expressions.tests.BasicExpressionsTests.test_filter_with_join", + "expressions.tests.BasicExpressionsTests.test_object_update", + "expressions.tests.BasicExpressionsTests.test_object_update_unsaved_objects", + "expressions.tests.BasicExpressionsTests.test_order_of_operations", + "expressions.tests.BasicExpressionsTests.test_parenthesis_priority", + "expressions.tests.BasicExpressionsTests.test_update", + "expressions.tests.BasicExpressionsTests.test_update_with_fk", + "expressions.tests.BasicExpressionsTests.test_update_with_none", + "expressions.tests.ExpressionsNumericTests.test_decimal_expression", + "expressions.tests.ExpressionsNumericTests.test_increment_value", + "expressions.tests.FTimeDeltaTests.test_delta_update", + "expressions.tests.FTimeDeltaTests.test_negative_timedelta_update", + "expressions.tests.ValueTests.test_update_TimeField_using_Value", + "expressions.tests.ValueTests.test_update_UUIDField_using_Value", "expressions_case.tests.CaseDocumentationExamples.test_conditional_update_example", "expressions_case.tests.CaseExpressionTests.test_update", "expressions_case.tests.CaseExpressionTests.test_update_big_integer", @@ -215,6 +245,9 @@ def django_test_expected_failures(self): "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_and_aggregate_values_chaining", "annotations.tests.NonAggregateAnnotationTestCase.test_filter_agg_with_double_f", "annotations.tests.NonAggregateAnnotationTestCase.test_values_with_pk_annotation", + "expressions.test_queryset_values.ValuesExpressionsTests.test_chained_values_with_expression", + "expressions.test_queryset_values.ValuesExpressionsTests.test_values_expression_group_by", + "expressions.tests.BasicExpressionsTests.test_annotate_values_aggregate", "expressions_case.tests.CaseExpressionTests.test_aggregate", "expressions_case.tests.CaseExpressionTests.test_aggregate_with_expression_as_condition", "expressions_case.tests.CaseExpressionTests.test_aggregate_with_expression_as_value", @@ -251,12 +284,12 @@ def django_test_expected_failures(self): "defer_regress.tests.DeferRegressionTest.test_basic", "defer_regress.tests.DeferRegressionTest.test_defer_annotate_select_related", "defer_regress.tests.DeferRegressionTest.test_ticket_16409", + "expressions.tests.BasicExpressionsTests.test_aggregate_subquery_annotation", + "expressions.tests.FieldTransformTests.test_month_aggregation", "expressions_case.tests.CaseDocumentationExamples.test_conditional_aggregation_example", # Func not implemented. "annotations.tests.NonAggregateAnnotationTestCase.test_custom_functions", "annotations.tests.NonAggregateAnnotationTestCase.test_custom_functions_can_ref_other_functions", - # BaseDatabaseOperations may require a format_for_duration_arithmetic(). - "annotations.tests.NonAggregateAnnotationTestCase.test_mixed_type_annotation_date_interval", # FieldDoesNotExist with ordering. "annotations.tests.AliasTests.test_order_by_alias", "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_with_m2m", @@ -271,6 +304,17 @@ def django_test_expected_failures(self): }, "Exists is not supported on MongoDB.": { "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_exists_none_query", + "expressions.tests.BasicExpressionsTests.test_annotation_with_deeply_nested_outerref", + "expressions.tests.BasicExpressionsTests.test_boolean_expression_combined", + "expressions.tests.BasicExpressionsTests.test_boolean_expression_combined_with_empty_Q", + "expressions.tests.BasicExpressionsTests.test_boolean_expression_in_Q", + "expressions.tests.BasicExpressionsTests.test_case_in_filter_if_boolean_output_field", + "expressions.tests.BasicExpressionsTests.test_exists_in_filter", + "expressions.tests.BasicExpressionsTests.test_subquery", + "expressions.tests.ExistsTests.test_filter_by_empty_exists", + "expressions.tests.ExistsTests.test_negated_empty_exists", + "expressions.tests.ExistsTests.test_optimizations", + "expressions.tests.ExistsTests.test_select_negated_empty_exists", "lookup.tests.LookupTests.test_exact_exists", "lookup.tests.LookupTests.test_nested_outerref_lhs", "lookup.tests.LookupQueryingTests.test_filter_exists_lhs", @@ -281,6 +325,23 @@ def django_test_expected_failures(self): "annotations.tests.NonAggregateAnnotationTestCase.test_empty_queryset_annotation", "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_outerref", "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_subquery_with_parameters", + "expressions.tests.BasicExpressionsTests.test_annotation_with_nested_outerref", + "expressions.tests.BasicExpressionsTests.test_annotation_with_outerref", + "expressions.tests.BasicExpressionsTests.test_annotations_within_subquery", + "expressions.tests.BasicExpressionsTests.test_in_subquery", + "expressions.tests.BasicExpressionsTests.test_nested_outerref_with_function", + "expressions.tests.BasicExpressionsTests.test_nested_subquery", + "expressions.tests.BasicExpressionsTests.test_nested_subquery_join_outer_ref", + "expressions.tests.BasicExpressionsTests.test_nested_subquery_outer_ref_2", + "expressions.tests.BasicExpressionsTests.test_nested_subquery_outer_ref_with_autofield", + "expressions.tests.BasicExpressionsTests.test_outerref_mixed_case_table_name", + "expressions.tests.BasicExpressionsTests.test_outerref_with_operator", + "expressions.tests.BasicExpressionsTests.test_subquery_filter_by_aggregate", + "expressions.tests.BasicExpressionsTests.test_subquery_filter_by_lazy", + "expressions.tests.BasicExpressionsTests.test_subquery_group_by_outerref_in_filter", + "expressions.tests.BasicExpressionsTests.test_subquery_in_filter", + "expressions.tests.BasicExpressionsTests.test_subquery_references_joined_table_twice", + "expressions.tests.BasicExpressionsTests.test_uuid_pk_subquery", "lookup.tests.LookupQueryingTests.test_filter_subquery_lhs", "model_fields.test_jsonfield.TestQuerying.test_nested_key_transform_on_subquery", "model_fields.test_jsonfield.TestQuerying.test_obj_subquery_lookup", @@ -290,6 +351,9 @@ def django_test_expected_failures(self): "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_and_alias_filter_related_in_subquery", "annotations.tests.NonAggregateAnnotationTestCase.test_empty_expression_annotation", "db_functions.comparison.test_coalesce.CoalesceTests.test_empty_queryset", + "expressions.tests.FTimeDeltaTests.test_date_subquery_subtraction", + "expressions.tests.FTimeDeltaTests.test_datetime_subquery_subtraction", + "expressions.tests.FTimeDeltaTests.test_time_subquery_subtraction", "expressions_case.tests.CaseExpressionTests.test_in_subquery", "lookup.tests.LookupTests.test_exact_query_rhs_with_selected_columns", "lookup.tests.LookupTests.test_exact_sliced_queryset_limit_one", @@ -344,6 +408,8 @@ def django_test_expected_failures(self): }, "Test executes raw SQL.": { "annotations.tests.NonAggregateAnnotationTestCase.test_raw_sql_with_inherited_field", + "expressions.tests.BasicExpressionsTests.test_annotate_values_filter", + "expressions.tests.BasicExpressionsTests.test_filtering_on_rawsql_that_is_boolean", "model_fields.test_jsonfield.TestQuerying.test_key_sql_injection_escape", "model_fields.test_jsonfield.TestQuerying.test_key_transform_raw_expression", "model_fields.test_jsonfield.TestQuerying.test_nested_key_transform_raw_expression", @@ -383,6 +449,7 @@ def django_test_expected_failures(self): "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_date_func", "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_date_none", "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_lookup_name_sql_injection", + "expressions.tests.FieldTransformTests.test_multiple_transforms_in_values", "model_fields.test_datetimefield.DateTimeFieldTests.test_lookup_date_with_use_tz", "model_fields.test_datetimefield.DateTimeFieldTests.test_lookup_date_without_use_tz", "timezones.tests.NewDatabaseTests.test_query_convert_timezones", diff --git a/django_mongodb/operations.py b/django_mongodb/operations.py index ffc53d387..78bbcf968 100644 --- a/django_mongodb/operations.py +++ b/django_mongodb/operations.py @@ -2,6 +2,7 @@ import json import re import uuid +from decimal import Decimal from bson.decimal128 import Decimal128 from django.conf import settings @@ -98,12 +99,22 @@ def convert_datetimefield_value(self, value, expression, connection): def convert_decimalfield_value(self, value, expression, connection): if value is not None: # from Decimal128 to decimal.Decimal() - value = value.to_decimal() + try: + value = value.to_decimal() + except AttributeError: + # `value` could be an integer in the case of an annotation + # like ExpressionWrapper(Value(1), output_field=DecimalField(). + return Decimal(value) return value def convert_durationfield_value(self, value, expression, connection): if value is not None: - value = datetime.timedelta(milliseconds=value) + try: + value = datetime.timedelta(milliseconds=value) + except TypeError: + # `value` could be Decimal128 if doing a computation with + # DurationField and Decimal128. + value = datetime.timedelta(milliseconds=int(str(value))) return value def convert_jsonfield_value(self, value, expression, connection): @@ -218,5 +229,8 @@ def datetime_cast_date_sql(self, sql, params, tzname): def datetime_cast_time_sql(self, sql, params, tzname): return f"({sql})::time", params + def format_for_duration_arithmetic(self, sql): + return "INTERVAL %s MILLISECOND" % sql + def time_trunc_sql(self, lookup_type, sql, params, tzname=None): return f"DATE_TRUNC(%s, {sql})::time", (lookup_type, *params)