From 3c24e10b66bb538689ee29c52ee36341b03d43fd Mon Sep 17 00:00:00 2001 From: Emanuel Lupi Date: Thu, 17 Oct 2024 14:10:00 -0400 Subject: [PATCH 1/2] improved readabilty of negated queries by using $not --- django_mongodb/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mongodb/query.py b/django_mongodb/query.py index fb732c011..e99500e14 100644 --- a/django_mongodb/query.py +++ b/django_mongodb/query.py @@ -269,7 +269,7 @@ def where_node(self, compiler, connection): raise FullResultSet if self.negated and mql: - mql = {"$eq": [mql, {"$literal": False}]} + mql = {"$not": mql} return mql From e9711203ac720f34ce14ecb53489828887c4ef64 Mon Sep 17 00:00:00 2001 From: Emanuel Lupi Date: Thu, 17 Oct 2024 11:27:28 -0400 Subject: [PATCH 2/2] add support for subqueries Subquery, Exists, and QuerySet as a lookup value. --- README.md | 3 - django_mongodb/compiler.py | 76 +++++++++----- django_mongodb/expressions.py | 95 ++++++++++++++++- django_mongodb/features.py | 186 +++++----------------------------- django_mongodb/lookups.py | 6 ++ django_mongodb/query.py | 32 +++++- django_mongodb/query_utils.py | 5 +- 7 files changed, 212 insertions(+), 191 deletions(-) diff --git a/README.md b/README.md index bdb7149fc..fe464a4cd 100644 --- a/README.md +++ b/README.md @@ -146,9 +146,6 @@ Congratulations, your project is ready to go! - `QuerySet.delete()` and `update()` do not support queries that span multiple collections. -- `Subquery`, `Exists`, and using a `QuerySet` in `QuerySet.annotate()` aren't - supported. - - `DateTimeField` doesn't support microsecond precision, and correspondingly, `DurationField` stores milliseconds rather than microseconds. diff --git a/django_mongodb/compiler.py b/django_mongodb/compiler.py index 282de7c79..d65ae608e 100644 --- a/django_mongodb/compiler.py +++ b/django_mongodb/compiler.py @@ -13,6 +13,7 @@ from django.db.models.lookups import IsNull from django.db.models.sql import compiler from django.db.models.sql.constants import GET_ITERATOR_CHUNK_SIZE, MULTI, SINGLE +from django.db.models.sql.datastructures import BaseTable from django.utils.functional import cached_property from pymongo import ASCENDING, DESCENDING @@ -25,12 +26,16 @@ class SQLCompiler(compiler.SQLCompiler): query_class = MongoQuery GROUP_SEPARATOR = "___" + PARENT_FIELD_TEMPLATE = "parent__field__{}" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.aggregation_pipeline = None + # Map columns to their subquery indices. + self.column_indices = {} # A list of OrderBy objects for this query. self.order_by_objs = None + self.subqueries = [] def _unfold_column(self, col): """ @@ -154,23 +159,40 @@ def _prepare_annotations_for_aggregation_pipeline(self, order_by): group.update(having_group) return group, replacements - def _get_group_id_expressions(self, order_by): - """Generate group ID expressions for the aggregation pipeline.""" - group_expressions = set() - replacements = {} + def _get_group_expressions(self, order_by): + if self.query.group_by is None: + return [] + seen = set() + expressions = set() + if self.query.group_by is not True: + # If group_by isn't True, then it's a list of expressions. + for expr in self.query.group_by: + if not hasattr(expr, "as_sql"): + expr = self.query.resolve_ref(expr) + if isinstance(expr, Ref): + if expr.refs not in seen: + seen.add(expr.refs) + expressions.add(expr.source) + else: + expressions.add(expr) + for expr, _, alias in self.select: + # Skip members that are already grouped. + if alias not in seen: + expressions |= set(expr.get_group_by_cols()) if not self._meta_ordering: for expr, (_, _, is_ref) in order_by: + # Skip references. if not is_ref: - group_expressions |= set(expr.get_group_by_cols()) - for expr, *_ in self.select: - group_expressions |= set(expr.get_group_by_cols()) + expressions |= set(expr.get_group_by_cols()) having_group_by = self.having.get_group_by_cols() if self.having else () for expr in having_group_by: - group_expressions.add(expr) - if isinstance(self.query.group_by, tuple | list): - group_expressions |= set(self.query.group_by) - elif self.query.group_by is None: - group_expressions = set() + expressions.add(expr) + return expressions + + def _get_group_id_expressions(self, order_by): + """Generate group ID expressions for the aggregation pipeline.""" + replacements = {} + group_expressions = self._get_group_expressions(order_by) if not group_expressions: ids = None else: @@ -186,6 +208,8 @@ def _get_group_id_expressions(self, order_by): ids[alias] = Value(True).as_mql(self, self.connection) if replacement is not None: replacements[col] = replacement + if isinstance(col, Ref): + replacements[col.source] = replacement return ids, replacements def _build_aggregation_pipeline(self, ids, group): @@ -228,15 +252,15 @@ def pre_sql_setup(self, with_col_aliases=False): all_replacements.update(replacements) pipeline = self._build_aggregation_pipeline(ids, group) if self.having: - pipeline.append( - { - "$match": { - "$expr": self.having.replace_expressions(all_replacements).as_mql( - self, self.connection - ) - } - } + having = self.having.replace_expressions(all_replacements).as_mql( + self, self.connection ) + # Add HAVING subqueries. + for query in self.subqueries or (): + pipeline.extend(query.get_pipeline()) + # Remove the added subqueries. + self.subqueries = [] + pipeline.append({"$match": {"$expr": having}}) self.aggregation_pipeline = pipeline self.annotations = { target: expr.replace_expressions(all_replacements) @@ -388,6 +412,7 @@ def build_query(self, columns=None): query.mongo_query = {"$expr": expr} if extra_fields: query.extra_fields = self.get_project_fields(extra_fields, force_expression=True) + query.subqueries = self.subqueries return query def get_columns(self): @@ -431,7 +456,12 @@ def project_field(column): @cached_property def collection_name(self): - return self.query.get_meta().db_table + base_table = next( + v + for k, v in self.query.alias_map.items() + if isinstance(v, BaseTable) and self.query.alias_refcount[k] + ) + return base_table.table_alias or base_table.table_name @cached_property def collection(self): @@ -581,7 +611,7 @@ def _get_ordering(self): return tuple(fields), sort_ordering, tuple(extra_fields) def get_where(self): - return self.where + return getattr(self, "where", self.query.where) def explain_query(self): # Validate format (none supported) and options. @@ -741,7 +771,7 @@ def build_query(self, columns=None): else None ) subquery = compiler.build_query(columns) - query.subquery = subquery + query.subqueries = [subquery] return query def _make_result(self, result, columns=None): diff --git a/django_mongodb/expressions.py b/django_mongodb/expressions.py index 7af3d71eb..a95e8b1c6 100644 --- a/django_mongodb/expressions.py +++ b/django_mongodb/expressions.py @@ -9,6 +9,7 @@ Case, Col, CombinedExpression, + Exists, ExpressionWrapper, F, NegatedExpression, @@ -50,6 +51,18 @@ def case(self, compiler, connection): def col(self, compiler, connection): # noqa: ARG001 + # If the column is part of a subquery and belongs to one of the parent + # queries, it will be stored for reference using $let in a $lookup stage. + if ( + self.alias not in compiler.query.alias_refcount + or compiler.query.alias_refcount[self.alias] == 0 + ): + try: + index = compiler.column_indices[self] + except KeyError: + index = len(compiler.column_indices) + compiler.column_indices[self] = index + return f"$${compiler.PARENT_FIELD_TEMPLATE.format(index)}" # Add the column's collection's alias for columns in joined collections. prefix = f"{self.alias}." if self.alias != compiler.collection_name else "" return f"${prefix}{self.target.column}" @@ -79,8 +92,73 @@ def order_by(self, compiler, connection): return self.expression.as_mql(compiler, connection) -def query(self, compiler, connection): # noqa: ARG001 - raise NotSupportedError("Using a QuerySet in annotate() is not supported on MongoDB.") +def query(self, compiler, connection, lookup_name=None): + subquery_compiler = self.get_compiler(connection=connection) + subquery_compiler.pre_sql_setup(with_col_aliases=False) + columns = subquery_compiler.get_columns() + field_name, expr = columns[0] + subquery = subquery_compiler.build_query( + columns + if subquery_compiler.query.annotations or not subquery_compiler.query.default_cols + else None + ) + table_output = f"__subquery{len(compiler.subqueries)}" + from_table = next( + e.table_name for alias, e in self.alias_map.items() if self.alias_refcount[alias] + ) + # To perform a subquery, a $lookup stage that escapsulates the entire + # subquery pipeline is added. The "let" clause defines the variables + # needed to bridge the main collection with the subquery. + subquery.subquery_lookup = { + "as": table_output, + "from": from_table, + "let": { + compiler.PARENT_FIELD_TEMPLATE.format(i): col.as_mql(compiler, connection) + for col, i in subquery_compiler.column_indices.items() + }, + } + # The result must be a list of values. The output is compressed with an + # aggregation pipeline. + if lookup_name in ("in", "range"): + if subquery.aggregation_pipeline is None: + subquery.aggregation_pipeline = [] + subquery.aggregation_pipeline.extend( + [ + { + "$facet": { + "group": [ + { + "$group": { + "_id": None, + "tmp_name": { + "$addToSet": expr.as_mql(subquery_compiler, connection) + }, + } + } + ] + } + }, + { + "$project": { + field_name: { + "$ifNull": [ + { + "$getField": { + "input": {"$arrayElemAt": ["$group", 0]}, + "field": "tmp_name", + } + }, + [], + ] + } + } + }, + ] + ) + # Erase project_fields since the required value is projected above. + subquery.project_fields = None + compiler.subqueries.append(subquery) + return f"${table_output}.{field_name}" def raw_sql(self, compiler, connection): # noqa: ARG001 @@ -100,8 +178,16 @@ def star(self, compiler, connection): # noqa: ARG001 return {"$literal": True} -def subquery(self, compiler, connection): # noqa: ARG001 - raise NotSupportedError(f"{self.__class__.__name__} is not supported on MongoDB.") +def subquery(self, compiler, connection, lookup_name=None): + return self.query.as_mql(compiler, connection, lookup_name=lookup_name) + + +def exists(self, compiler, connection, lookup_name=None): + try: + lhs_mql = subquery(self, compiler, connection, lookup_name=lookup_name) + except EmptyResultSet: + return Value(False).as_mql(compiler, connection) + return connection.mongo_operators["isnull"](lhs_mql, False) def when(self, compiler, connection): @@ -130,6 +216,7 @@ def register_expressions(): Case.as_mql = case Col.as_mql = col CombinedExpression.as_mql = combined_expression + Exists.as_mql = exists ExpressionWrapper.as_mql = expression_wrapper F.as_mql = f NegatedExpression.as_mql = negated_expression diff --git a/django_mongodb/features.py b/django_mongodb/features.py index 891f30cb7..8b0707434 100644 --- a/django_mongodb/features.py +++ b/django_mongodb/features.py @@ -10,6 +10,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): greatest_least_ignores_nulls = True has_json_object_function = False has_native_json_field = True + supports_boolean_expr_in_select_clause = True supports_collation_on_charfield = False supports_column_check_constraints = False supports_date_lookup_using_string = False @@ -91,6 +92,19 @@ class DatabaseFeatures(BaseDatabaseFeatures): "schema.tests.SchemaTests.test_composed_constraint_with_fk", "schema.tests.SchemaTests.test_remove_ignored_unique_constraint_not_create_fk_index", "schema.tests.SchemaTests.test_unique_constraint", + # Column default values aren't handled when a field raises + # EmptyResultSet: https://github.com/mongodb-labs/django-mongodb/issues/155 + "annotations.tests.NonAggregateAnnotationTestCase.test_empty_queryset_annotation", + "db_functions.comparison.test_coalesce.CoalesceTests.test_empty_queryset", + # Union as subquery is not mapping the parent parameter and collections: + # https://github.com/mongodb-labs/django-mongodb/issues/156 + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_in_subquery_related_outerref", + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_in_subquery", + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_in_with_ordering", + # ObjectId type mismatch in a subquery: + # https://github.com/mongodb-labs/django-mongodb/issues/161 + "queries.tests.RelatedLookupTypeTests.test_values_queryset_lookup", + "queries.tests.ValuesSubqueryTests.test_values_in_subquery", } # $bitAnd, #bitOr, and $bitXor are new in MongoDB 6.3. _django_test_expected_failures_bitwise = { @@ -204,167 +218,10 @@ def django_test_expected_failures(self): }, "Test assumes integer primary key.": { "db_functions.comparison.test_cast.CastTests.test_cast_to_integer_foreign_key", + "expressions.tests.BasicExpressionsTests.test_nested_subquery_outer_ref_with_autofield", "model_fields.test_foreignkey.ForeignKeyTests.test_to_python", "queries.test_qs_combinators.QuerySetSetOperationTests.test_order_raises_on_non_selected_column", }, - "Exists is not supported on MongoDB.": { - "aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_on_exists", - "aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_multiple_subquery_annotation", - "aggregation.tests.AggregateTestCase.test_aggregation_exists_multivalued_outeref", - "aggregation.tests.AggregateTestCase.test_group_by_exists_annotation", - "aggregation.tests.AggregateTestCase.test_exists_none_with_aggregate", - "aggregation.tests.AggregateTestCase.test_exists_extra_where_with_aggregate", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_exists_aggregate_values_chaining", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_exists_none_query", - "aggregation_regress.tests.AggregationTests.test_annotate_and_join", - "delete_regress.tests.DeleteTests.test_self_reference_with_through_m2m_at_second_level", - "expressions.tests.BasicExpressionsTests.test_annotation_with_deeply_nested_outerref", - "expressions.tests.BasicExpressionsTests.test_boolean_expression_combined", - "expressions.tests.BasicExpressionsTests.test_boolean_expression_combined_with_empty_Q", - "expressions.tests.BasicExpressionsTests.test_boolean_expression_in_Q", - "expressions.tests.BasicExpressionsTests.test_case_in_filter_if_boolean_output_field", - "expressions.tests.BasicExpressionsTests.test_exists_in_filter", - "expressions.tests.BasicExpressionsTests.test_order_by_exists", - "expressions.tests.BasicExpressionsTests.test_subquery", - "expressions.tests.ExistsTests.test_filter_by_empty_exists", - "expressions.tests.ExistsTests.test_negated_empty_exists", - "expressions.tests.ExistsTests.test_optimizations", - "expressions.tests.ExistsTests.test_select_negated_empty_exists", - "lookup.tests.LookupTests.test_exact_exists", - "lookup.tests.LookupTests.test_nested_outerref_lhs", - "lookup.tests.LookupQueryingTests.test_filter_exists_lhs", - "model_forms.tests.LimitChoicesToTests.test_fields_for_model_applies_limit_choices_to", - "model_forms.tests.LimitChoicesToTests.test_limit_choices_to_callable_for_fk_rel", - "model_forms.tests.LimitChoicesToTests.test_limit_choices_to_callable_for_m2m_rel", - "model_forms.tests.LimitChoicesToTests.test_limit_choices_to_m2m_through", - "model_forms.tests.LimitChoicesToTests.test_limit_choices_to_no_duplicates", - "null_queries.tests.NullQueriesTests.test_reverse_relations", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_values_list_on_annotated_and_unannotated", - "queries.tests.ExcludeTest17600.test_exclude_plain", - "queries.tests.ExcludeTest17600.test_exclude_with_q_is_equal_to_plain_exclude_variation", - "queries.tests.ExcludeTest17600.test_exclude_with_q_object_no_distinct", - "queries.tests.ExcludeTests.test_exclude_multivalued_exists", - "queries.tests.ExcludeTests.test_exclude_reverse_fk_field_ref", - "queries.tests.ExcludeTests.test_exclude_with_circular_fk_relation", - "queries.tests.ExcludeTests.test_subquery_exclude_outerref", - "queries.tests.ExcludeTests.test_to_field", - "queries.tests.ForeignKeyToBaseExcludeTests.test_ticket_21787", - "queries.tests.JoinReuseTest.test_inverted_q_across_relations", - "queries.tests.ManyToManyExcludeTest.test_exclude_many_to_many", - "queries.tests.ManyToManyExcludeTest.test_ticket_12823", - "queries.tests.Queries1Tests.test_double_exclude", - "queries.tests.Queries1Tests.test_exclude", - "queries.tests.Queries1Tests.test_exclude_in", - "queries.tests.Queries1Tests.test_excluded_intermediary_m2m_table_joined", - "queries.tests.Queries1Tests.test_nested_exclude", - "queries.tests.Queries4Tests.test_join_reuse_order", - "queries.tests.Queries4Tests.test_ticket24525", - "queries.tests.Queries6Tests.test_tickets_8921_9188", - "queries.tests.Queries6Tests.test_xor_subquery", - "queries.tests.QuerySetBitwiseOperationTests.test_subquery_aliases", - "queries.tests.TestTicket24605.test_ticket_24605", - "queries.tests.Ticket20101Tests.test_ticket_20101", - "queries.tests.Ticket20788Tests.test_ticket_20788", - "queries.tests.Ticket22429Tests.test_ticket_22429", - }, - "Subquery is not supported on MongoDB.": { - "aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_subquery_annotation", - "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_composed_subquery_requires_wrapping", - "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_subquery_requires_wrapping", - "aggregation.tests.AggregateTestCase.test_aggregation_nested_subquery_outerref", - "aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation", - "aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_multivalued", - "aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_related_field", - "aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_values", - "aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_values_collision", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_filter_with_subquery", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_and_aggregate_values_chaining", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_outerref_transform", - "annotations.tests.NonAggregateAnnotationTestCase.test_empty_queryset_annotation", - "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_outerref", - "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_subquery_with_parameters", - "expressions.tests.BasicExpressionsTests.test_aggregate_subquery_annotation", - "expressions.tests.BasicExpressionsTests.test_annotation_with_nested_outerref", - "expressions.tests.BasicExpressionsTests.test_annotation_with_outerref", - "expressions.tests.BasicExpressionsTests.test_annotations_within_subquery", - "expressions.tests.BasicExpressionsTests.test_in_subquery", - "expressions.tests.BasicExpressionsTests.test_nested_outerref_with_function", - "expressions.tests.BasicExpressionsTests.test_nested_subquery", - "expressions.tests.BasicExpressionsTests.test_nested_subquery_join_outer_ref", - "expressions.tests.BasicExpressionsTests.test_nested_subquery_outer_ref_2", - "expressions.tests.BasicExpressionsTests.test_nested_subquery_outer_ref_with_autofield", - "expressions.tests.BasicExpressionsTests.test_outerref_mixed_case_table_name", - "expressions.tests.BasicExpressionsTests.test_outerref_with_operator", - "expressions.tests.BasicExpressionsTests.test_subquery_filter_by_aggregate", - "expressions.tests.BasicExpressionsTests.test_subquery_filter_by_lazy", - "expressions.tests.BasicExpressionsTests.test_subquery_group_by_outerref_in_filter", - "expressions.tests.BasicExpressionsTests.test_subquery_in_filter", - "expressions.tests.BasicExpressionsTests.test_subquery_references_joined_table_twice", - "expressions.tests.BasicExpressionsTests.test_uuid_pk_subquery", - "lookup.tests.LookupQueryingTests.test_filter_subquery_lhs", - "model_fields.test_jsonfield.TestQuerying.test_nested_key_transform_on_subquery", - "model_fields.test_jsonfield.TestQuerying.test_obj_subquery_lookup", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_in_subquery", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_in_subquery_related_outerref", - }, - "Using a QuerySet in annotate() is not supported on MongoDB.": { - "aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_reused_subquery", - "aggregation.tests.AggregateTestCase.test_filter_in_subquery_or_aggregation", - "aggregation.tests.AggregateTestCase.test_group_by_subquery_annotation", - "aggregation.tests.AggregateTestCase.test_group_by_reference_subquery", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_and_alias_filter_in_subquery", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_and_alias_filter_related_in_subquery", - "annotations.tests.NonAggregateAnnotationTestCase.test_empty_expression_annotation", - "aggregation_regress.tests.AggregationTests.test_aggregates_in_where_clause", - "aggregation_regress.tests.AggregationTests.test_aggregates_in_where_clause_pre_eval", - "aggregation_regress.tests.AggregationTests.test_f_expression_annotation", - "aggregation_regress.tests.AggregationTests.test_having_subquery_select", - "aggregation_regress.tests.AggregationTests.test_more_more4", - "aggregation_regress.tests.AggregationTests.test_more_more_more5", - "aggregation_regress.tests.AggregationTests.test_negated_aggregation", - "db_functions.comparison.test_coalesce.CoalesceTests.test_empty_queryset", - "expressions.tests.FTimeDeltaTests.test_date_subquery_subtraction", - "expressions.tests.FTimeDeltaTests.test_datetime_subquery_subtraction", - "expressions.tests.FTimeDeltaTests.test_time_subquery_subtraction", - "expressions_case.tests.CaseExpressionTests.test_annotate_with_in_clause", - "expressions_case.tests.CaseExpressionTests.test_in_subquery", - "lookup.tests.LookupTests.test_exact_query_rhs_with_selected_columns", - "lookup.tests.LookupTests.test_exact_sliced_queryset_limit_one", - "lookup.tests.LookupTests.test_exact_sliced_queryset_limit_one_offset", - "lookup.tests.LookupTests.test_in_different_database", - "many_to_many.tests.ManyToManyTests.test_assign", - "many_to_many.tests.ManyToManyTests.test_assign_ids", - "many_to_many.tests.ManyToManyTests.test_clear", - "many_to_many.tests.ManyToManyTests.test_remove", - "many_to_many.tests.ManyToManyTests.test_reverse_assign_with_queryset", - "many_to_many.tests.ManyToManyTests.test_set", - "many_to_many.tests.ManyToManyTests.test_set_existing_different_type", - "many_to_one.tests.ManyToOneTests.test_get_prefetch_queryset_reverse_warning", - "model_fields.test_jsonfield.TestQuerying.test_usage_in_subquery", - "one_to_one.tests.OneToOneTests.test_get_prefetch_queryset_warning", - "one_to_one.tests.OneToOneTests.test_rel_pk_subquery", - "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_in_with_ordering", - "queries.tests.CloneTests.test_evaluated_queryset_as_argument", - "queries.tests.DoubleInSubqueryTests.test_double_subquery_in", - "queries.tests.EmptyQuerySetTests.test_values_subquery", - "queries.tests.ExcludeTests.test_exclude_subquery", - "queries.tests.NullInExcludeTest.test_null_in_exclude_qs", - "queries.tests.Queries1Tests.test_ticket9985", - "queries.tests.Queries1Tests.test_ticket9997", - "queries.tests.Queries1Tests.test_ticket10742", - "queries.tests.Queries4Tests.test_ticket10181", - "queries.tests.Queries5Tests.test_queryset_reuse", - "queries.tests.QuerySetBitwiseOperationTests.test_conflicting_aliases_during_combine", - "queries.tests.RelabelCloneTest.test_ticket_19964", - "queries.tests.RelatedLookupTypeTests.test_correct_lookup", - "queries.tests.RelatedLookupTypeTests.test_values_queryset_lookup", - "queries.tests.Ticket23605Tests.test_ticket_23605", - "queries.tests.ToFieldTests.test_in_subquery", - "queries.tests.ToFieldTests.test_nested_in_subquery", - "queries.tests.ValuesSubqueryTests.test_values_in_subquery", - "queries.tests.WeirdQuerysetSlicingTests.test_empty_sliced_subquery", - "queries.tests.WeirdQuerysetSlicingTests.test_empty_sliced_subquery_exclude", - }, "Cannot use QuerySet.delete() when querying across multiple collections on MongoDB.": { "delete.tests.FastDeleteTests.test_fast_delete_aggregation", "delete.tests.FastDeleteTests.test_fast_delete_empty_no_update_can_self_select", @@ -376,6 +233,16 @@ def django_test_expected_failures(self): "delete_regress.tests.Ticket19102Tests.test_ticket_19102_select_related", "one_to_one.tests.OneToOneTests.test_o2o_primary_key_delete", }, + "Cannot use QuerySet.delete() when a subquery is required.": { + "delete_regress.tests.DeleteTests.test_self_reference_with_through_m2m_at_second_level", + "many_to_many.tests.ManyToManyTests.test_assign", + "many_to_many.tests.ManyToManyTests.test_assign_ids", + "many_to_many.tests.ManyToManyTests.test_clear", + "many_to_many.tests.ManyToManyTests.test_remove", + "many_to_many.tests.ManyToManyTests.test_reverse_assign_with_queryset", + "many_to_many.tests.ManyToManyTests.test_set", + "many_to_many.tests.ManyToManyTests.test_set_existing_different_type", + }, "Cannot use QuerySet.update() when querying across multiple collections on MongoDB.": { "expressions.tests.BasicExpressionsTests.test_filter_with_join", "queries.tests.Queries4Tests.test_ticket7095", @@ -417,6 +284,7 @@ def django_test_expected_failures(self): "ordering.tests.OrderingTests.test_orders_nulls_first_on_filtered_subquery", "queries.tests.ExcludeTest17600.test_exclude_plain_distinct", "queries.tests.ExcludeTest17600.test_exclude_with_q_is_equal_to_plain_exclude", + "queries.tests.ExcludeTest17600.test_exclude_with_q_is_equal_to_plain_exclude_variation", "queries.tests.ExcludeTest17600.test_exclude_with_q_object_distinct", "queries.tests.ExcludeTests.test_exclude_m2m_through", "queries.tests.ExistsSql.test_distinct_exists", @@ -433,6 +301,7 @@ def django_test_expected_failures(self): "update.tests.AdvancedTests.test_update_all", }, "QuerySet.extra() is not supported.": { + "aggregation.tests.AggregateTestCase.test_exists_extra_where_with_aggregate", "annotations.tests.NonAggregateAnnotationTestCase.test_column_field_ordering", "annotations.tests.NonAggregateAnnotationTestCase.test_column_field_ordering_with_deferred", "basic.tests.ModelTest.test_extra_method_select_argument_with_dashes", @@ -468,6 +337,7 @@ def django_test_expected_failures(self): "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_aggregate_annotation_kept", "aggregation.tests.AggregateTestCase.test_count_star", "delete.tests.DeletionTests.test_only_referenced_fields_selected", + "expressions.tests.ExistsTests.test_optimizations", "lookup.tests.LookupTests.test_in_ignore_none", "lookup.tests.LookupTests.test_textfield_exact_null", "migrations.test_commands.MigrateTests.test_migrate_syncdb_app_label", diff --git a/django_mongodb/lookups.py b/django_mongodb/lookups.py index 503edb24a..c651dd6af 100644 --- a/django_mongodb/lookups.py +++ b/django_mongodb/lookups.py @@ -36,6 +36,12 @@ def field_resolve_expression_parameter(self, compiler, connection, sql, param): def in_(self, compiler, connection): if isinstance(self.lhs, MultiColSource): raise NotImplementedError("MultiColSource is not supported.") + db_rhs = getattr(self.rhs, "_db", None) + if db_rhs is not None and db_rhs != connection.alias: + raise ValueError( + "Subqueries aren't allowed across different databases. Force " + "the inner query to be evaluated using `list(inner_query)`." + ) return builtin_lookup(self, compiler, connection) diff --git a/django_mongodb/query.py b/django_mongodb/query.py index e99500e14..1d7dd951d 100644 --- a/django_mongodb/query.py +++ b/django_mongodb/query.py @@ -50,12 +50,15 @@ def __init__(self, compiler): self.collection = self.compiler.collection self.collection_name = self.compiler.collection_name self.mongo_query = getattr(compiler.query, "raw_query", {}) - self.subquery = None + self.subqueries = None self.lookup_pipeline = None self.project_fields = None self.aggregation_pipeline = compiler.aggregation_pipeline self.extra_fields = None self.combinator_pipeline = None + # $lookup stage that encapsulates the pipeline for performing a nested + # subquery. + self.subquery_lookup = None def __repr__(self): return f"" @@ -63,6 +66,8 @@ def __repr__(self): @wrap_database_errors def delete(self): """Execute a delete query.""" + if self.compiler.subqueries: + raise NotSupportedError("Cannot use QuerySet.delete() when a subquery is required.") return self.collection.delete_many(self.mongo_query).deleted_count @wrap_database_errors @@ -74,9 +79,11 @@ def get_cursor(self): return self.collection.aggregate(self.get_pipeline()) def get_pipeline(self): - pipeline = self.subquery.get_pipeline() if self.subquery else [] + pipeline = [] if self.lookup_pipeline: pipeline.extend(self.lookup_pipeline) + for query in self.subqueries or (): + pipeline.extend(query.get_pipeline()) if self.mongo_query: pipeline.append({"$match": self.mongo_query}) if self.aggregation_pipeline: @@ -93,6 +100,27 @@ def get_pipeline(self): pipeline.append({"$skip": self.query.low_mark}) if self.query.high_mark is not None: pipeline.append({"$limit": self.query.high_mark - self.query.low_mark}) + if self.subquery_lookup: + table_output = self.subquery_lookup["as"] + pipeline = [ + {"$lookup": {**self.subquery_lookup, "pipeline": pipeline}}, + { + "$set": { + table_output: { + "$cond": { + "if": { + "$or": [ + {"$eq": [{"$type": f"${table_output}"}, "missing"]}, + {"$eq": [{"$size": f"${table_output}"}, 0]}, + ] + }, + "then": {}, + "else": {"$arrayElemAt": [f"${table_output}", 0]}, + } + } + } + }, + ] return pipeline diff --git a/django_mongodb/query_utils.py b/django_mongodb/query_utils.py index fe3fed9b8..ff98a1ed5 100644 --- a/django_mongodb/query_utils.py +++ b/django_mongodb/query_utils.py @@ -28,7 +28,10 @@ def process_lhs(node, compiler, connection): def process_rhs(node, compiler, connection): rhs = node.rhs if hasattr(rhs, "as_mql"): - value = rhs.as_mql(compiler, connection) + if getattr(rhs, "subquery", False): + value = rhs.as_mql(compiler, connection, lookup_name=node.lookup_name) + else: + value = rhs.as_mql(compiler, connection) else: _, value = node.process_rhs(compiler, connection) lookup_name = node.lookup_name