diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 6634bf5fc..ca4c91999 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -89,10 +89,19 @@ jobs: defer_regress from_db_value lookup + m2m_and_m2o + m2m_intermediary + m2m_multiple + m2m_recursive + m2m_regress + m2m_signals + m2m_through + m2o_recursive model_fields ordering or_lookups queries.tests.Ticket12807Tests.test_ticket_12807 + select_related sessions_tests timezones update diff --git a/README.md b/README.md index fe2785b9b..8e0c45f35 100644 --- a/README.md +++ b/README.md @@ -114,13 +114,10 @@ Migrations for 'admin': - `datetimes()` - `distinct()` - `extra()` - - `select_related()` - `Subquery`, `Exists`, and using a `QuerySet` in `QuerySet.annotate()` aren't supported. -- Queries with joins aren't supported. - - `DateTimeField` doesn't support microsecond precision, and correspondingly, `DurationField` stores milliseconds rather than microseconds. diff --git a/django_mongodb/compiler.py b/django_mongodb/compiler.py index 8e9747feb..fc5b3b442 100644 --- a/django_mongodb/compiler.py +++ b/django_mongodb/compiler.py @@ -5,6 +5,7 @@ from django.db.models.constants import LOOKUP_SEP from django.db.models.sql import compiler from django.db.models.sql.constants import GET_ITERATOR_CHUNK_SIZE, MULTI +from django.utils.functional import cached_property from .base import Cursor from .query import MongoQuery, wrap_database_errors @@ -82,7 +83,15 @@ def _make_result(self, entity, columns, converters, tuple_expected=False): result = [] for name, col in columns: field = col.field - value = entity.get(name, NOT_PROVIDED) + column_alias = getattr(col, "alias", None) + obj = ( + # Use the related object... + entity.get(column_alias, {}) + # ...if this column refers to an object for select_related(). + if column_alias is not None and column_alias != self.collection_name + else entity + ) + value = obj.get(name, NOT_PROVIDED) if value is NOT_PROVIDED: value = field.get_default() elif converters: @@ -110,10 +119,6 @@ def check_query(self): raise NotSupportedError("QuerySet.distinct() is not supported on MongoDB.") if self.query.extra: raise NotSupportedError("QuerySet.extra() is not supported on MongoDB.") - if self.query.select_related: - raise NotSupportedError("QuerySet.select_related() is not supported on MongoDB.") - if len([a for a in self.query.alias_map if self.query.alias_refcount[a]]) > 1: - raise NotSupportedError("Queries with multiple tables are not supported on MongoDB.") if any( isinstance(a, Aggregate) and not isinstance(a, Count) for a in self.query.annotations.values() @@ -147,6 +152,7 @@ def build_query(self, columns=None): self.check_query() self.setup_query() query = self.query_class(self, columns) + query.lookup_pipeline = self.get_lookup_pipeline() try: query.mongo_query = {"$expr": self.query.where.as_mql(self, self.connection)} except FullResultSet: @@ -163,9 +169,17 @@ def get_columns(self): columns = ( self.get_default_columns(select_mask) if self.query.default_cols else self.query.select ) + # Populate QuerySet.select_related() data. + related_columns = [] + if self.query.select_related: + self.get_related_selections(related_columns, select_mask) + if related_columns: + related_columns, _ = zip(*related_columns, strict=True) + annotation_idx = 1 - result = [] - for column in columns: + + def project_field(column): + nonlocal annotation_idx if hasattr(column, "target"): # column is a Col. target = column.target.column @@ -174,8 +188,13 @@ def get_columns(self): # name for $proj. target = f"__annotation{annotation_idx}" annotation_idx += 1 - result.append((target, column)) - return tuple(result) + tuple(self.query.annotation_select.items()) + return target, column + + return ( + tuple(map(project_field, columns)) + + tuple(self.query.annotation_select.items()) + + tuple(map(project_field, related_columns)) + ) def _get_ordering(self): """ @@ -212,8 +231,20 @@ def _get_ordering(self): field_ordering.append((opts.get_field(name), ascending)) return field_ordering + @cached_property + def collection_name(self): + return self.query.get_meta().db_table + def get_collection(self): - return self.connection.get_collection(self.query.get_meta().db_table) + return self.connection.get_collection(self.collection_name) + + def get_lookup_pipeline(self): + result = [] + for alias in tuple(self.query.alias_map): + if not self.query.alias_refcount[alias] or self.collection_name == alias: + continue + result += self.query.alias_map[alias].as_mql(self, self.connection) + return result class SQLInsertCompiler(SQLCompiler): diff --git a/django_mongodb/expressions.py b/django_mongodb/expressions.py index d00c3e295..9dc216643 100644 --- a/django_mongodb/expressions.py +++ b/django_mongodb/expressions.py @@ -43,7 +43,9 @@ def case(self, compiler, connection): def col(self, compiler, connection): # noqa: ARG001 - return f"${self.target.column}" + # Add the column's collection's alias for columns in joined collections. + prefix = f"{self.alias}." if self.alias != compiler.collection_name else "" + return f"${prefix}{self.target.column}" def combined_expression(self, compiler, connection): diff --git a/django_mongodb/features.py b/django_mongodb/features.py index 28ee01973..f015bee66 100644 --- a/django_mongodb/features.py +++ b/django_mongodb/features.py @@ -34,12 +34,31 @@ class DatabaseFeatures(BaseDatabaseFeatures): "db_functions.text.test_strindex.StrIndexTests.test_order_by", "expressions_case.tests.CaseExpressionTests.test_order_by_conditional_explicit", "lookup.tests.LookupQueryingTests.test_lookup_in_order_by", + "ordering.tests.OrderingTests.test_default_ordering", "ordering.tests.OrderingTests.test_default_ordering_by_f_expression", "ordering.tests.OrderingTests.test_default_ordering_does_not_affect_group_by", + "ordering.tests.OrderingTests.test_order_by_expr_query_reuse", "ordering.tests.OrderingTests.test_order_by_expression_ref", "ordering.tests.OrderingTests.test_order_by_f_expression", "ordering.tests.OrderingTests.test_order_by_f_expression_duplicates", + "ordering.tests.OrderingTests.test_order_by_fk_attname", + "ordering.tests.OrderingTests.test_order_by_nulls_first", + "ordering.tests.OrderingTests.test_order_by_nulls_last", + "ordering.tests.OrderingTests.test_ordering_select_related_collision", + "ordering.tests.OrderingTests.test_order_by_self_referential_fk", + "ordering.tests.OrderingTests.test_orders_nulls_first_on_filtered_subquery", + "ordering.tests.OrderingTests.test_related_ordering_duplicate_table_reference", "ordering.tests.OrderingTests.test_reverse_ordering_pure", + "ordering.tests.OrderingTests.test_reverse_meta_ordering_pure", + "ordering.tests.OrderingTests.test_reversed_ordering", + "update.tests.AdvancedTests.test_update_ordered_by_inline_m2m_annotation", + "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation", + "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation_desc", + # 'ManyToOneRel' object has no attribute 'column' + "m2m_through.tests.M2mThroughTests.test_order_by_relational_field_through_model", + # pymongo: ValueError: update cannot be empty + "update.tests.SimpleTest.test_empty_update_with_inheritance", + "update.tests.SimpleTest.test_nonempty_update_with_inheritance", # Pattern lookups that use regexMatch don't work on JSONField: # Unsupported conversion from array to string in $convert "model_fields.test_jsonfield.TestQuerying.test_icontains", @@ -59,6 +78,19 @@ class DatabaseFeatures(BaseDatabaseFeatures): "model_fields.test_jsonfield.TestQuerying.test_order_grouping_custom_decoder", "model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform", "model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_key_transform", + # DecimalField lookup with F expression crashes: + # decimal.InvalidOperation: [] + "lookup.tests.LookupTests.test_lookup_rhs", + # Wrong results in queries with multiple tables. + "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_aggregate_with_m2o", + "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_reverse_m2m", + "annotations.tests.NonAggregateAnnotationTestCase.test_chaining_annotation_filter_with_m2m", + "lookup.tests.LookupTests.test_lookup_collision", + "expressions_case.tests.CaseExpressionTests.test_join_promotion", + "expressions_case.tests.CaseExpressionTests.test_join_promotion_multiple_annotations", + "ordering.tests.OrderingTests.test_order_by_grandparent_fk_with_expression_in_default_ordering", + "ordering.tests.OrderingTests.test_order_by_parent_fk_with_expression_in_default_ordering", + "ordering.tests.OrderingTests.test_order_by_ptr_field_with_default_ordering_by_expression", } # $bitAnd, #bitOr, and $bitXor are new in MongoDB 6.3. _django_test_expected_failures_bitwise = { @@ -145,30 +177,12 @@ def django_test_expected_failures(self): }, "AutoField not supported.": { "bulk_create.tests.BulkCreateTests.test_bulk_insert_nullable_fields", + "lookup.tests.LookupTests.test_filter_by_reverse_related_field_transform", "lookup.tests.LookupTests.test_in_ignore_none_with_unhashable_items", "model_fields.test_autofield.AutoFieldTests", "model_fields.test_autofield.BigAutoFieldTests", "model_fields.test_autofield.SmallAutoFieldTests", }, - "QuerySet.select_related() not supported.": { - "annotations.tests.AliasTests.test_joined_alias_annotation", - "annotations.tests.NonAggregateAnnotationTestCase.test_joined_annotation", - "defer.tests.DeferTests.test_defer_foreign_keys_are_deferred_and_not_traversed", - "defer.tests.DeferTests.test_defer_with_select_related", - "defer.tests.DeferTests.test_only_with_select_related", - "defer.tests.TestDefer2.test_defer_proxy", - "defer_regress.tests.DeferRegressionTest.test_basic", - "defer_regress.tests.DeferRegressionTest.test_common_model_different_mask", - "model_fields.test_booleanfield.BooleanFieldTests.test_select_related", - "model_fields.test_foreignkey.ForeignKeyTests.test_empty_string_fk", - "defer_regress.tests.DeferRegressionTest.test_defer_annotate_select_related", - "defer_regress.tests.DeferRegressionTest.test_defer_with_select_related", - "defer_regress.tests.DeferRegressionTest.test_only_with_select_related", - "defer_regress.tests.DeferRegressionTest.test_proxy_model_defer_with_select_related", - "defer_regress.tests.DeferRegressionTest.test_reverse_one_to_one_relations", - "defer_regress.tests.DeferRegressionTest.test_ticket_23270", - "ordering.tests.OrderingTests.test_ordering_select_related_collision", - }, "MongoDB does not enforce UNIQUE constraints.": { "auth_tests.test_basic.BasicTestCase.test_unicode_username", "auth_tests.test_migrations.ProxyModelWithSameAppLabelTests.test_migrate_with_existing_target_permission", @@ -192,6 +206,7 @@ def django_test_expected_failures(self): }, # https://github.com/mongodb-labs/django-mongodb/issues/12 "QuerySet.aggregate() not supported.": { + "annotations.tests.AliasTests.test_alias_default_alias_expression", "annotations.tests.AliasTests.test_filter_alias_agg_with_double_f", "annotations.tests.NonAggregateAnnotationTestCase.test_aggregate_over_annotation", "annotations.tests.NonAggregateAnnotationTestCase.test_aggregate_over_full_expression_annotation", @@ -199,14 +214,27 @@ def django_test_expected_failures(self): "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_in_f_grouped_by_annotation", "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_and_aggregate_values_chaining", "annotations.tests.NonAggregateAnnotationTestCase.test_filter_agg_with_double_f", + "annotations.tests.NonAggregateAnnotationTestCase.test_values_with_pk_annotation", "expressions_case.tests.CaseExpressionTests.test_aggregate", "expressions_case.tests.CaseExpressionTests.test_aggregate_with_expression_as_condition", "expressions_case.tests.CaseExpressionTests.test_aggregate_with_expression_as_value", "expressions_case.tests.CaseExpressionTests.test_aggregation_empty_cases", + "expressions_case.tests.CaseExpressionTests.test_annotate_with_aggregation_in_condition", + "expressions_case.tests.CaseExpressionTests.test_annotate_with_aggregation_in_predicate", + "expressions_case.tests.CaseExpressionTests.test_annotate_with_aggregation_in_value", + "expressions_case.tests.CaseExpressionTests.test_annotate_with_in_clause", + "expressions_case.tests.CaseExpressionTests.test_filter_with_aggregation_in_condition", + "expressions_case.tests.CaseExpressionTests.test_filter_with_aggregation_in_predicate", + "expressions_case.tests.CaseExpressionTests.test_filter_with_aggregation_in_value", + "expressions_case.tests.CaseExpressionTests.test_m2m_exclude", + "expressions_case.tests.CaseExpressionTests.test_m2m_reuse", + "lookup.test_decimalfield.DecimalFieldLookupTests", "lookup.tests.LookupQueryingTests.test_aggregate_combined_lookup", "from_db_value.tests.FromDBValueTest.test_aggregation", "timezones.tests.LegacyDatabaseTests.test_query_aggregation", + "timezones.tests.LegacyDatabaseTests.test_query_annotation", "timezones.tests.NewDatabaseTests.test_query_aggregation", + "timezones.tests.NewDatabaseTests.test_query_annotation", }, "QuerySet.annotate() has some limitations.": { # annotate() with combined expressions doesn't work: @@ -220,6 +248,9 @@ def django_test_expected_failures(self): "annotations.tests.NonAggregateAnnotationTestCase.test_full_expression_annotation_with_aggregation", "annotations.tests.NonAggregateAnnotationTestCase.test_grouping_by_q_expression_annotation", "annotations.tests.NonAggregateAnnotationTestCase.test_q_expression_annotation_with_aggregation", + "defer_regress.tests.DeferRegressionTest.test_basic", + "defer_regress.tests.DeferRegressionTest.test_defer_annotate_select_related", + "defer_regress.tests.DeferRegressionTest.test_ticket_16409", "expressions_case.tests.CaseDocumentationExamples.test_conditional_aggregation_example", # Func not implemented. "annotations.tests.NonAggregateAnnotationTestCase.test_custom_functions", @@ -228,6 +259,8 @@ def django_test_expected_failures(self): "annotations.tests.NonAggregateAnnotationTestCase.test_mixed_type_annotation_date_interval", # FieldDoesNotExist with ordering. "annotations.tests.AliasTests.test_order_by_alias", + "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_with_m2m", + "annotations.tests.NonAggregateAnnotationTestCase.test_mti_annotations", "annotations.tests.NonAggregateAnnotationTestCase.test_order_by_aggregate", "annotations.tests.NonAggregateAnnotationTestCase.test_order_by_annotation", "expressions.tests.NegatedExpressionTests.test_filter", @@ -243,6 +276,8 @@ def django_test_expected_failures(self): "lookup.tests.LookupQueryingTests.test_filter_exists_lhs", }, "Subquery is not supported on MongoDB.": { + "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_filter_with_subquery", + "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_outerref_transform", "annotations.tests.NonAggregateAnnotationTestCase.test_empty_queryset_annotation", "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_outerref", "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_subquery_with_parameters", @@ -252,6 +287,7 @@ def django_test_expected_failures(self): }, "Using a QuerySet in annotate() is not supported on MongoDB.": { "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_and_alias_filter_in_subquery", + "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_and_alias_filter_related_in_subquery", "annotations.tests.NonAggregateAnnotationTestCase.test_empty_expression_annotation", "db_functions.comparison.test_coalesce.CoalesceTests.test_empty_queryset", "expressions_case.tests.CaseExpressionTests.test_in_subquery", @@ -297,80 +333,10 @@ def django_test_expected_failures(self): "ordering.tests.OrderingTests.test_extra_ordering", "ordering.tests.OrderingTests.test_extra_ordering_quoting", "ordering.tests.OrderingTests.test_extra_ordering_with_table_name", + "select_related.tests.SelectRelatedTests.test_select_related_with_extra", }, - "Queries with multiple tables are not supported.": { - "annotations.tests.AliasTests.test_alias_default_alias_expression", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_aggregate_with_m2o", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_and_alias_filter_related_in_subquery", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_filter_with_subquery", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_reverse_m2m", - "annotations.tests.NonAggregateAnnotationTestCase.test_joined_transformed_annotation", - "annotations.tests.NonAggregateAnnotationTestCase.test_mti_annotations", - "annotations.tests.NonAggregateAnnotationTestCase.test_values_with_pk_annotation", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_subquery_outerref_transform", - "annotations.tests.NonAggregateAnnotationTestCase.test_annotation_with_m2m", - "annotations.tests.NonAggregateAnnotationTestCase.test_chaining_annotation_filter_with_m2m", - "db_functions.comparison.test_least.LeastTests.test_related_field", - "db_functions.comparison.test_greatest.GreatestTests.test_related_field", - "defer.tests.BigChildDeferTests.test_defer_baseclass_when_subclass_has_added_field", - "defer.tests.BigChildDeferTests.test_defer_subclass", - "defer.tests.BigChildDeferTests.test_defer_subclass_both", - "defer.tests.BigChildDeferTests.test_only_baseclass_when_subclass_has_added_field", - "defer.tests.BigChildDeferTests.test_only_subclass", - "defer.tests.DeferTests.test_defer_baseclass_when_subclass_has_no_added_fields", - "defer.tests.DeferTests.test_defer_of_overridden_scalar", - "defer.tests.DeferTests.test_only_baseclass_when_subclass_has_no_added_fields", - "defer.tests.TestDefer2.test_defer_inheritance_pk_chaining", - "defer_regress.tests.DeferRegressionTest.test_ticket_16409", - "expressions_case.tests.CaseExpressionTests.test_annotate_with_aggregation_in_condition", - "expressions_case.tests.CaseExpressionTests.test_annotate_with_aggregation_in_predicate", - "expressions_case.tests.CaseExpressionTests.test_annotate_with_aggregation_in_value", - "expressions_case.tests.CaseExpressionTests.test_annotate_with_in_clause", - "expressions_case.tests.CaseExpressionTests.test_annotate_with_join_in_condition", - "expressions_case.tests.CaseExpressionTests.test_annotate_with_join_in_predicate", - "expressions_case.tests.CaseExpressionTests.test_annotate_with_join_in_value", - "expressions_case.tests.CaseExpressionTests.test_filter_with_aggregation_in_condition", - "expressions_case.tests.CaseExpressionTests.test_filter_with_aggregation_in_predicate", - "expressions_case.tests.CaseExpressionTests.test_filter_with_aggregation_in_value", - "expressions_case.tests.CaseExpressionTests.test_filter_with_join_in_condition", - "expressions_case.tests.CaseExpressionTests.test_filter_with_join_in_predicate", - "expressions_case.tests.CaseExpressionTests.test_filter_with_join_in_value", - "expressions_case.tests.CaseExpressionTests.test_join_promotion", - "expressions_case.tests.CaseExpressionTests.test_join_promotion_multiple_annotations", - "expressions_case.tests.CaseExpressionTests.test_m2m_exclude", - "expressions_case.tests.CaseExpressionTests.test_m2m_reuse", - "lookup.test_decimalfield.DecimalFieldLookupTests", - "lookup.tests.LookupQueryingTests.test_multivalued_join_reuse", - "lookup.tests.LookupTests.test_filter_by_reverse_related_field_transform", - "lookup.tests.LookupTests.test_lookup_collision", - "lookup.tests.LookupTests.test_lookup_rhs", - "lookup.tests.LookupTests.test_isnull_non_boolean_value", - "model_fields.test_jsonfield.TestQuerying.test_join_key_transform_annotation_expression", - "model_fields.test_manytomanyfield.ManyToManyFieldDBTests.test_value_from_object_instance_with_pk", - "model_fields.test_uuid.TestAsPrimaryKey.test_two_level_foreign_keys", - "ordering.tests.OrderingTests.test_default_ordering", - "ordering.tests.OrderingTests.test_order_by_expr_query_reuse", - "ordering.tests.OrderingTests.test_order_by_fk_attname", - "ordering.tests.OrderingTests.test_order_by_grandparent_fk_with_expression_in_default_ordering", - "ordering.tests.OrderingTests.test_order_by_nulls_first", - "ordering.tests.OrderingTests.test_order_by_nulls_last", - "ordering.tests.OrderingTests.test_order_by_parent_fk_with_expression_in_default_ordering", - "ordering.tests.OrderingTests.test_order_by_ptr_field_with_default_ordering_by_expression", - "ordering.tests.OrderingTests.test_order_by_self_referential_fk", - "ordering.tests.OrderingTests.test_orders_nulls_first_on_filtered_subquery", - "ordering.tests.OrderingTests.test_related_ordering_duplicate_table_reference", - "ordering.tests.OrderingTests.test_reverse_meta_ordering_pure", - "ordering.tests.OrderingTests.test_reversed_ordering", - "timezones.tests.LegacyDatabaseTests.test_query_annotation", - "timezones.tests.NewDatabaseTests.test_query_annotation", + "QuerySet.update() crash: Unrecognized expression '$count'": { "update.tests.AdvancedTests.test_update_annotated_multi_table_queryset", - "update.tests.AdvancedTests.test_update_fk", - "update.tests.AdvancedTests.test_update_ordered_by_inline_m2m_annotation", - "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation", - "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation_desc", - "update.tests.SimpleTest.test_empty_update_with_inheritance", - "update.tests.SimpleTest.test_foreign_key_update_with_id", - "update.tests.SimpleTest.test_nonempty_update_with_inheritance", }, "Test inspects query for SQL": { "lookup.tests.LookupTests.test_in_ignore_none", diff --git a/django_mongodb/query.py b/django_mongodb/query.py index fd7974421..e56016868 100644 --- a/django_mongodb/query.py +++ b/django_mongodb/query.py @@ -3,6 +3,8 @@ from django.core.exceptions import EmptyResultSet, FullResultSet from django.db import DatabaseError, IntegrityError from django.db.models import Value +from django.db.models.sql.constants import INNER +from django.db.models.sql.datastructures import Join from django.db.models.sql.where import AND, XOR, WhereNode from pymongo import ASCENDING, DESCENDING from pymongo.errors import DuplicateKeyError, PyMongoError @@ -41,7 +43,9 @@ def __init__(self, compiler, columns): self._negated = False self.ordering = [] self.collection = self.compiler.get_collection() + self.collection_name = self.compiler.collection_name self.mongo_query = getattr(compiler.query, "raw_query", {}) + self.lookup_pipeline = None def __repr__(self): return f"" @@ -106,7 +110,15 @@ def get_cursor(self): # If name != column, then this is an annotatation referencing # another column. fields[name] = 1 if name == column else f"${column}" + if fields: + # Add related fields. + for alias in self.query.alias_map: + if self.query.alias_refcount[alias] and self.collection_name != alias: + fields[alias] = 1 + # Construct the query pipeline. pipeline = [] + if self.lookup_pipeline: + pipeline.extend(self.lookup_pipeline) if self.mongo_query: pipeline.append({"$match": self.mongo_query}) if fields: @@ -120,6 +132,83 @@ def get_cursor(self): return self.collection.aggregate(pipeline) +def join(self, compiler, connection): + lookup_pipeline = [] + lhs_fields = [] + rhs_fields = [] + # Add a join condition for each pair of joining fields. + for lhs, rhs in self.join_fields: + lhs, rhs = connection.ops.prepare_join_on_clause( + self.parent_alias, lhs, self.table_name, rhs + ) + lhs_fields.append(lhs.as_mql(compiler, connection)) + # In the lookup stage, the reference to this column doesn't include + # the collection name. + rhs_fields.append(rhs.as_mql(compiler, connection).replace(f"{self.table_name}.", "", 1)) + + parent_template = "parent__field__" + lookup_pipeline = [ + { + "$lookup": { + # The right-hand table to join. + "from": self.table_name, + # The pipeline variables to be matched in the pipeline's + # expression. + "let": { + f"{parent_template}{i}": parent_field + for i, parent_field in enumerate(lhs_fields) + }, + "pipeline": [ + { + # Match the conditions: + # self.table_name.field1 = parent_table.field1 + # AND + # self.table_name.field2 = parent_table.field2 + # AND + # ... + "$match": { + "$expr": { + "$and": [ + {"$eq": [f"$${parent_template}{i}", field]} + for i, field in enumerate(rhs_fields) + ] + } + } + } + ], + # Rename the output as table_alias. + "as": self.table_alias, + } + }, + ] + # To avoid missing data when using $unwind, an empty collection is added if + # the join isn't an inner join. For inner joins, rows with empty arrays are + # removed, as $unwind unrolls or unnests the array and removes the row if + # it's empty. This is the expected behavior for inner joins. For left outer + # joins (LOUTER), however, an empty collection is returned. + if self.join_type != INNER: + lookup_pipeline.append( + { + "$set": { + self.table_alias: { + "$cond": { + "if": { + "$or": [ + {"$eq": [{"$type": f"${self.table_alias}"}, "missing"]}, + {"$eq": [{"$size": f"${self.table_alias}"}, 0]}, + ] + }, + "then": [{}], + "else": f"${self.table_alias}", + } + } + } + } + ) + lookup_pipeline.append({"$unwind": f"${self.table_alias}"}) + return lookup_pipeline + + def where_node(self, compiler, connection): if self.connector == AND: full_needed, empty_needed = len(self.children), 1 @@ -170,4 +259,5 @@ def where_node(self, compiler, connection): def register_nodes(): + Join.as_mql = join WhereNode.as_mql = where_node