diff --git a/django_mongodb_backend/query.py b/django_mongodb_backend/query.py index 8bad9346..865b4ae1 100644 --- a/django_mongodb_backend/query.py +++ b/django_mongodb_backend/query.py @@ -11,6 +11,8 @@ from django.db.models.sql.where import AND, OR, XOR, ExtraWhere, NothingNode, WhereNode from pymongo.errors import BulkWriteError, DuplicateKeyError, PyMongoError +from django_mongodb_backend.query_conversion.query_optimizer import QueryOptimizer + def wrap_database_errors(func): @wraps(func) @@ -55,6 +57,7 @@ def __init__(self, compiler): # $lookup stage that encapsulates the pipeline for performing a nested # subquery. self.subquery_lookup = None + self.query_optimizer = QueryOptimizer() def __repr__(self): return f"" @@ -87,7 +90,7 @@ def get_pipeline(self): for query in self.subqueries or (): pipeline.extend(query.get_pipeline()) if self.match_mql: - pipeline.append({"$match": self.match_mql}) + pipeline.extend(self.query_optimizer.convert_expr_to_match(self.match_mql)) if self.aggregation_pipeline: pipeline.extend(self.aggregation_pipeline) if self.project_fields: diff --git a/django_mongodb_backend/query_conversion/__init__.py b/django_mongodb_backend/query_conversion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/django_mongodb_backend/query_conversion/expression_converters.py b/django_mongodb_backend/query_conversion/expression_converters.py new file mode 100644 index 00000000..2564058f --- /dev/null +++ b/django_mongodb_backend/query_conversion/expression_converters.py @@ -0,0 +1,129 @@ +"""Expression To Match Converters""" + + +class _BaseExpressionConverter: + """ + Base class for optimizers that handle specific operations in MQL queries. + This class can be extended to implement optimizations for other operations. + """ + + @classmethod + def convert(cls, expr): + raise NotImplementedError("Subclasses should implement this method.") + + @classmethod + def is_simple_value(cls, value): + """ + Check if the value is a simple type (not a dict). + """ + if isinstance(value, str) and value.startswith("$"): + return False + if isinstance(value, list | tuple | set): + return all(cls.is_simple_value(v) for v in value) + # TODO: Expand functionality to support `$getField` conversion + return not isinstance(value, dict) or value is None + + @classmethod + def is_convertable_field_name(cls, field_name): + """Validate a field_name is one that can be represented in $match""" + # This needs work and re-evaluation + return ( + isinstance(field_name, str) + and field_name.startswith("$") + and not field_name[:1].isalnum() + ) + + +class _EqExpressionConverter(_BaseExpressionConverter): + """Convert $eq operation to a $match compatible query.""" + + @classmethod + def convert(cls, eq_args): + if isinstance(eq_args, list) and len(eq_args) == 2: + field_expr, value = eq_args + + # Check if first argument is a simple field reference + if ( + isinstance(field_expr, str) + and field_expr.startswith("$") + and cls.is_simple_value(value) + ): + field_name = field_expr[1:] # Remove the $ prefix + return {field_name: value} + + return None + + +class _InExpressionConverter(_BaseExpressionConverter): + """Convert $in operation to a $match compatible query.""" + + @classmethod + def convert(cls, in_args): + if isinstance(in_args, list) and len(in_args) == 2: + field_expr, values = in_args + + # Check if first argument is a simple field reference + if isinstance(field_expr, str) and field_expr.startswith("$"): + field_name = field_expr[1:] # Remove the $ prefix + if isinstance(values, list | tuple | set) and all( + cls.is_simple_value(v) for v in values + ): + return {field_name: {"$in": values}} + + return None + + +class _LogicalExpressionConverter(_BaseExpressionConverter): + """Generic for converting logical operations to a $match compatible query.""" + + @classmethod + def convert(cls, combined_conditions): + if isinstance(combined_conditions, list): + optimized_conditions = [] + for condition in combined_conditions: + if isinstance(condition, dict) and len(condition) == 1: + if optimized_condition := convert_expression(condition): + optimized_conditions.append(optimized_condition) + else: + # Any failure should stop optimization + return None + if optimized_conditions: + return {cls._logical_op: optimized_conditions} + return None + + +class _OrExpressionConverter(_LogicalExpressionConverter): + """Convert $or operation to a $match compatible query.""" + + _logical_op = "$or" + + +class _AndExpressionConverter(_LogicalExpressionConverter): + """Convert $and operation to a $match compatible query.""" + + _logical_op = "$and" + + +OPTIMIZABLE_OPS = { + "$eq": _EqExpressionConverter, + "$in": _InExpressionConverter, + "$and": _AndExpressionConverter, + "$or": _OrExpressionConverter, +} + + +def convert_expression(expr): + """ + Optimize an MQL expression by extracting optimizable conditions. + + Args: + expr: Dictionary containing the MQL expression + + Returns: + Optimized match condition or None if not optimizable + """ + if isinstance(expr, dict) and len(expr) == 1: + op = next(iter(expr.keys())) + if op in OPTIMIZABLE_OPS: + return OPTIMIZABLE_OPS[op].convert(expr[op]) + return None diff --git a/django_mongodb_backend/query_conversion/query_optimizer.py b/django_mongodb_backend/query_conversion/query_optimizer.py new file mode 100644 index 00000000..6c0af11b --- /dev/null +++ b/django_mongodb_backend/query_conversion/query_optimizer.py @@ -0,0 +1,99 @@ +from copy import deepcopy + +from django_mongodb_backend.query_conversion.expression_converters import convert_expression + + +class QueryOptimizer: + def convert_expr_to_match(self, expr): + """ + Takes an MQL query with $expr and optimizes it by extracting + optimizable conditions into separate $match stages. + + Args: + expr_query: Dictionary containing the $expr query + + Returns: + List of optimized match conditions + """ + expr_query = deepcopy(expr) + + if "$expr" not in expr_query: + return [expr_query] + + if expr_query["$expr"] == {}: + return [{"$match": {}}] + + expr_content = expr_query["$expr"] + + # Handle the expression content + return self._process_expression(expr_content) + + def _process_expression(self, expr): + """ + Process an expression and extract optimizable conditions. + + Args: + expr: The expression to process + """ + match_conditions = [] + remaining_conditions = [] + if isinstance(expr, dict): + # Check if this is an $and operation + has_and = "$and" in expr + has_or = "$or" in expr + # Do a top-level check for $and or $or because these should inform + # If they fail, they should failover to a remaining conditions list + # There's probably a better way to do this, but this is a start + if has_and: + and_match_conditions = self._process_logical_conditions("$and", expr["$and"]) + match_conditions.extend(and_match_conditions) + if has_or: + or_match_conditions = self._process_logical_conditions("$or", expr["$or"]) + match_conditions.extend(or_match_conditions) + if not has_and and not has_or: + # Process single condition + optimized = convert_expression(expr) + if optimized: + match_conditions.append({"$match": optimized}) + else: + remaining_conditions.append({"$match": {"$expr": expr}}) + else: + # Can't optimize + remaining_conditions.append({"$expr": expr}) + return match_conditions + remaining_conditions + + def _process_logical_conditions(self, logical_op, logical_conditions): + """ + Process conditions within a logical array. + + Args: + logical_conditions: List of conditions within logical operator + """ + optimized_conditions = [] + match_conditions = [] + remaining_conditions = [] + for condition in logical_conditions: + _remaining_conditions = [] + if isinstance(condition, dict): + if optimized := convert_expression(condition): + optimized_conditions.append(optimized) + else: + _remaining_conditions.append(condition) + else: + _remaining_conditions.append(condition) + if _remaining_conditions: + # Any expressions we can't optimize must remain + # in an $expr that preserves the logical operator + if len(_remaining_conditions) > 1: + remaining_conditions.append({"$expr": {logical_op: _remaining_conditions}}) + else: + remaining_conditions.append({"$expr": _remaining_conditions[0]}) + if optimized_conditions: + optimized_conditions.extend(remaining_conditions) + if len(optimized_conditions) > 1: + match_conditions.append({"$match": {logical_op: optimized_conditions}}) + else: + match_conditions.append({"$match": optimized_conditions[0]}) + else: + match_conditions.append({"$match": {logical_op: remaining_conditions}}) + return match_conditions diff --git a/tests/expression_converter_/__init__.py b/tests/expression_converter_/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/expression_converter_/models.py b/tests/expression_converter_/models.py new file mode 100644 index 00000000..234c2cec --- /dev/null +++ b/tests/expression_converter_/models.py @@ -0,0 +1,18 @@ +from django.db import models + + +class Author(models.Model): + name = models.CharField(max_length=100) + age = models.IntegerField() + author_city = models.CharField(max_length=100) + + def __str__(self): + return self.name + + +class Book(models.Model): + title = models.CharField(max_length=10) + author = models.ForeignKey(Author, models.CASCADE) + + def __str__(self): + return self.title diff --git a/tests/expression_converter_/test_match_conversion.py b/tests/expression_converter_/test_match_conversion.py new file mode 100644 index 00000000..bc390c3b --- /dev/null +++ b/tests/expression_converter_/test_match_conversion.py @@ -0,0 +1,253 @@ +from django.test import SimpleTestCase, TestCase + +from django_mongodb_backend.query_conversion.query_optimizer import QueryOptimizer + +from .models import Author + +optimizer = QueryOptimizer() + + +class QueryOptimizerTests(SimpleTestCase): + def assertOptimizerEqual(self, input, expected): + result = QueryOptimizer().convert_expr_to_match(input) + self.assertEqual(result, expected) + + def test_multiple_optimizable_conditions(self): + expr = { + "$expr": { + "$and": [ + {"$eq": ["$status", "active"]}, + {"$in": ["$category", ["electronics", "books"]]}, + {"$eq": ["$verified", True]}, + ] + } + } + expected = [ + { + "$match": { + "$and": [ + {"status": "active"}, + {"category": {"$in": ["electronics", "books"]}}, + {"verified": True}, + ] + } + } + ] + self.assertOptimizerEqual(expr, expected) + + def test_mixed_optimizable_and_non_optimizable_conditions(self): + expr = { + "$expr": { + "$and": [ + {"$eq": ["$status", "active"]}, + {"$gt": ["$price", 100]}, # Not optimizable + {"$in": ["$category", ["electronics"]]}, + ] + } + } + expected = [ + { + "$match": { + "$and": [ + {"status": "active"}, + {"category": {"$in": ["electronics"]}}, + {"$expr": {"$gt": ["$price", 100]}}, + ], + } + } + ] + self.assertOptimizerEqual(expr, expected) + + def test_non_optimizable_condition(self): + expr = {"$expr": {"$gt": ["$price", 100]}} + expected = [ + { + "$match": { + "$expr": {"$gt": ["$price", 100]}, + } + } + ] + self.assertOptimizerEqual(expr, expected) + + def test_nested_logical_conditions(self): + expr = { + "$expr": { + "$or": [ + {"$eq": ["$status", "active"]}, + {"$in": ["$category", ["electronics", "books"]]}, + {"$and": [{"$eq": ["$verified", True]}, {"$gt": ["$price", 50]}]}, + ] + } + } + expected = [ + { + "$match": { + "$or": [ + {"status": "active"}, + {"category": {"$in": ["electronics", "books"]}}, + { + "$expr": { + "$and": [{"$eq": ["$verified", True]}, {"$gt": ["$price", 50]}] + } + }, + ] + } + } + ] + self.assertOptimizerEqual(expr, expected) + + def test_complex_nested_with_non_optimizable_parts(self): + expr = { + "$expr": { + "$and": [ + { + "$or": [ # Not optimizable because of $gt + {"$eq": ["$status", "active"]}, + {"$gt": ["$views", 1000]}, + ] + }, + {"$in": ["$category", ["electronics", "books"]]}, + {"$eq": ["$verified", True]}, + {"$gt": ["$price", 50]}, # Not optimizable + ] + } + } + expected = [ + { + "$match": { + "$and": [ + {"category": {"$in": ["electronics", "books"]}}, + {"verified": True}, + { + "$expr": { + "$or": [ + {"$eq": ["$status", "active"]}, + {"$gt": ["$views", 1000]}, + ] + } + }, + {"$expr": {"$gt": ["$price", 50]}}, + ] + } + } + ] + self.assertOptimizerEqual(expr, expected) + + def test_london_in_case(self): + expr = {"$expr": {"$in": ["$author_city", ["London"]]}} + expected = [{"$match": {"author_city": {"$in": ["London"]}}}] + self.assertOptimizerEqual(expr, expected) + + def test_deeply_nested_logical_operators(self): + expr = { + "$expr": { + "$and": [ + { + "$or": [ + {"$eq": ["$type", "premium"]}, + { + "$and": [ + {"$eq": ["$type", "standard"]}, + {"$in": ["$region", ["US", "CA"]]}, + ] + }, + ] + }, + {"$eq": ["$active", True]}, + ] + } + } + expected = [ + { + "$match": { + "$and": [ + { + "$or": [ + {"type": "premium"}, + { + "$and": [ + {"type": "standard"}, + {"region": {"$in": ["US", "CA"]}}, + ] + }, + ] + }, + {"active": True}, + ] + } + } + ] + self.assertOptimizerEqual(expr, expected) + + def test_deeply_nested_logical_operator_with_variable(self): + expr = { + "$expr": { + "$and": [ + { + "$or": [ + {"$eq": ["$type", "premium"]}, + { + "$and": [ + # Not optimizable because of Variable + {"$eq": ["$type", "$$standard"]}, + {"$in": ["$region", ["US", "CA"]]}, + ] + }, + ] + }, + {"$eq": ["$active", True]}, + ] + } + } + expected = [ + { + "$match": { + "$and": [ + {"active": True}, + { + "$expr": { + "$or": [ + {"$eq": ["$type", "premium"]}, + { + "$and": [ + {"$eq": ["$type", "$$standard"]}, + {"$in": ["$region", ["US", "CA"]]}, + ] + }, + ] + } + }, + ] + } + } + ] + self.assertOptimizerEqual(expr, expected) + + +class OptimizedMatchMQLTests(TestCase): + def test_in_query(self): + with self.assertNumQueries(1) as ctx: + list(Author.objects.filter(author_city__in=["London"])) + query = ctx.captured_queries[0]["sql"] + expected = ( + "db.expression_converter__author.aggregate([{'$match': " + + "{'author_city': {'$in': ('London',)}}}])" + ) + self.assertEqual(query, expected) + + def test_eq_query(self): + with self.assertNumQueries(1) as ctx: + list(Author.objects.filter(name="Alice")) + query = ctx.captured_queries[0]["sql"] + expected = "db.expression_converter__author.aggregate([{'$match': {'name': 'Alice'}}])" + self.assertEqual(query, expected) + + def test_eq_and_in_query(self): + with self.assertNumQueries(1) as ctx: + list(Author.objects.filter(name="Alice", author_city__in=["London", "New York"])) + query = ctx.captured_queries[0]["sql"] + expected = ( + "db.expression_converter__author.aggregate([{'$match': {'$and': " + + "[{'author_city': {'$in': ('London', 'New York')}}, {'name': 'Alice'}]}}])" + ) + self.assertEqual(query, expected) diff --git a/tests/expression_converter_/test_op_expressions.py b/tests/expression_converter_/test_op_expressions.py new file mode 100644 index 00000000..cd4c0e92 --- /dev/null +++ b/tests/expression_converter_/test_op_expressions.py @@ -0,0 +1,205 @@ +import datetime +from uuid import UUID + +from bson import Decimal128 +from django.test import SimpleTestCase + +from django_mongodb_backend.query_conversion.expression_converters import convert_expression + + +class TestBaseExpressionConversionCase(SimpleTestCase): + CONVERTIBLE_TYPES = { + "int": 42, + "float": 3.14, + "decimal128": Decimal128("3.14"), + "boolean": True, + "NoneType": None, + "string": "string", + "datetime": datetime.datetime.now(datetime.UTC), + "duration": datetime.timedelta(days=5, hours=3), + "uuid": UUID("12345678123456781234567812345678"), + } + + def assertConversionEqual(self, input, expected): + result = convert_expression(input) + self.assertEqual(result, expected) + + def assertNotOptimizable(self, input): + result = convert_expression(input) + self.assertIsNone(result) + + def test_non_dict_expression(self): + expr = ["$status", "active"] + self.assertNotOptimizable(expr) + + def test_empty_dict_expression(self): + expr = {} + self.assertNotOptimizable(expr) + + def test_non_convertible(self): + expr = {"$gt": ["$price", 100]} + self.assertNotOptimizable(expr) + + def _test_conversion_various_types(self, conversion_test): + for _type, val in self.CONVERTIBLE_TYPES.items(): + with self.subTest(_type=_type, val=val): + conversion_test(val) + + +class TestEqExprConversionCase(TestBaseExpressionConversionCase): + def test_eq_conversion(self): + expr = {"$eq": ["$status", "active"]} + expected = {"status": "active"} + self.assertConversionEqual(expr, expected) + + def test_eq_no_conversion_non_string_field(self): + expr = {"$eq": [123, "active"]} + self.assertNotOptimizable(expr) + + def test_eq_no_conversion_dict_value(self): + expr = {"$eq": ["$status", {"$gt": 5}]} + self.assertNotOptimizable(expr) + + def _test_eq_conversion_valid_type(self, _type): + expr = {"$eq": ["$age", _type]} + expected = {"age": _type} + self.assertConversionEqual(expr, expected) + + def _test_eq_conversion_valid_array_type(self, _type): + expr = {"$eq": ["$age", _type]} + expected = {"age": _type} + self.assertConversionEqual(expr, expected) + + def test_eq_conversion_various_types(self): + self._test_conversion_various_types(self._test_eq_conversion_valid_type) + + def test_eq_conversion_various_array_types(self): + self._test_conversion_various_types(self._test_eq_conversion_valid_array_type) + + +class TestInExprConversionCase(TestBaseExpressionConversionCase): + def test_in_conversion(self): + expr = {"$in": ["$category", ["electronics", "books", "clothing"]]} + expected = {"category": {"$in": ["electronics", "books", "clothing"]}} + self.assertConversionEqual(expr, expected) + + def test_in_no_conversion_non_string_field(self): + expr = {"$in": [123, ["electronics", "books"]]} + self.assertNotOptimizable(expr) + + def test_in_no_conversion_dict_value(self): + expr = {"$in": ["$status", [{"bad": "val"}]]} + self.assertNotOptimizable(expr) + + def _test_in_conversion_valid_type(self, _type): + expr = { + "$in": [ + "$age", + [ + _type, + ], + ] + } + expected = { + "age": { + "$in": [ + _type, + ] + } + } + self.assertConversionEqual(expr, expected) + + def test_in_conversion_various_types(self): + for _type, val in self.CONVERTIBLE_TYPES.items(): + with self.subTest(_type=_type, val=val): + self._test_in_conversion_valid_type(val) + + +class TestLogicalExpressionConversionCase(TestBaseExpressionConversionCase): + def test_logical_and_conversion(self): + expr = { + "$and": [ + {"$eq": ["$status", "active"]}, + {"$in": ["$category", ["electronics", "books"]]}, + {"$eq": ["$verified", True]}, + ] + } + expected = { + "$and": [ + {"status": "active"}, + {"category": {"$in": ["electronics", "books"]}}, + {"verified": True}, + ] + } + self.assertConversionEqual(expr, expected) + + def test_logical_or_conversion(self): + expr = { + "$or": [ + {"$eq": ["$status", "active"]}, + {"$in": ["$category", ["electronics", "books"]]}, + ] + } + expected = { + "$or": [ + {"status": "active"}, + {"category": {"$in": ["electronics", "books"]}}, + ] + } + self.assertConversionEqual(expr, expected) + + def test_logical_or_conversion_failure(self): + expr = { + "$or": [ + {"$eq": ["$status", "active"]}, + {"$in": ["$category", ["electronics", "books"]]}, + { + "$and": [ + {"verified": True}, + {"$gt": ["$price", 50]}, # Not optimizable + ] + }, + ] + } + self.assertNotOptimizable(expr) + + def test_logical_mixed_conversion(self): + expr = { + "$and": [ + { + "$or": [ + {"$eq": ["$status", "active"]}, + ] + }, + {"$in": ["$category", ["electronics", "books"]]}, + {"$eq": ["$verified", True]}, + ] + } + expected = { + "$and": [ + { + "$or": [ + {"status": "active"}, + ] + }, + {"category": {"$in": ["electronics", "books"]}}, + {"verified": True}, + ] + } + self.assertConversionEqual(expr, expected) + + def test_logical_mixed_conversion_failure(self): + expr = { + "$and": [ + { + "$or": [ + {"$eq": ["$status", "active"]}, + {"$gt": ["$views", 1000]}, + ] + }, + {"$in": ["$category", ["electronics", "books"]]}, + {"$eq": ["$verified", True]}, + {"$gt": ["$price", 50]}, # Not optimizable + ] + } + self.assertNotOptimizable(expr) diff --git a/tests/queries_/test_explain.py b/tests/queries_/test_explain.py index d0e96415..6b74379b 100644 --- a/tests/queries_/test_explain.py +++ b/tests/queries_/test_explain.py @@ -20,9 +20,7 @@ def test_object_id(self): id = ObjectId() result = Author.objects.filter(id=id).explain() parsed = json_util.loads(result) - self.assertEqual( - parsed["command"]["pipeline"], [{"$match": {"$expr": {"$eq": ["$_id", id]}}}] - ) + self.assertEqual(parsed["command"]["pipeline"], [{"$match": {"_id": id}}]) def test_non_ascii(self): """The json is dumped with ensure_ascii=False.""" @@ -32,6 +30,4 @@ def test_non_ascii(self): # non-ASCII characters. self.assertIn(name, result) parsed = json.loads(result) - self.assertEqual( - parsed["command"]["pipeline"], [{"$match": {"$expr": {"$eq": ["$name", name]}}}] - ) + self.assertEqual(parsed["command"]["pipeline"], [{"$match": {"name": name}}]) diff --git a/tests/queries_/test_mql.py b/tests/queries_/test_mql.py index fed955a9..9b7c7bcc 100644 --- a/tests/queries_/test_mql.py +++ b/tests/queries_/test_mql.py @@ -25,7 +25,7 @@ def test_all(self): with self.assertNumQueries(1) as ctx: list(Author.objects.all()) query = ctx.captured_queries[0]["sql"] - self.assertEqual(query, "db.queries__author.aggregate([{'$match': {'$expr': {}}}])") + self.assertEqual(query, "db.queries__author.aggregate([{'$match': {}}])") def test_join(self): with self.assertNumQueries(1) as ctx: @@ -40,12 +40,13 @@ def test_join(self): "{'$and': [{'$eq': ['$$parent__field__0', '$_id']}, " "{'$eq': ['$name', 'Bob']}]}}}], 'as': 'queries__author'}}, " "{'$unwind': '$queries__author'}, " - "{'$match': {'$expr': {'$eq': ['$queries__author.name', 'Bob']}}}])", + "{'$match': {'queries__author.name': 'Bob'}}])", ) class FKLookupConditionPushdownTests(TestCase): def test_filter_on_local_and_related_fields(self): + self.maxDiff = None with self.assertNumQueries(1) as ctx: list(Book.objects.filter(title="Don", author__name="John")) query = ctx.captured_queries[0]["sql"] @@ -57,8 +58,8 @@ def test_filter_on_local_and_related_fields(self): "{'$match': {'$expr': {'$and': [{'$eq': ['$$parent__field__0', " "'$_id']}, {'$eq': ['$name', 'John']}]}}}], 'as': " "'queries__author'}}, {'$unwind': '$queries__author'}, {'$match': " - "{'$expr': {'$and': [{'$eq': ['$queries__author.name', 'John']}, " - "{'$eq': ['$title', 'Don']}]}}}])", + "{'$and': [{'queries__author.name': 'John'}, " + "{'title': 'Don'}]}}])", ) def test_or_mixing_local_and_related_fields_is_not_pushable(self): @@ -71,9 +72,9 @@ def test_or_mixing_local_and_related_fields_is_not_pushable(self): "'queries__author', 'let': {'parent__field__0': '$author_id'}, " "'pipeline': [{'$match': {'$expr': {'$and': [{'$eq': " "['$$parent__field__0', '$_id']}]}}}], 'as': 'queries__author'}}, " - "{'$unwind': '$queries__author'}, {'$match': {'$expr': {'$or': " - "[{'$eq': ['$title', 'Don']}, {'$eq': ['$queries__author.name', " - "'John']}]}}}])", + "{'$unwind': '$queries__author'}, {'$match': {'$or': " + "[{'title': 'Don'}, {'queries__author.name': " + "'John'}]}}])", ) def test_filter_on_self_join_fields(self): @@ -90,9 +91,9 @@ def test_filter_on_self_join_fields(self): "{'parent__field__0': '$parent_id'}, 'pipeline': [{'$match': {'$expr': " "{'$and': [{'$eq': ['$$parent__field__0', '$_id']}, {'$and': [{'$eq': " "['$group_id', ObjectId('6891ff7822e475eddc20f159')]}, {'$eq': ['$name', " - "'parent']}]}]}}}], 'as': 'T2'}}, {'$unwind': '$T2'}, {'$match': {'$expr': " - "{'$and': [{'$eq': ['$T2.group_id', ObjectId('6891ff7822e475eddc20f159')]}, " - "{'$eq': ['$T2.name', 'parent']}]}}}])", + "'parent']}]}]}}}], 'as': 'T2'}}, {'$unwind': '$T2'}, {'$match': " + "{'$and': [{'T2.group_id': ObjectId('6891ff7822e475eddc20f159')}, " + "{'T2.name': 'parent'}]}}])", ) def test_filter_on_reverse_foreignkey_relation(self): @@ -107,12 +108,13 @@ def test_filter_on_reverse_foreignkey_relation(self): "['$$parent__field__0', '$order_id']}, {'$eq': ['$status', " "ObjectId('6891ff7822e475eddc20f159')]}]}}}], 'as': " "'queries__orderitem'}}, {'$unwind': '$queries__orderitem'}, " - "{'$match': {'$expr': {'$eq': ['$queries__orderitem.status', " - "ObjectId('6891ff7822e475eddc20f159')]}}}, " + "{'$match': {'queries__orderitem.status': " + "ObjectId('6891ff7822e475eddc20f159')}}, " "{'$addFields': {'_id': '$_id'}}, {'$sort': SON([('_id', 1)])}])", ) def test_filter_on_local_and_nested_join_fields(self): + self.maxDiff = None with self.assertNumQueries(1) as ctx: list( Order.objects.filter( @@ -134,11 +136,11 @@ def test_filter_on_local_and_nested_join_fields(self): "{'parent__field__0': '$queries__orderitem.order_id'}, " "'pipeline': [{'$match': {'$expr': {'$and': [{'$eq': " "['$$parent__field__0', '$_id']}, {'$eq': ['$name', 'My Order']}]}" - "}}], 'as': 'T3'}}, {'$unwind': '$T3'}, {'$match': {'$expr': " - "{'$and': [{'$eq': ['$T3.name', 'My Order']}, {'$eq': " - "['$queries__orderitem.status', " - "ObjectId('6891ff7822e475eddc20f159')]}, {'$eq': ['$name', " - "'My Order']}]}}}, {'$addFields': {'_id': '$_id'}}, " + "}}], 'as': 'T3'}}, {'$unwind': '$T3'}, {'$match': " + "{'$and': [{'T3.name': 'My Order'}, " + "{'queries__orderitem.status': ObjectId('6891ff7822e475eddc20f159')}, " + "{'name': 'My Order'}]}}, " + "{'$addFields': {'_id': '$_id'}}, " "{'$sort': SON([('_id', 1)])}])", ) @@ -157,13 +159,14 @@ def test_negated_related_filter_is_not_pushable(self): ) def test_or_on_local_fields_only(self): + self.maxDiff = None with self.assertNumQueries(1) as ctx: list(Order.objects.filter(models.Q(name="A") | models.Q(name="B"))) query = ctx.captured_queries[0]["sql"] self.assertEqual( query, - "db.queries__order.aggregate([{'$match': {'$expr': {'$or': " - "[{'$eq': ['$name', 'A']}, {'$eq': ['$name', 'B']}]}}}, " + "db.queries__order.aggregate([{'$match': {'$or': " + "[{'name': 'A'}, {'name': 'B'}]}}, " "{'$addFields': {'_id': '$_id'}}, {'$sort': SON([('_id', 1)])}])", ) @@ -177,9 +180,8 @@ def test_or_with_mixed_pushable_and_non_pushable_fields(self): "'queries__author', 'let': {'parent__field__0': '$author_id'}, " "'pipeline': [{'$match': {'$expr': {'$and': [{'$eq': " "['$$parent__field__0', '$_id']}]}}}], 'as': 'queries__author'}}, " - "{'$unwind': '$queries__author'}, {'$match': {'$expr': {'$or': " - "[{'$eq': ['$queries__author.name', 'John']}, {'$eq': ['$title', " - "'Don']}]}}}])", + "{'$unwind': '$queries__author'}, {'$match': {'$or': " + "[{'queries__author.name': 'John'}, {'title': 'Don'}]}}])", ) def test_push_equality_between_parent_and_child_fields(self): @@ -201,6 +203,7 @@ def test_push_equality_between_parent_and_child_fields(self): class M2MLookupConditionPushdownTests(TestCase): def test_simple_related_filter_is_pushed(self): + self.maxDiff = None with self.assertNumQueries(1) as ctx: list(Library.objects.filter(readers__name="Alice")) query = ctx.captured_queries[0]["sql"] @@ -246,7 +249,7 @@ def test_simple_related_filter_is_pushed(self): ], "as": "queries__reader" }}, {"$unwind": "$queries__reader"}, - {"$match": {"$expr": {"$eq": ["$queries__reader.name", "Alice"]}}} + {"$match": {"queries__reader.name": "Alice"}} ]) """ self.assertEqual(query, uglify_mongo_aggregate(expected_query)) @@ -285,12 +288,12 @@ def test_subquery_join_is_pushed(self): {"$unwind": "$U2"}, { "$match": { - "$expr": { - "$and": [ - {"$eq": ["$U2.name", "Alice"]}, + "$and": [ + {"U2.name": "Alice"}, + {"$expr": {"$eq": ["$library_id","$$parent__field__0"]} - ] - } + } + ] } }, {"$project": {"a": {"$literal": 1}}}, @@ -385,12 +388,10 @@ def test_filter_on_local_and_related_fields(self): {"$unwind": "$queries__reader"}, { "$match": { - "$expr": { - "$and": [ - {"$eq": ["$name", "Central"]}, - {"$eq": ["$queries__reader.name", "Alice"]} - ] - } + "$and": [ + {"name": "Central"}, + {"queries__reader.name": "Alice"} + ] } } ] @@ -473,7 +474,7 @@ def test_or_on_local_fields_only(self): } }, {"$unwind": "$queries__reader"}, - {"$match": {"$expr": {"$eq": ["$name", "Ateneo"]}}}, + {"$match": {"name": "Ateneo"}}, { "$project": { "queries__reader": {"foreing_field": "$queries__reader.name"}, @@ -556,12 +557,10 @@ def test_or_with_mixed_pushable_and_non_pushable_fields(self): {"$unwind": "$queries__reader"}, { "$match": { - "$expr": { - "$or": [ - {"$eq": ["$queries__reader.name", "Alice"]}, - {"$eq": ["$name", "Central"]} - ] - } + "$or": [ + {"queries__reader.name": "Alice"}, + {"name": "Central"} + ] } } ])