Merge pull request #29 from cloudblue/feature/LITE-12104_nested-filters

LITE-12104 Operator "tuple" for filtration by nested fields added

Author: maxipavlovic

dj_rql/_dataclasses.py

@@ -21,18 +21,22 @@ def __init__(self, queryset, select_data, filter_tree, filter_node=None, filter_
 
 
 class FilterArgs:
-    def __init__(self, filter_name, operator, str_value, list_operator=None, **kwargs):
+    def __init__(self, filter_name, operator, str_value, list_operator=None,
+                 namespace=None, **kwargs):
         """
         :param str filter_name: Full filter name (f.e. ns1.ns2.filter1)
         :param str operator: RQL operator (f.e. eq, like, etc.)
         :param str str_value: Raw value from RQL query
         :param str or None list_operator: This is filled only if operation is done within IN or OUT
+        :param list or None namespace: List of namespaces
         :param dict kwargs: Other auxiliary data (f.e. to ease custom filtering)
         """
-        self.filter_name = filter_name
+        self.filter_basename = filter_name
+        self.filter_name = '.'.join((namespace or []) + [filter_name])
         self.operator = operator
         self.str_value = str_value
         self.list_operator = list_operator
+        self.namespace = namespace
 
         self.filter_lookup = kwargs.get('filter_lookup')
         self.django_lookup = kwargs.get('django_lookup')

dj_rql/filter_cls.py

@@ -242,6 +242,12 @@ def build_q_for_filter(self, data):
         """
         filter_name, operator, str_value = data.filter_name, data.operator, data.str_value
         list_operator = data.list_operator
+        filter_basename, namespace = data.filter_basename, data.namespace
+
+        if namespace and filter_basename == RQL_SEARCH_PARAM:
+            raise RQLFilterLookupError(details={
+                'error': f'Filter "{filter_basename}" can be applied only on top level.',
+            })
 
         if filter_name == RQL_SEARCH_PARAM:
             return self._build_q_for_search(operator, str_value)

dj_rql/grammar.py

@@ -16,6 +16,7 @@
 
 term: expr_term
     | logical
+    | tuple
     
 expr_term: comp
     | listing
@@ -59,9 +60,10 @@
     | _L_BRACE sign_prop (_COMMA sign_prop)* _R_BRACE
     
 val: prop
+    | tuple
     | QUOTED_VAL
     | UNQUOTED_VAL
-    
+
 prop: comp_term
     | logical_term
     | list_term
@@ -70,6 +72,8 @@
     | select_term
     | PROP
     
+tuple: _TUPLE _L_BRACE (comp|searching) (_COMMA (comp|searching))* _R_BRACE
+
 !sign_prop: ["+"|"-"] prop
     
 !comp_term: "eq" | "ne" | "gt" | "ge" | "lt" | "le"
@@ -93,6 +97,7 @@
 _AND: "and"
 _OR: "or"
 _NOT: "not"
+_TUPLE: "t"
 
 _COMMA: ","
 _L_BRACE: "("

dj_rql/transformer.py

@@ -42,6 +42,10 @@ def _extract_comparison(cls, args):
     def _get_value(obj):
         while isinstance(obj, Tree):
             obj = obj.children[0]
+
+        if isinstance(obj, Q):
+            return obj
+
         return obj.value
 
     def sign_prop(self, args):
@@ -72,13 +76,47 @@ class RQLToDjangoORMTransformer(BaseRQLTransformer):
         They are applied later in FilterCls. This is done on purpose, because transformer knows
         nothing about the mappings between filter names and orm fields.
     """
+    NAMESPACE_PROVIDERS = ('comp', 'listing')
+    NAMESPACE_FILLERS = ('prop',)
+    NAMESPACE_ACTIVATORS = ('tuple',)
+
     def __init__(self, filter_cls_instance):
         self._filter_cls_instance = filter_cls_instance
 
         self._ordering = []
         self._select = []
         self._filtered_props = set()
 
+        self._namespace = []
+        self._active_namespace = 0
+
+        self.__visit_tokens__ = False
+
+    def _push_namespace(self, tree):
+        if tree.data in self.NAMESPACE_PROVIDERS:
+            self._namespace.append(None)
+        elif tree.data in self.NAMESPACE_ACTIVATORS:
+            self._active_namespace = len(self._namespace)
+        elif (tree.data in self.NAMESPACE_FILLERS
+                and self._namespace
+                and self._namespace[-1] is None):
+            self._namespace[-1] = self._get_value(tree)
+
+    def _pop_namespace(self, tree):
+        if tree.data in self.NAMESPACE_PROVIDERS:
+            self._namespace.pop()
+        elif tree.data in self.NAMESPACE_ACTIVATORS:
+            self._active_namespace -= 1
+
+    def _get_current_namespace(self):
+        return self._namespace[:self._active_namespace]
+
+    def _transform_tree(self, tree):
+        self._push_namespace(tree)
+        ret_value = super()._transform_tree(tree)
+        self._pop_namespace(tree)
+        return ret_value
+
     @property
     def ordering_filters(self):
         return self._ordering
@@ -94,9 +132,19 @@ def start(self, args):
 
     def comp(self, args):
         prop, operation, value = self._extract_comparison(args)
-        self._filtered_props.add(prop)
 
-        return self._filter_cls_instance.build_q_for_filter(FilterArgs(prop, operation, value))
+        if isinstance(value, Q):
+            if operation == ComparisonOperators.EQ:
+                return value
+            else:
+                return ~value
+
+        filter_args = FilterArgs(prop, operation, value, namespace=self._get_current_namespace())
+        self._filtered_props.add(filter_args.filter_name)
+        return self._filter_cls_instance.build_q_for_filter(filter_args)
+
+    def tuple(self, args):
+        return Q(*args)
 
     def logical(self, args):
         operation = args[0].data
@@ -119,10 +167,17 @@ def listing(self, args):
 
         q = Q()
         for value_tree in args[2:]:
-            field_q = self._filter_cls_instance.build_q_for_filter(FilterArgs(
-                prop, f_op, self._get_value(value_tree),
-                list_operator=operation,
-            ))
+            value = self._get_value(value_tree)
+            if isinstance(value, Q):
+                if f_op == ComparisonOperators.EQ:
+                    field_q = value
+                else:
+                    field_q = ~value
+            else:
+                field_q = self._filter_cls_instance.build_q_for_filter(FilterArgs(
+                    prop, f_op, value,
+                    list_operator=operation,
+                ))
             if operation == ListOperators.IN:
                 q |= field_q
             else:
@@ -135,9 +190,9 @@ def listing(self, args):
     def searching(self, args):
         # like, ilike
         operation, prop, val = tuple(self._get_value(args[index]) for index in range(3))
-        self._filtered_props.add(prop)
-
-        return self._filter_cls_instance.build_q_for_filter(FilterArgs(prop, operation, val))
+        filter_args = FilterArgs(prop, operation, val, namespace=self._get_current_namespace())
+        self._filtered_props.add(filter_args.filter_name)
+        return self._filter_cls_instance.build_q_for_filter(filter_args)
 
     def ordering(self, args):
         props = args[1:]

setup.cfg

@@ -5,7 +5,7 @@ test = pytest
 exclude = .idea,.git,venv*/,.eggs/,*.egg-info,_generated_filters*.py
 max-line-length = 100
 show-source = True
-ignore = W605
+ignore = W503,W605
 
 [tool:pytest]
 addopts = --show-capture=no --create-db --nomigrations --junitxml=tests/reports/out.xml --cov=dj_rql --cov-report xml:tests/reports/coverage.xml

tests/test_drf/test_django_filters_backend.py

@@ -64,6 +64,9 @@ def test_compatibility_modify_initial_query(backend):
     ('limit=10;k__in=2', True),
     ('(k=v;k=z)', False),
     ('limit=10;k__in=2;k=y)', True),
+    ('t(email=1)', False),
+    ('author=t(email=email)', False),
+    ('k__in=v&t(auhtor=1)', False),
 ))
 def test_old_syntax(mocker, query, expected):
     request = mocker.MagicMock(query_params=QueryDict(query))

tests/test_filter_cls/test_apply_filters.py

@@ -158,6 +158,93 @@ def test_out():
     assert apply_out_listing_filters('23') == books
 
 
+@pytest.mark.django_db
+@pytest.mark.parametrize('filter_string', (
+    't(author.email={email},title=null())',
+    't(search={email})',
+    't(ge(published.at,{published_at}))',
+    't(author.publisher.id={publisher_id})',
+    'title=null()&t(author.email={email})',
+    't(author=t(email={email}))',
+    'author=t(email={email},is_male=true)',
+    'author=t(publisher=t(id={publisher_id}))',
+    'author=t(email={email},ne(is_male,false))',
+    'ne(author,t(email={second_book_email},is_male=true))',
+    'and(author=t(email={email}),author=t(is_male=true))',
+    'and(title=null(),author=t(is_male=true,publisher=t(id={publisher_id})))',
+    'in(author.email,({email}))',
+    'in(author,(t(publisher.id=null()),t(email={email})))',
+    'out(author,(t(email={second_book_email})))',
+))
+def test_tuple(filter_string):
+    books = create_books()
+    comp_filter = filter_string.format(
+        email=books[0].author.email,
+        published_at=books[0].published_at.date(),
+        publisher_id=books[0].author.publisher.id,
+        second_book_email=books[1].author.email,
+    )
+    assert apply_filters(comp_filter) == [books[0]]
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('filter_string', (
+    'author=t(like=1)',
+    'author=t(ilike=1)',
+    'author=t(in=1)',
+    'author=t(out=1)',
+    'author=t(eq=1)',
+    'author=t(ne=1)',
+    'author=t(and=1)',
+    'author=t(or=1)',
+    'author=t(limit=1)',
+    'author=t(offset=1)',
+))
+def test_tuple_syntax_terms_not_fail(filter_string):
+    books = create_books()
+    assert apply_filters(filter_string) == books
+
+
+@pytest.mark.parametrize('filter_string', (
+    't()',
+    't(1=1)',
+    'author=t(t(t(name=1))'
+    'author=t(male)',
+    'author=t(test=in(male,(true,false)))',
+    'in(t(is_male=true),(author))',
+    'select(t(author.publisher))',
+    'author=t(limit(email))',
+    'author=t(offset(email))',
+    'author=t(select(email))',
+    'author=t(ordering(email))',
+    'author=t(and(a=1,b=2))',
+    'author=t(or(a=1,b=2))',
+    'author=t(not(a=1))',
+    'author=t(search(x,term))',
+    'auhtor=t(select(+test))',
+))
+def test_tuple_parse_error(filter_string):
+    with pytest.raises(RQLFilterParsingError) as e:
+        apply_filters(filter_string)
+
+    expected = 'Bad filter query.'
+    assert e.value.details['error'] == expected
+
+
+def test_tuple_search_inside_namespace():
+    with pytest.raises(RQLFilterLookupError) as e:
+        apply_filters('author=t(search=term)')
+
+    expected = 'Filter "search" can be applied only on top level.'
+    assert e.value.details['error'] == expected
+
+
+def test_tuple_lookup_error():
+    with pytest.raises(RQLFilterLookupError) as e:
+        apply_filters('author=t(ge(email,1))')
+    assert e.value.details == {'filter': 'author.email', 'lookup': 'ge', 'value': '1'}
+
+
 @pytest.mark.django_db
 def test_null():
     books = create_books()

tests/test_filter_cls/utils.py

@@ -2,13 +2,25 @@
 #  Copyright © 2020 Ingram Micro Inc. All rights reserved.
 #
 
-from tests.dj_rf.models import Book
+from datetime import timedelta
+
+from django.utils import timezone
+
+from tests.dj_rf.models import Author, Book, Publisher
 from tests.dj_rf.view import apply_annotations
 
+
 book_qs = apply_annotations(Book.objects.order_by('id'))
 
 
 def create_books(count=2):
-    Book.objects.bulk_create([Book() for _ in range(count)])
+    for i in range(count):
+        author = Author.objects.create(
+            name=f'author{i}',
+            email=f'author{i}@example.com',
+            is_male=True,
+            publisher=Publisher.objects.create(),
+        )
+        Book.objects.create(author=author, published_at=timezone.now() - timedelta(days=i))
     books = list(book_qs)
     return books