check complex queries

Author: Gerit Wagner

search_query/constants.py

@@ -6,6 +6,7 @@
 from typing import Tuple
 
 # noqa: E501
+# ruff: noqa: E501
 
 # pylint: disable=too-few-public-methods
 # pylint: disable=line-too-long

search_query/linter_base.py

@@ -27,6 +27,7 @@
 
 # pylint: disable=too-many-public-methods
 # pylint: disable=too-many-lines
+# ruff: noqa: E501
 
 
 # Could indeed be a general Validator class
@@ -1060,6 +1061,60 @@ def _check_redundant_terms(
                             )
                             redundant_terms.append(term_b)
 
+    def _check_for_opportunities_to_combine_subqueries(
+        self, term_field_query: Query
+    ) -> None:
+        """Check for opportunities to combine subqueries with the same search field."""
+
+        # Only consider top-level OR-connected subqueries with two children each
+        if term_field_query.operator and term_field_query.value == Operators.OR:
+            candidates = [
+                q
+                for q in term_field_query.children
+                if q.operator and q.value == Operators.AND and len(q.children) == 2
+            ]
+
+            for i, q1 in enumerate(candidates):
+                for q2 in candidates[i + 1 :]:
+                    a1, a2 = q1.children
+                    b1, b2 = q2.children
+
+                    # Identify identical and differing pairs
+                    if a1.value == b1.value and a2.value != b2.value:
+                        identical = (a1, b1)
+                        differing = (a2, b2)
+                    elif a2.value == b2.value and a1.value != b1.value:
+                        identical = (a2, b2)
+                        differing = (a1, b1)
+                    else:
+                        continue  # Skip if no clearly matching pair
+
+                    details = (
+                        f"The queries share {Colors.GREY}identical query parts{Colors.END}:"
+                        f"\n({Colors.GREY}{identical[0].to_string()}{Colors.END} AND "
+                        f"{Colors.ORANGE}{differing[0].to_string()}{Colors.END}) OR \n"
+                        f"({Colors.GREY}{identical[1].to_string()}{Colors.END} AND "
+                        f"{Colors.ORANGE}{differing[1].to_string()}{Colors.END})\n"
+                        f"Combine the {Colors.ORANGE}differing parts{Colors.END} into a "
+                        f"{Colors.GREEN}single OR-group{Colors.END} to reduce redundancy:\n"
+                        f"({Colors.GREY}{identical[0].to_string()}{Colors.END} AND "
+                        f"({Colors.GREEN}{differing[0].to_string()} OR "
+                        f"{differing[1].to_string()}{Colors.END}))"
+                    )
+
+                    positions = [differing[0].position, differing[1].position]
+
+                    self.add_linter_message(
+                        QueryErrorCode.QUERY_STRUCTURE_COMPLEX,
+                        positions=positions,  # type: ignore
+                        details=details,
+                    )
+
+        # iterate over subqueries
+        if term_field_query.children:
+            for child in term_field_query.children:
+                self._check_for_opportunities_to_combine_subqueries(child)
+
 
 class QueryListLinter:
     """Class for Query List Validation"""

search_query/pubmed/linter.py

@@ -316,6 +316,12 @@ def validate_query_tree(self, query: Query) -> None:
         # "Sleep"[mh] AND "vigilant attention"[ti]
         # "Sleep Deprivation"[mh] AND "vigilant attention"[ti]
 
+    def validate_platform_query(self, query: Query) -> None:
+        """Validate the query for the PubMed platform"""
+
+        term_field_query = self.get_query_with_fields_at_terms(query)
+        self._check_for_opportunities_to_combine_subqueries(term_field_query)
+
     def syntax_str_to_generic_search_field_set(self, field_value: str) -> set:
         """Translate a search field"""
 

search_query/pubmed/parser.py

@@ -248,6 +248,9 @@ def parse(self) -> Query:
 
         query.set_platform_unchecked(PLATFORM.PUBMED.value, silent=True)
 
+        self.linter.validate_platform_query(query)  # type: ignore
+        self.linter.check_status()
+
         return query
 
 
@@ -347,6 +350,10 @@ def parse(self) -> Query:
 
         query = list(self.query_dict.values())[-1]["query"]
 
+        linter = PubmedQueryStringLinter(query_str=self.query_list)
+        linter.validate_query_tree(query)
+        linter.check_status()
+
         # linter.check_status() ?
         query.set_platform_unchecked(PLATFORM.PUBMED.value)
 

search_query/pubmed/serializer.py

@@ -15,6 +15,12 @@ def to_string_pubmed(query: Query) -> str:
 
     # to do combine querys for PLATFORM_COMBINED_FIELDS_MAP
 
+    if not query.children:
+        # query has no children, so it is a leaf node
+        if query.search_field:
+            return f"{query.value}{query.search_field.value}"
+        return query.value
+
     result = ""
     for child in query.children:
         if not child.operator:

test/test_pubmed.py

@@ -709,6 +709,66 @@ def test_pubmed_invalid_token_sequences(
                 }
             ],
         ),
+        (
+            '((AI OR "Artificial Intelligence") AND Aversion) OR ((AI OR "Artificial Intelligence") AND Appreciation)',
+            "",
+            [
+                {
+                    "code": "E0001",
+                    "label": "search-field-missing",
+                    "message": "Expected search field is missing",
+                    "is_fatal": False,
+                    "position": [(-1, -1)],
+                    "details": "Search field is missing (TODO: default?)",
+                },
+                {
+                    "code": "W0004",
+                    "label": "query-structure-unnecessarily-complex",
+                    "message": "Query structure is more complex than necessary",
+                    "is_fatal": False,
+                    "position": [(39, 47), (91, 103)],
+                    "details": 'The queries share \x1b[90midentical query parts\x1b[0m:\n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND \x1b[93mAversion[all]\x1b[0m) OR \n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND \x1b[93mAppreciation[all]\x1b[0m)\nCombine the \x1b[93mdiffering parts\x1b[0m into a \x1b[92msingle OR-group\x1b[0m to reduce redundancy:\n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND (\x1b[92mAversion[all] OR Appreciation[all]\x1b[0m))',
+                },
+            ],
+        ),
+        (
+            '(Algorithm* Aversion) OR (Algorithm* Appreciation) OR ((AI OR "Artificial Intelligence") AND Aversion) OR ((AI OR "Artificial Intelligence") AND Appreciation) OR ("AI recommendation" OR "Artificial intelligence recommendation" OR "Machine learning recommendation" OR "ML recommendation") OR ("AI decision*" OR "Artificial intelligence decision*" OR "Algorithm* decision" OR "Machine learning decision*" OR "ML decision*") OR ("AI Advice" OR "Artificial intelligence advice" OR "Algorithm* advice" OR "Machine learning advice" OR "ML advice") OR (("AI" OR "Artificial Intelligence" OR "Algorithm*" OR "Machine learning" OR "ML") AND "Decision aid")',
+            "",
+            [
+                {
+                    "code": "E0001",
+                    "label": "search-field-missing",
+                    "message": "Expected search field is missing",
+                    "is_fatal": False,
+                    "position": [(-1, -1)],
+                    "details": "Search field is missing (TODO: default?)",
+                },
+                {
+                    "code": "W0004",
+                    "label": "query-structure-unnecessarily-complex",
+                    "message": "Query structure is more complex than necessary",
+                    "is_fatal": False,
+                    "position": [(93, 101), (145, 157)],
+                    "details": 'The queries share \x1b[90midentical query parts\x1b[0m:\n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND \x1b[93mAversion[all]\x1b[0m) OR \n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND \x1b[93mAppreciation[all]\x1b[0m)\nCombine the \x1b[93mdiffering parts\x1b[0m into a \x1b[92msingle OR-group\x1b[0m to reduce redundancy:\n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND (\x1b[92mAversion[all] OR Appreciation[all]\x1b[0m))',
+                },
+                {
+                    "code": "W0004",
+                    "label": "query-structure-unnecessarily-complex",
+                    "message": "Query structure is more complex than necessary",
+                    "is_fatal": False,
+                    "position": [(93, 101), (632, 646)],
+                    "details": 'The queries share \x1b[90midentical query parts\x1b[0m:\n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND \x1b[93mAversion[all]\x1b[0m) OR \n(\x1b[90m("AI"[all] OR "Artificial Intelligence"[all] OR "Algorithm*"[all] OR "Machine learning"[all] OR "ML"[all])\x1b[0m AND \x1b[93m"Decision aid"[all]\x1b[0m)\nCombine the \x1b[93mdiffering parts\x1b[0m into a \x1b[92msingle OR-group\x1b[0m to reduce redundancy:\n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND (\x1b[92mAversion[all] OR "Decision aid"[all]\x1b[0m))',
+                },
+                {
+                    "code": "W0004",
+                    "label": "query-structure-unnecessarily-complex",
+                    "message": "Query structure is more complex than necessary",
+                    "is_fatal": False,
+                    "position": [(145, 157), (632, 646)],
+                    "details": 'The queries share \x1b[90midentical query parts\x1b[0m:\n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND \x1b[93mAppreciation[all]\x1b[0m) OR \n(\x1b[90m("AI"[all] OR "Artificial Intelligence"[all] OR "Algorithm*"[all] OR "Machine learning"[all] OR "ML"[all])\x1b[0m AND \x1b[93m"Decision aid"[all]\x1b[0m)\nCombine the \x1b[93mdiffering parts\x1b[0m into a \x1b[92msingle OR-group\x1b[0m to reduce redundancy:\n(\x1b[90m(AI[all] OR "Artificial Intelligence"[all])\x1b[0m AND (\x1b[92mAppreciation[all] OR "Decision aid"[all]\x1b[0m))',
+                },
+            ],
+        ),
     ],
 )
 def test_linter(