Add GQL filter to objects endpoints (#512)

* Add gramps_ql dependence and version

* Clean up test files

* Add GQL option to objects endpoints

* Try manually upgrading pyparsing

* Revert change to workflow

* catch one more error

* Use gql > 0.3

Author: DavidMStraub

gramps_webapi/api/resources/base.py

@@ -23,6 +23,7 @@
 from abc import abstractmethod
 from typing import Dict, List
 
+import gramps_ql as gql
 from flask import Response, abort, request
 from gramps.gen.const import GRAMPS_LOCALE as glocale
 from gramps.gen.db import DbTxn
@@ -31,6 +32,7 @@
 from gramps.gen.lib.primaryobj import BasicPrimaryObject as GrampsObject
 from gramps.gen.lib.serialize import from_json
 from gramps.gen.utils.grampslocale import GrampsLocale
+from pyparsing.exceptions import ParseBaseException
 from webargs import fields, validate
 
 from ...auth.const import PERM_ADD_OBJ, PERM_DEL_OBJ, PERM_EDIT_OBJ
@@ -343,6 +345,7 @@ class GrampsObjectsResource(GrampsObjectResourceHelper, Resource):
                 fields.Str(validate=validate.Length(min=1))
             ),
             "format_options": fields.Str(validate=validate.Length(min=1)),
+            "gql": fields.Str(validate=validate.Length(min=1)),
             "gramps_id": fields.Str(validate=validate.Length(min=1)),
             "keys": fields.DelimitedList(fields.Str(validate=validate.Length(min=1))),
             "locale": fields.Str(
@@ -413,6 +416,16 @@ def get(self, args: Dict) -> Response:
             )
             objects = [obj for obj in objects if obj.handle in set(handles)]
 
+        if "gql" in args:
+            try:
+                objects = [
+                    obj
+                    for obj in objects
+                    if gql.match(query=args["gql"], obj=obj, db=self.db_handle)
+                ]
+            except (ParseBaseException, ValueError, TypeError) as e:
+                abort_with_message(422, str(e))
+
         if self.gramps_class_name == "Media" and args.get("filemissing"):
             objects = filter_missing_files(objects)
 

gramps_webapi/api/resources/metadata.py

@@ -20,6 +20,7 @@
 
 """Metadata API resource."""
 
+import gramps_ql as gql
 import pytesseract
 from flask import Response, current_app
 from gramps.cli.clidbman import CLIDbManager
@@ -96,6 +97,7 @@ def get(self, args) -> Response:
                 "schema": VERSION,
                 "version": VERSION,
             },
+            "gramps_ql": {"version": gql.__version__},
             "locale": {
                 "lang": GRAMPS_LOCALE.lang,
                 "language": GRAMPS_LOCALE.language[0],

gramps_webapi/data/apispec.yaml

@@ -792,6 +792,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "media_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -1382,6 +1388,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "media_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -1857,6 +1869,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "media_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -2238,6 +2256,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "media_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -2561,6 +2585,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "media_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -2872,6 +2902,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "media_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -3155,6 +3191,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "media_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -3450,6 +3492,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "note_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -4137,6 +4185,12 @@ paths:
 
 
           The regex value is a boolean, true or false, that indicates if text values should be treated as regular expressions. If not present the default is false.
+      - name: gql
+        in: query
+        required: false
+        type: string
+        description: "A Gramps QL query string that is used to filter the objects."
+        example: "tag_list.length >= 10"
       - name: strip
         in: query
         required: false
@@ -9841,6 +9895,14 @@ definitions:
             description: "The version of the Gramps Web API code."
             type: string
             example: "0.1-dev"
+      gramps_ql:
+        description: "Information about the installed Gramps QL library."
+        type: object
+        properties:
+          version:
+            description: "The version of the Gramps Gramps QL library."
+            type: string
+            example: "0.3.0"
       locale:
         description: "The active locale."
         type: object

setup.py

@@ -52,6 +52,7 @@
     "celery[redis]",
     "Unidecode",
     "pytesseract",
+    "gramps-ql>=0.3.0",
 ]
 
 setup(

tests/test_endpoints/__init__.py

@@ -33,7 +33,7 @@
 import gramps_webapi.app
 from gramps_webapi.api.util import get_search_indexer
 from gramps_webapi.app import create_app
-from gramps_webapi.auth import user_db, add_user
+from gramps_webapi.auth import add_user, user_db
 from gramps_webapi.auth.const import (
     ROLE_ADMIN,
     ROLE_EDITOR,
@@ -121,4 +121,4 @@ def setUpModule():
 def tearDownModule():
     """Test module tear down."""
     if TEST_GRAMPSHOME and os.path.isdir(TEST_GRAMPSHOME):
-        pass  # shutil.rmtree(TEST_GRAMPSHOME)
+        shutil.rmtree(TEST_GRAMPSHOME)

tests/test_endpoints/test_people.py

@@ -21,6 +21,7 @@
 """Tests for the /api/people endpoints using example_gramps."""
 
 import unittest
+from urllib.parse import quote
 
 from . import BASE_URL, get_object_count, get_test_client
 from .checks import (
@@ -377,6 +378,35 @@ def test_get_people_parameter_rules_expected_response_invert(self):
             if item["gender"] == 1:
                 self.assertLess(len(item["family_list"]), 2)
 
+    def test_get_people_parameter_gql_validate_semantics(self):
+        """Test invalid rules syntax."""
+        check_invalid_semantics(self, TEST_URL + "?gql=()")
+
+    def test_get_people_parameter_gql_handle(self):
+        """Test invalid rules syntax."""
+        rv = check_success(
+            self,
+            TEST_URL + "?gql=" + quote("gramps_id=I0044"),
+        )
+        assert len(rv) == 1
+        assert rv[0]["gramps_id"] == "I0044"
+
+    def test_get_people_parameter_gql_like(self):
+        """Test invalid rules syntax."""
+        rv = check_success(
+            self,
+            TEST_URL + "?gql=" + quote("gramps_id ~ I004"),
+        )
+        assert len(rv) == 10
+
+    def test_get_people_parameter_gql_or(self):
+        """Test invalid rules syntax."""
+        rv = check_success(
+            self,
+            TEST_URL + "?gql=" + quote("(gramps_id ~ I004 or gramps_id ~ I003)"),
+        )
+        assert len(rv) == 20
+
     def test_get_people_parameter_extend_validate_semantics(self):
         """Test invalid extend parameter and values."""
         check_invalid_semantics(self, TEST_URL + "?extend", check="list")