New endpoint to delete all objects (#499)

* Endpoint to delete all objects

* Add apispec

* Fix progress indicator

Author: DavidMStraub

gramps_webapi/api/__init__.py

@@ -63,7 +63,7 @@
 from .resources.name_formats import NameFormatsResource
 from .resources.name_groups import NameGroupsResource
 from .resources.notes import NoteResource, NotesResource
-from .resources.objects import CreateObjectsResource
+from .resources.objects import CreateObjectsResource, DeleteObjectsResource
 from .resources.ocr import MediaOcrResource
 from .resources.people import PeopleResource, PersonResource
 from .resources.places import PlaceResource, PlacesResource
@@ -130,6 +130,7 @@ def register_endpt(resource: Type[Resource], url: str, name: str):
 
 # Objects
 register_endpt(CreateObjectsResource, "/objects/", "objects")
+register_endpt(DeleteObjectsResource, "/objects/delete/", "delete_objects")
 # Transactions
 register_endpt(TransactionsResource, "/transactions/", "transactions")
 # Token

gramps_webapi/api/auth.py

@@ -46,6 +46,23 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
+def fresh_jwt_required(func):
+    """Check JWT and require it to be fresh.
+
+    Raise if claims include limited_scope key.
+    """
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        verify_jwt_in_request(fresh=True)
+        claims = get_jwt()
+        if claims.get(CLAIM_LIMITED_SCOPE):
+            raise NoAuthorizationError
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
 def jwt_limited_scope_required(func):
     """Check JWT.
 

gramps_webapi/api/resources/__init__.py

@@ -22,6 +22,7 @@
 from flask.views import MethodView
 
 from ..auth import (
+    fresh_jwt_required,
     jwt_limited_scope_required,
     jwt_refresh_token_required,
     jwt_required,
@@ -38,6 +39,12 @@ class ProtectedResource(Resource):
     decorators = [jwt_required]
 
 
+class FreshProtectedResource(Resource):
+    """Resource requiring a fresh JWT token."""
+
+    decorators = [fresh_jwt_required]
+
+
 class RefreshProtectedResource(Resource):
     """Resource requiring a JWT refresh token."""
 

gramps_webapi/api/resources/delete.py

@@ -27,7 +27,7 @@
 
 """Functions for deleting objects with references."""
 
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 from gramps.gen.db import DbTxn, DbWriteBase
 from gramps.gen.utils.db import (
@@ -37,6 +37,8 @@
     get_source_and_citation_referents,
 )
 
+from ...const import GRAMPS_OBJECT_PLURAL
+from ..search import get_total_number_of_objects
 from .util import transaction_to_json
 
 
@@ -333,7 +335,7 @@ def delete_source(db_handle: DbWriteBase, handle: str, trans: DbTxn) -> None:
     citation_list = citation_list[0]
 
     # (1) delete the references to the citation
-    for (citation_handle, refs) in citation_referents_list:
+    for citation_handle, refs in citation_referents_list:
         (
             person_list,
             family_list,
@@ -439,3 +441,28 @@ def delete_object(
         method(db_handle, handle, trans=trans)
         trans_dict = transaction_to_json(trans)
     return trans_dict
+
+
+def delete_all_objects(
+    db_handle: DbWriteBase,
+    namespaces: Optional[List[str]] = None,
+    progress_cb: Optional[Callable] = None,
+) -> List[Dict[str, Any]]:
+    """Delete all objects, optionally restricting to one or more types (namespaces)."""
+    if progress_cb:
+        total = get_total_number_of_objects(db_handle)
+    if namespaces is not None:
+        unknown_namespaces = set(namespaces) - set(GRAMPS_OBJECT_PLURAL.values())
+        if unknown_namespaces:
+            raise ValueError(f"Unknown namespace {unknown_namespaces}")
+    i = 0
+    for class_name, namespace in GRAMPS_OBJECT_PLURAL.items():
+        if namespaces is None or namespace in namespaces:
+            with DbTxn(f"Delete {namespaces or 'all objects'}", db_handle) as trans:
+                iter_handles = db_handle.method("iter_%s_handles", class_name)
+                del_method = delete_methods[class_name.lower()]
+                for handle in iter_handles():
+                    if progress_cb:
+                        progress_cb(current=i, total=total)
+                    i += 1
+                    del_method(db_handle=db_handle, handle=handle, trans=trans)

gramps_webapi/api/resources/objects.py

@@ -22,24 +22,28 @@
 import json
 from typing import Sequence
 
-from flask import Response, abort, current_app, request
+from flask import Response, jsonify, request
 from gramps.gen.db import DbTxn
 from gramps.gen.lib import Family, Person
 from gramps.gen.lib.primaryobj import BasicPrimaryObject as GrampsObject
 from gramps.gen.lib.serialize import from_json
+from webargs import fields, validate
 
-from ...auth.const import PERM_ADD_OBJ, PERM_EDIT_OBJ
+from ...auth.const import PERM_ADD_OBJ, PERM_DEL_OBJ_BATCH, PERM_EDIT_OBJ
+from ...const import GRAMPS_OBJECT_PLURAL
 from ..auth import require_permissions
 from ..search import SearchIndexer
+from ..tasks import AsyncResult, delete_objects, make_task_response, run_task
 from ..util import (
     abort_with_message,
     check_quota_people,
     get_db_handle,
     get_search_indexer,
     get_tree_from_jwt,
     update_usage_people,
+    use_args,
 )
-from . import ProtectedResource
+from . import FreshProtectedResource, ProtectedResource
 from .util import add_object, fix_object_dict, transaction_to_json, validate_object_dict
 
 
@@ -100,3 +104,31 @@ def post(self) -> Response:
         )
         res.headers.add("X-Total-Count", len(trans_dict))
         return res
+
+
+class DeleteObjectsResource(FreshProtectedResource):
+    """Resource for deleting multiple objects."""
+
+    @use_args(
+        {
+            "namespaces": fields.DelimitedList(
+                fields.Str(validate=validate.Length(min=1)),
+                validate=validate.ContainsOnly(
+                    choices=list(GRAMPS_OBJECT_PLURAL.values())
+                ),
+            ),
+        },
+        location="query",
+    )
+    def post(self, args) -> Response:
+        """Delete the objects."""
+        require_permissions([PERM_DEL_OBJ_BATCH])
+        tree = get_tree_from_jwt()
+        task = run_task(
+            delete_objects,
+            tree=tree,
+            namespaces=args.get("namespaces") or None,
+        )
+        if isinstance(task, AsyncResult):
+            return make_task_response(task)
+        return jsonify(task), 200

gramps_webapi/api/resources/token.py

@@ -49,12 +49,15 @@ def get_tokens(
     permissions: Iterable[str],
     tree_id: Optional[str] = None,
     include_refresh: bool = False,
+    fresh: bool = False,
 ):
     """Create access token (and refresh token if desired)."""
     claims = {"permissions": list(permissions)}
     if tree_id:
         claims["tree"] = tree_id
-    access_token = create_access_token(identity=str(user_id), additional_claims=claims)
+    access_token = create_access_token(
+        identity=str(user_id), additional_claims=claims, fresh=fresh
+    )
     if not include_refresh:
         return {"access_token": access_token}
     refresh_token = create_refresh_token(identity=str(user_id))
@@ -91,6 +94,7 @@ def post(self, args):
             permissions=permissions,
             tree_id=tree_id,
             include_refresh=True,
+            fresh=True,
         )
 
 
@@ -99,7 +103,7 @@ class TokenRefreshResource(RefreshProtectedResource):
 
     @limiter.limit("1/second")
     def post(self):
-        """Fetch a fresh token."""
+        """Fetch a new token."""
         user_id = get_jwt_identity()
         try:
             username = get_name(user_id)
@@ -114,6 +118,7 @@ def post(self):
             permissions=permissions,
             tree_id=tree_id,
             include_refresh=False,
+            fresh=False,
         )
 
 

gramps_webapi/api/search.py

@@ -76,6 +76,22 @@ def object_to_strings(obj) -> Tuple[str, str]:
     return process_strings(strings), process_strings(private_strings)
 
 
+def get_total_number_of_objects(db_handle: DbReadBase):
+    """Get the total number of searchable objects in the database."""
+    return (
+        db_handle.get_number_of_people()
+        + db_handle.get_number_of_families()
+        + db_handle.get_number_of_sources()
+        + db_handle.get_number_of_citations()
+        + db_handle.get_number_of_events()
+        + db_handle.get_number_of_media()
+        + db_handle.get_number_of_places()
+        + db_handle.get_number_of_repositories()
+        + db_handle.get_number_of_notes()
+        + db_handle.get_number_of_tags()
+    )
+
+
 def process_strings(strings: Sequence[str]) -> str:
     """Process a list of strings to a joined string.
 
@@ -206,27 +222,12 @@ def _add_obj_strings(self, writer, obj_dict):
                 "Failed adding object {}".format(obj_dict["handle"])
             )
 
-    def _get_total_number_of_objects(self, db_handle):
-        """Get the total number of searchable objects in the database."""
-        return (
-            db_handle.get_number_of_people()
-            + db_handle.get_number_of_families()
-            + db_handle.get_number_of_sources()
-            + db_handle.get_number_of_citations()
-            + db_handle.get_number_of_events()
-            + db_handle.get_number_of_media()
-            + db_handle.get_number_of_places()
-            + db_handle.get_number_of_repositories()
-            + db_handle.get_number_of_notes()
-            + db_handle.get_number_of_tags()
-        )
-
     def reindex_full(
         self, db_handle: DbReadBase, progress_cb: Optional[Callable] = None
     ):
         """Reindex the whole database."""
         if progress_cb:
-            total = self._get_total_number_of_objects(db_handle)
+            total = get_total_number_of_objects(db_handle)
         with self.index(overwrite=True).writer() as writer:
             for i, obj_dict in enumerate(iter_obj_strings(db_handle)):
                 if progress_cb:

gramps_webapi/api/tasks.py

@@ -22,7 +22,7 @@
 import uuid
 from gettext import gettext as _
 from http import HTTPStatus
-from typing import Any, Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, List, Optional, Union
 
 from celery import shared_task
 from celery.result import AsyncResult
@@ -35,6 +35,7 @@
 from .media import get_media_handler
 from .media_importer import MediaImporter
 from .report import run_report
+from .resources.delete import delete_all_objects
 from .resources.util import dry_run_import, run_import
 from .util import (
     check_quota_people,
@@ -287,3 +288,14 @@ def check_repair_database(self, tree: str):
 def upgrade_database_schema(self, tree: str):
     """Upgrade a Gramps database (tree) schema."""
     return upgrade_gramps_database(tree=tree, task=self)
+
+
+@shared_task(bind=True)
+def delete_objects(self, tree: str, namespaces: Optional[List[str]] = None):
+    """Delete all objects of a given type."""
+    db_handle = get_db_outside_request(tree=tree, view_private=True, readonly=False)
+    delete_all_objects(
+        db_handle=db_handle,
+        namespaces=namespaces,
+        progress_cb=progress_callback_count(self),
+    )

gramps_webapi/auth/const.py

@@ -56,6 +56,7 @@
 PERM_EDIT_OBJ = "EditObject"
 PERM_ADD_OBJ = "AddObject"
 PERM_DEL_OBJ = "DeleteObject"
+PERM_DEL_OBJ_BATCH = "BatchDeleteObjects"
 PERM_IMPORT_FILE = "ImportFile"
 PERM_VIEW_SETTINGS = "ViewSettings"
 PERM_EDIT_SETTINGS = "EditSettings"
@@ -101,6 +102,7 @@
     PERM_EDIT_TREE,
     PERM_REPAIR_TREE,
     PERM_UPGRADE_TREE_SCHEMA,
+    PERM_DEL_OBJ_BATCH,
 }
 
 PERMISSIONS[ROLE_ADMIN] = PERMISSIONS[ROLE_OWNER] | {

gramps_webapi/config.py

@@ -65,3 +65,4 @@ class DefaultConfigJWT(object):
     JWT_TOKEN_LOCATION = ["headers", "query_string"]
     JWT_ACCESS_TOKEN_EXPIRES = datetime.timedelta(minutes=15)
     JWT_REFRESH_TOKEN_EXPIRES = False
+    JWT_ERROR_MESSAGE_KEY = "message"