Delete multiple records endpoints (#307)

* Delete multiple records endpoints

* Test for deleting a record

* Remove comments

* Submodule update

* Submodule update

Author: JWittmeyer

app.py

@@ -30,6 +30,7 @@
 from fast_api.routes.weak_supervision import router as weak_supervision_router
 from fast_api.routes.labeling_tasks import router as labeling_tasks_router
 from fast_api.routes.task_execution import router as task_execution_router
+from fast_api.routes.record_internal import router as record_internal_router
 from fast_api.routes.playground import router as playground_router
 from middleware.database_session import handle_db_session
 from middleware.starlette_tmp_middleware import DatabaseSessionHandler
@@ -53,6 +54,7 @@
     PREFIX_DATA_BROWSER,
     PREFIX_LABELING,
     PREFIX_RECORD,
+    PREFIX_RECORD_INTERNAL,
     PREFIX_WEAK_SUPERVISION,
     PREFIX_LABELING_TASKS,
     PREFIX_TASK_EXECUTION,
@@ -116,6 +118,11 @@
     task_execution_router, prefix=PREFIX_TASK_EXECUTION, tags=["task-execution"]
 )
 
+fastapi_app_internal.include_router(
+    record_internal_router, prefix=PREFIX_RECORD_INTERNAL, tags=["record-internal"]
+)
+
+
 routes = [
     Route("/full_config", FullConfigRest),
     Route("/notify/{path:path}", Notify),

controller/record/manager.py

@@ -12,6 +12,7 @@
     tokenization,
     task_queue,
     record_label_association,
+    comments,
 )
 from service.search import search
 from submodules.model import enums
@@ -22,6 +23,10 @@
 from controller.embedding import manager as embedding_manager
 from controller.tokenization import tokenization_service
 from util.miscellaneous_functions import chunk_list
+from controller.tokenization.tokenization_service import (
+    request_reupload_docbins,
+)
+from util import notification
 import time
 import traceback
 
@@ -307,3 +312,46 @@ def __check_and_prep_edit_records(
         "rla_delete_tuples": rla_delete_tuples,
         "embedding_rebuilds": embedding_rebuilds,
     }
+
+
+def delete_records(
+    project_id: str,
+    record_ids: List[str],
+    as_thread: Optional[bool] = False,
+) -> None:
+    if not record_ids or len(record_ids) == 0:
+        return
+    if as_thread:
+        daemon.run_with_db_token(
+            __delete_records,
+            project_id,
+            record_ids,
+        )
+    else:
+        __delete_records(project_id, record_ids)
+
+
+def __delete_records(project_id: str, record_ids: List[str]) -> None:
+    try:
+        row_count = record.delete_many(project_id, record_ids)
+        if row_count == 0:
+            print(
+                f"No records found to delete for {record_ids} in project {project_id}",
+                flush=True,
+            )
+            return
+        comments.delete_by_type_and_xfkey(
+            project_id, record_ids, enums.CommentCategory.RECORD
+        )
+        general.commit()
+
+        request_reupload_docbins(project_id)
+
+        all_embeddings = embedding.get_all_embeddings_by_project_id(project_id)
+
+        for embedding_item in all_embeddings:
+            embedding_manager.request_tensor_upload(project_id, str(embedding_item.id))
+        notification.send_organization_update(project_id, "records_changed")
+
+    except Exception:
+        print(traceback.format_exc(), flush=True)

fast_api/models.py

@@ -511,3 +511,7 @@ class InviteUsersBody(BaseModel):
 
 class CheckInviteUsersBody(BaseModel):
     emails: List[StrictStr]
+
+
+class RecordDeletion(BaseModel):
+    record_ids: List[str]

fast_api/routes/record.py

@@ -1,11 +1,12 @@
+from typing import Optional
 from fastapi import APIRouter, Depends, Request, Body
 from controller.record import manager
 from controller.auth import manager as auth_manager
 from fast_api.routes.client_response import (
     get_custom_response,
     get_silent_success,
 )
-from fast_api.models import RecordSyncBody
+from fast_api.models import RecordSyncBody, RecordDeletion
 from util import notification
 from fastapi import status
 import json
@@ -33,3 +34,16 @@ def sync_records(
 
     notification.send_organization_update(project_id, "records_changed")
     return get_silent_success()
+
+
+@router.delete(
+    "/{project_id}/delete-records",
+    dependencies=[Depends(auth_manager.check_project_access_dep)],
+)
+def delete_by_record_ids(
+    project_id: str,
+    body: RecordDeletion,
+    as_thread: Optional[bool] = False,
+):
+    manager.delete_records(project_id, body.record_ids, as_thread)
+    return get_silent_success()

fast_api/routes/record_internal.py

@@ -0,0 +1,19 @@
+from typing import Optional
+from fastapi import APIRouter
+from controller.record import manager
+from fast_api.routes.client_response import (
+    get_silent_success,
+)
+from fast_api.models import RecordDeletion
+
+router = APIRouter()
+
+
+@router.delete("/{project_id}/delete-records")
+def delete_by_record_ids(
+    project_id: str,
+    body: RecordDeletion,
+    as_thread: Optional[bool] = False,
+):
+    manager.delete_records(project_id, body.record_ids, as_thread)
+    return get_silent_success()

route_prefix.py

@@ -13,6 +13,7 @@
 PREFIX_DATA_BROWSER = PREFIX + "/data-browser"
 PREFIX_LABELING = PREFIX + "/labeling"
 PREFIX_RECORD = PREFIX + "/record"
+PREFIX_RECORD_INTERNAL = PREFIX + "/record"
 PREFIX_WEAK_SUPERVISION = PREFIX + "/weak-supervision"
 PREFIX_LABELING_TASKS = PREFIX + "/labeling-tasks"
 PREFIX_TASK_EXECUTION = PREFIX + "/task-execution"

submodules/model

@@ -149,3 +149,25 @@ def test_update_records_to_project(
     emb = embedding_bo.get_all_embeddings_by_project_id(refinery_project.id)
     assert len(emb) > 0
     assert emb[0].current_delta_record_count > 0
+
+
+def test_delete_records_from_project(
+    client: TestClient, refinery_project: RefineryProject
+):
+    assert record_bo.count(refinery_project.id) == 2
+
+    record_ids = record_bo.get_all_ids(refinery_project.id)
+    to_del = record_ids[:1]  # delete one record
+
+    ##note that .delete doesn'T seem to support bodies so we use the request directly
+    response = client.request(
+        "DELETE",
+        f"/api/v1/record/{refinery_project.id}/delete-records",
+        json={"record_ids": to_del},  # delete one record
+    )
+
+    assert response.status_code == 200
+
+    record_ids = record_bo.get_all_ids(refinery_project.id)
+    assert len(record_ids) == 1
+    assert record_ids[0] != to_del[0]  # the deleted record should not be present