Merge pull request #325 from mapswipe/dev

Dev

Author: Hagellach37

mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py

@@ -1,12 +1,22 @@
 """
 Archive a project.
 """
+import re
+from typing import Iterable
+
+from firebase_admin import exceptions
 
 from mapswipe_workers import auth
-from mapswipe_workers.definitions import logger
+from mapswipe_workers.definitions import logger, CustomError
+
 
+def chunks(data: list, size: int = 250) -> Iterable[list]:
+    """Yield successive n-sized chunks from list."""
+    for i in range(0, len(data), size):
+        yield data[i : i + size]  # noqa E203
 
-def archive_project(project_ids: list) -> None:
+
+def archive_project(project_ids: list) -> bool:
     """
     Archive a project.
 
@@ -15,46 +25,55 @@ def archive_project(project_ids: list) -> None:
     """
     for project_id in project_ids:
         logger.info(f"Archive project with the id {project_id}")
-        logger.info(f"Delete results of project with the id {project_id}")
 
         fb_db = auth.firebaseDB()
-        fb_db.reference(f"v2/results/{project_id}").set({})
-
-        # get group keys for this project to estimate size in firebase
-        groups = fb_db.reference(f"v2/groups/{project_id}").get(shallow=True)
-
-        if not groups:
-            logger.info("no groups to delete in firebase")
-        else:
-            group_keys = list(groups.keys())
-            chunk_size = 250
-            chunks = int(len(group_keys) / chunk_size) + 1
-
-            # delete groups, tasks in firebase for each chunk using the update function
-            for i in range(0, chunks):
-                logger.info(
-                    f"Delete max {chunk_size} groups and tasks"
-                    f"of project with the id {project_id}"
-                )
-                update_dict = {}
-                for group_id in group_keys[:chunk_size]:
-                    update_dict[group_id] = None
-                fb_db.reference(f"v2/groups/{project_id}").update(update_dict)
-                fb_db.reference(f"v2/tasks/{project_id}").update(update_dict)
-                group_keys = group_keys[chunk_size:]
-
-        logger.info(
-            f"Set status=archived in Firebase for project with the id {project_id}"
-        )
-        fb_db = auth.firebaseDB()
+        ref = fb_db.reference(f"v2/results/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                {ref.path}"""
+            )
+        try:
+            ref.delete()
+        except exceptions.InvalidArgumentError:
+            # Data to write exceeds the maximum size that can be modified
+            # with a single request. Delete chunks of data instead.
+            childs = ref.get(shallow=True)
+            for chunk in chunks(list(childs.keys())):
+                ref.update({key: None for key in chunk})
+            ref.delete()
+
+        ref = fb_db.reference(f"v2/tasks/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                {ref.path}"""
+            )
+        try:
+            ref.delete()
+        except exceptions.InvalidArgumentError:
+            # Data to write exceeds the maximum size that can be modified
+            # with a single request. Delete chunks of data instead.
+            childs = ref.get(shallow=True)
+            for chunk in chunks(list(childs.keys())):
+                ref.update({key: None for key in chunk})
+            ref.delete()
+
+        ref = fb_db.reference(f"v2/groups/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                {ref.path}"""
+            )
+        ref.delete()
+
         fb_db.reference(f"v2/projects/{project_id}/status").set("archived")
 
-        logger.info(
-            f"Set status=archived in Postgres for project with the id {project_id}"
-        )
         pg_db = auth.postgresDB()
         sql_query = """
             UPDATE projects SET status = 'archived'
             WHERE project_id = %(project_id)s;
         """
         pg_db.query(sql_query, {"project_id": project_id})
+
+    return True

mapswipe_workers/mapswipe_workers/firebase_to_postgres/delete_project.py

@@ -0,0 +1,86 @@
+"""
+Delete projects.
+"""
+import re
+from typing import Iterable
+
+from firebase_admin import exceptions
+
+from mapswipe_workers import auth
+from mapswipe_workers.definitions import logger, CustomError
+
+
+def chunks(data: list, size: int = 250) -> Iterable[list]:
+    """Yield successive n-sized chunks from list."""
+    for i in range(0, len(data), size):
+        yield data[i : i + size]  # noqa E203
+
+
+def delete_project(project_ids: list) -> None:
+    """
+    Deletes project, groups, tasks and results from Firebase and Postgres.
+    """
+    for project_id in project_ids:
+        logger.info(
+            f"Delete project, groups, tasks and results of project: {project_id}"
+        )
+
+        fb_db = auth.firebaseDB()
+        ref = fb_db.reference(f"v2/results/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                {ref.path}"""
+            )
+        try:
+            ref.delete()
+        except exceptions.InvalidArgumentError:
+            # Data to write exceeds the maximum size that can be modified
+            # with a single request. Delete chunks of data instead.
+            childs = ref.get(shallow=True)
+            for chunk in chunks(list(childs.keys())):
+                ref.update({key: None for key in chunk})
+            ref.delete()
+
+        ref = fb_db.reference(f"v2/tasks/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                {ref.path}"""
+            )
+        try:
+            ref.delete()
+        except exceptions.InvalidArgumentError:
+            # Data to write exceeds the maximum size that can be modified
+            # with a single request. Delete chunks of data instead.
+            childs = ref.get(shallow=True)
+            for chunk in chunks(list(childs.keys())):
+                ref.update({key: None for key in chunk})
+            ref.delete()
+
+        ref = fb_db.reference(f"v2/groups/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                {ref.path}"""
+            )
+        ref.delete()
+        ref = fb_db.reference(f"v2/projects/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                {ref.path}"""
+            )
+        ref.delete()
+
+        pg_db = auth.postgresDB()
+        sql_query = "DELETE FROM results WHERE project_id = %(project_id)s;"
+        pg_db.query(sql_query, {"project_id": project_id})
+        sql_query = "DELETE FROM tasks WHERE project_id = %(project_id)s;"
+        pg_db.query(sql_query, {"project_id": project_id})
+        sql_query = "DELETE FROM groups WHERE project_id = %(project_id)s;"
+        pg_db.query(sql_query, {"project_id": project_id})
+        sql_query = "DELETE FROM projects WHERE project_id = %(project_id)s;"
+        pg_db.query(sql_query, {"project_id": project_id})
+
+    return True

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -4,12 +4,14 @@
 import json
 import time
 
-import click
 import schedule as sched
+
+import click
 from mapswipe_workers import auth
 from mapswipe_workers.definitions import CustomError, logger, sentry
 from mapswipe_workers.firebase_to_postgres import (
     archive_project,
+    delete_project,
     transfer_results,
     update_data,
 )
@@ -198,11 +200,59 @@ def run_create_tutorial(input_file) -> None:
 )
 def run_archive_project(project_id, project_ids):
     """Archive projects in Postgres. Delete groups, tasks and results from Firebase."""
-    if not project_ids:
+    if not project_ids and not project_id:
+        click.echo("Missing argument")
+        return None
+    elif not project_ids:
         project_ids = [project_id]
+    click.echo("Start archive")
     update_data.update_project_data(project_ids)
     transfer_results.transfer_results(project_ids)
-    archive_project.archive_project(project_ids)
+    if archive_project.archive_project(project_ids):
+        click.echo("Finished archive")
+
+
+@cli.command("delete")
+@click.option(
+    "--project-id", "-i", help=("Delete project with giving project id"), type=str,
+)
+@click.option(
+    "--project-ids",
+    cls=PythonLiteralOption,
+    default="[]",
+    help=(
+        f"Delete multiple projects. "
+        f"Provide project id strings as a list: "
+        f"""["project_a", "project_b"]"""
+    ),
+)
+def run_delete_project(project_id, project_ids):
+    """Delete tasks, groups, project and results."""
+    if not project_ids and not project_id:
+        click.echo("Missing argument")
+        return None
+    elif not project_ids:
+        project_ids = [project_id]
+
+    click.echo(
+        "Projects and all associated data including results "
+        + "with following project ids will be deleted permantly:"
+    )
+    for project_id in project_ids:
+        click.echo(project_id)
+    click.echo()
+    click.echo("Continue with deletion? [y/n] ", nl=False)
+    click.echo()
+    c = click.getchar()
+
+    if c == "y":
+        click.echo("Start deletion")
+        if delete_project.delete_project(project_ids):
+            click.echo("Finished deletions")
+    elif c == "n":
+        click.echo("Abort!")
+    else:
+        click.echo("Invalid input")
 
 
 @cli.command("run")