Merge branch 'dev' into scheduling

Author: Matthias-Schaub

mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py

@@ -0,0 +1,38 @@
+"""
+Archive a project.
+"""
+
+from mapswipe_workers import auth
+from mapswipe_workers.definitions import logger
+
+
+def archive_project(project_ids: list) -> None:
+    """
+    Archive a project.
+
+    Deletes groups, tasks and results from Firebase.
+    Set status = archived for project in Firebase and Postgres.
+    """
+    for project_id in project_ids:
+        logger.info("Archive project with the id {0}".format(project_id))
+        logger.info(
+            "Delete results, groups and tasks of project with the id {0}".format(
+                project_id
+            )
+        )
+
+        fb_db = auth.firebaseDB()
+        fb_db.reference("v2/results/{0}".format(project_id)).set({})
+        fb_db.reference("v2/groups/{0}".format(project_id)).set({})
+        fb_db.reference("v2/tasks/{0}".format(project_id)).set({})
+
+        fb_db = auth.firebaseDB()
+        ref = fb_db.reference("v2/projects/{0}/status".format(project_id))
+        ref.set({"archived"})
+
+        pg_db = auth.postgresDB()
+        sql_query = (
+            "UPDATE projects SET status = 'archived' "
+            + "WHERE project_id = {0}".format(project_id)
+        )
+        pg_db.query(sql_query)

mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py

@@ -4,10 +4,8 @@
 from mapswipe_workers.definitions import logger
 
 
-def update_user_data(user_ids=None):
-    """
-    Copies new users from Firebase to Postgres
-    """
+def update_user_data(user_ids: list = []):
+    """Copies new users from Firebase to Postgres."""
     # TODO: On Conflict
     fb_db = auth.firebaseDB()
     pg_db = auth.postgresDB()
@@ -24,31 +22,29 @@ def update_user_data(user_ids=None):
     try:
         last_updated = last_updated[0][0]
         logger.info(f"got last updated timestamp: {last_updated}")
-    except:
+    except IndexError:
         logger.info("could not get last timestamp")
-        last_updated = None
+        last_updated = []
 
-    if last_updated is None:
-        # No users in the Postgres database yet.
-        # Get all users from Firebase.
-        users = fb_ref.get()
-    else:
+    if last_updated:
         # Get only new users from Firebase.
         last_updated = last_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
         fb_query = fb_ref.order_by_child("created").start_at(last_updated)
         users = fb_query.get()
         # Delete first user in ordered dict.
         # This user is already in the database (user.created = last_updated).
         if len(users) == 0:
-            logger.info(
-                f"there are no new users in firebase based on created timestamp"
-            )
+            logger.info("there are no new users in firebase based on created timestamp")
         else:
             users.popitem(last=False)
+    else:
+        # No users in the Postgres database yet.
+        # Get all users from Firebase.
+        users = fb_ref.get()
 
     # update users specified in user_ids list
     if user_ids:
-        logger.info(f"will add users to copy_new_users based on user_ids provided")
+        logger.info("will add users to copy_new_users based on user_ids provided")
         for user_id in user_ids:
             user = fb_ref.child(user_id).get()
             users[user_id] = user
@@ -65,7 +61,8 @@ def update_user_data(user_ids=None):
                 # if user has no "created" attribute, we set it to current time
                 created = dt.datetime.utcnow().isoformat()[0:-3] + "Z"
                 logger.info(
-                    f"user {user_id} didn't have a 'created' attribute. Set it to '{created}' now."
+                    f"user {user_id} didn't have a 'created' attribute. "
+                    f"Set it to '{created}' now."
                 )
 
             try:
@@ -74,7 +71,8 @@ def update_user_data(user_ids=None):
                 # if user has no "username" attribute, we set it to None
                 username = None
                 logger.info(
-                    f"user {user_id} didn't have a 'username' attribute. Set it to '{username}' now."
+                    f"user {user_id} didn't have a 'username' attribute. "
+                    f"Set it to '{username}' now."
                 )
 
             query_update_user = """
@@ -98,17 +96,14 @@ def update_user_data(user_ids=None):
     logger.info("Updated user data in Potgres")
 
 
-def update_project_data(project_ids=None):
+def update_project_data(project_ids: list = []):
     """
     Gets status of projects
     from Firebase and updates them in Postgres.
+
     Default behavior is to update all projects.
     If called with a list of project ids as parameter
     only those projects will be updated.
-
-    Parameters
-    ----------
-    project_ids: list
     """
 
     fb_db = auth.firebaseDB()

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -13,10 +13,16 @@
     CustomError,
     logger,
 )
-from mapswipe_workers.firebase_to_postgres import transfer_results, update_data
+from mapswipe_workers.firebase_to_postgres import (
+    archive_project,
+    transfer_results,
+    update_data,
+)
 from mapswipe_workers.generate_stats import generate_stats
 from mapswipe_workers.project_types.build_area import build_area_tutorial
-from mapswipe_workers.project_types.change_detection import change_detection_tutorial
+from mapswipe_workers.project_types.change_detection import (
+    change_detection_tutorial,
+)
 from mapswipe_workers.utils import sentry, slack, user_management
 
 
@@ -183,7 +189,32 @@ def run_create_tutorial(input_file) -> None:
         logger.exception(e)
 
 
-@cli.command("run")
+@click.command("archive")
+@click.option(
+    "--project-id",
+    "-i",
+    help=("Archive project with giving project id"),
+    type=str,
+)
+@click.option(
+    "--project-ids",
+    cls=PythonLiteralOption,
+    help=(
+        f"Archive multiple projects. "
+        f"Provide project id strings as a list: "
+        f"""["project_a", "project_b"]"""
+    ),
+)
+def run_archive_project(project_id, project_ids):
+    """Archive projects in Postgres. Delete groups, tasks and results from Firebase."""
+    if not project_ids:
+        project_ids = [project_id]
+    update_data.update_project_data(project_ids)
+    transfer_results.transfer_results(project_ids)
+    archive_project.archive_project(project_ids)
+
+
+@click.command("run")
 @click.option(
     "--schedule", is_flag=True, help=("Schedule jobs to run every 10 minutes.")
 )
@@ -206,4 +237,4 @@ def _run():
             sched.run_pending()
             time.sleep(1)
     else:
-        _run()
+        _run()

mapswipe_workers/tests/test_archive_project.py

@@ -0,0 +1,50 @@
+import unittest
+
+from mapswipe_workers import auth
+from mapswipe_workers.firebase_to_postgres import archive_project
+
+
+class TestArchiveProject(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        """Set up before tests are run."""
+        cls.project_id = "test_archive_project"
+        # TODO: Create project
+        archive_project.archive_project(cls.project_id)
+
+    @classmethod
+    def tearDownClass(cls):
+        """Tear down after tests are run."""
+        pg_db = auth.postgresDB()
+        sql_query = "".format(cls.project_id)
+        pg_db.query(sql_query)
+
+        fb_db = auth.firebaseDB()
+        ref = fb_db.reference("v2/projects/{0}".format(cls.project_id))
+        ref.set({})
+
+    def test_firebase_changes(self):
+        """Test if groups, tasks and results are deleted from Firebase."""
+        fb_db = auth.firebaseDB()
+
+        ref = fb_db.reference("v2/groups/{0}".format(self.project_id))
+        self.assertFalse(ref.get())
+
+        ref = fb_db.reference("v2/tasks/{0}".format(self.project_id))
+        self.assertFalse(ref.get())
+
+        ref = fb_db.reference("v2/results/{0}".format(self.project_id))
+        self.assertFalse(ref.get())
+
+    def test_postgres_changes(self):
+        """Test if postgres project is archived."""
+        pg_db = auth.postgresDB()
+        sql_query = "SELECT archived FROM projects WHERE project_id = {}".format(
+            self.project_id
+        )
+        result = pg_db.retr_query(sql_query)
+        self.assertEqual(result, "archived")
+
+
+if __name__ == "__main__":
+    unittest.main()

postgres/initdb.sql

@@ -2,7 +2,6 @@
 CREATE EXTENSION postgis;
 
 CREATE TABLE IF NOT EXISTS projects (
-    archive boolean,
     created timestamp,
     created_by varchar,
     geom geometry(MULTIPOLYGON, 4326),
@@ -48,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks (
     project_type_specifics json,
     PRIMARY KEY (project_id, group_id, task_id),
     FOREIGN KEY (project_id) REFERENCES projects (project_id),
-    FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id)
+    FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id)
 );
 
 CREATE INDEX IF NOT EXISTS tasks_task_id ON public.tasks
@@ -78,7 +77,7 @@ CREATE TABLE IF NOT EXISTS results (
     result int,
     PRIMARY KEY (project_id, group_id, task_id, user_id),
     FOREIGN KEY (project_id) REFERENCES projects (project_id),
-    FOREIGN KEY (project_id, group_id) REFERENCES GROUPS (project_id, group_id),
+    FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id),
     FOREIGN KEY (project_id, group_id, task_id) REFERENCES tasks (project_id, group_id, task_id),
     FOREIGN KEY (user_id) REFERENCES users (user_id)
 );