Merge pull request #440 from mapswipe/dev

Manager Dashboard Docker Volume and Project Creation Bug Fixes

Author: Hagellach37

docker-compose.yaml

@@ -44,6 +44,8 @@ services:
         restart: unless-stopped
         expose:
             - "80"
+        volumes:
+            - ./manager_dashboard/manager_dashboard:/usr/share/nginx/html/manager_dashboard
         networks:
             - manager_dashboard
 

docs/source/cli.md

@@ -102,6 +102,46 @@ Options:
   --help             Show this message and exit.
 ```
 
+## Archive Projects
+
+```
+Usage: mapswipe_workers archive [OPTIONS]
+
+  Archive projects in Postgres. Delete groups, tasks and results from
+  Firebase.
+
+Options:
+  -i, --project-id TEXT  Archive project with giving project id
+  --project-ids TEXT     Archive multiple projects. Provide project id strings
+                         as a list: '["project_a", "project_b"]'
+
+  --help                 Show this message and exit.
+
+```
+
+You can get the project ids for the projects to be archive either from the manager dashboard or query directly in postgres. For example:
+
+```
+SELECT project_id
+FROM projects
+WHERE status = 'finished' AND created < '2020-09-01'
+```
+
+## Delete Projects
+
+```
+Usage: mapswipe_workers delete [OPTIONS]
+
+  Delete tasks, groups, project and results.
+
+Options:
+  -i, --project-id TEXT  Delete project with giving project id
+  --project-ids TEXT     Delete multiple projects. Provide project id strings
+                         as a list: '["project_a", "project_b"]'
+
+  --help                 Show this message and exit.
+```
+
 
 ## Create Tutorial from json file (e.g. provided in sample data)
 

manager_dashboard/Dockerfile

@@ -1,3 +1,2 @@
 FROM nginx:stable
-COPY nginx.conf /etc/nginx/conf.d/
-COPY manager_dashboard/ /usr/share/nginx/html/manager_dashboard/
+COPY nginx.conf /etc/nginx/conf.d/

mapswipe_workers/mapswipe_workers/definitions.py

@@ -145,6 +145,9 @@ def constructor(self):
     @property
     def tutorial(self):
         # Imports are first made once this method get called to avoid circular imports.
+        from mapswipe_workers.project_types.arbitrary_geometry.tutorial import (
+            Tutorial as ArbitraryGeometryTutorial,
+        )
         from mapswipe_workers.project_types.tile_map_service_grid.tutorial import (
             Tutorial as tmsg_tutorial,
         )
@@ -156,7 +159,7 @@ def tutorial(self):
 
         project_type_classes = {
             1: tmsg_tutorial,
-            # 2: ArbitraryGeometriesTutorial,
+            2: ArbitraryGeometryTutorial,
             3: tmsg_tutorial,
             4: tmsg_tutorial,
         }

mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py

@@ -69,6 +69,7 @@ def transfer(current_results):
                 f"{project_id}: Firebase transaction for "
                 f"transfering results failed to commit"
             )
+            sentry.capture_exception()
 
     del fb_db
     return project_id_list

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -90,10 +90,12 @@ def run_create_projects():
             project.save_project()
             send_slack_message(MessageType.SUCCESS, project_name, project.projectId)
             logger.info("Success: Project Creation ({0})".format(project_name))
-        except CustomError:
+        except CustomError as e:
             ref = fb_db.reference(f"v2/projectDrafts/{project_draft_id}")
             ref.set({})
-            send_slack_message(MessageType.FAIL, project_name, project.projectId)
+            send_slack_message(
+                MessageType.FAIL, project_name, project.projectId, str(e)
+            )
             logger.exception("Failed: Project Creation ({0}))".format(project_name))
             sentry.capture_exception()
         continue
@@ -118,6 +120,7 @@ def run_firebase_to_postgres() -> list:
     help=(
         "Project ids for which to generate stats as a list of strings: "
         """ '["project_a", "project_b"]' """
+        "(You need the quotes.)"
     ),
 )
 def run_generate_stats(project_ids: list) -> None:
@@ -142,8 +145,16 @@ def run_generate_stats_all_projects() -> None:
 
 
 @cli.command("user-management")
+@click.option("--email", help="The email of the MapSwipe user.", type=str)
 @click.option(
-    "--email", help="The email of the MapSwipe user.", required=True, type=str
+    "--emails",
+    cls=PythonLiteralOption,
+    default="[]",
+    help=(
+        "The emails of the MapSwipe users as a list of strings: "
+        """ '["email_a", "email_b"]' """
+        "(You need the quotes.)"
+    ),
 )
 @click.option("--team_id", "-i", help="The id of the team in Firebase.", type=str)
 @click.option(
@@ -158,27 +169,33 @@ def run_generate_stats_all_projects() -> None:
         ["add-manager-rights", "remove-manager-right", "add-team", "remove-team"]
     ),
 )
-def run_user_management(email, action, team_id) -> None:
+def run_user_management(email, emails, action, team_id) -> None:
     """
     Manage project manager credentials
 
     Grant or remove credentials.
     """
-    try:
-        if action == "add-manager-rights":
-            user_management.set_project_manager_rights(email)
-        elif action == "remove-manager-rights":
-            user_management.remove_project_manager_rights(email)
-        elif action == "add-team":
-            if not team_id:
-                click.echo("Missing argument: --team_id")
-                return None
-            user_management.add_user_to_team(email, team_id)
-        elif action == "remove-team":
-            user_management.remove_user_from_team(email)
-    except Exception:
-        logger.exception("grant user credentials failed")
-        sentry.capture_exception()
+    if email:
+        email_list = [email]
+    else:
+        email_list = emails
+
+    for email in email_list:
+        try:
+            if action == "add-manager-rights":
+                user_management.set_project_manager_rights(email)
+            elif action == "remove-manager-rights":
+                user_management.remove_project_manager_rights(email)
+            elif action == "add-team":
+                if not team_id:
+                    click.echo("Missing argument: --team_id")
+                    return None
+                user_management.add_user_to_team(email, team_id)
+            elif action == "remove-team":
+                user_management.remove_user_from_team(email)
+        except Exception:
+            logger.exception("grant user credentials failed")
+            sentry.capture_exception()
 
 
 @cli.command("team-management")
@@ -298,7 +315,7 @@ def run_create_tutorials() -> None:
     help=(
         "Archive multiple projects. "
         "Provide project id strings as a list: "
-        """["project_a", "project_b"]"""
+        """'["project_a", "project_b"]'"""
     ),
 )
 def run_archive_project(project_id, project_ids):
@@ -326,7 +343,7 @@ def run_archive_project(project_id, project_ids):
     help=(
         "Delete multiple projects. "
         "Provide project id strings as a list: "
-        """["project_a", "project_b"]"""
+        """'["project_a", "project_b"]'"""
     ),
 )
 def run_delete_project(project_id, project_ids):

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py

@@ -9,7 +9,12 @@ def __init__(self, project: object, groupId: int) -> None:
         super().__init__(project, groupId)
 
     def create_tasks(
-        self, feature_ids: List, feature_geometries: List, center_points: List
+        self,
+        feature_ids: List,
+        feature_geometries: List,
+        center_points: List,
+        reference: List,
+        screen: List,
     ) -> None:
         """Create tasks for a group
 
@@ -19,6 +24,13 @@ def create_tasks(
         Every coordinate pair is a vertex.
         """
         for i in range(0, len(feature_ids)):
-            task = Task(self, feature_ids[i], feature_geometries[i], center_points[i])
+            task = Task(
+                self,
+                feature_ids[i],
+                feature_geometries[i],
+                center_points[i],
+                reference[i],
+                screen[i],
+            )
             self.tasks.append(task)
         self.numberOfTasks = len(self.tasks)

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py

@@ -66,18 +66,34 @@ def group_input_geometries(input_geometries_file, group_size):
                 "feature_ids": [],
                 "feature_geometries": [],
                 "center_points": [],
+                "reference": [],
+                "screen": [],
             }
 
         groups[group_id_string]["feature_ids"].append(feature.GetFID())
         groups[group_id_string]["feature_geometries"].append(
             json.loads(feature.GetGeometryRef().ExportToJson())
         )
-        try:
-            center_x = feature.GetFieldAsDouble("center_x")
-            center_y = feature.GetFieldAsDouble("center_y")
+
+        # from gdal documentation GetFieldAsDouble():
+        # OFTInteger fields will be cast to double.
+        # Other field types, or errors will result in a return value of zero.
+        center_x = feature.GetFieldAsDouble("center_x")
+        center_y = feature.GetFieldAsDouble("center_y")
+
+        # check if center attribute has been provided in geojson
+        # normal tasks will never have a center of 0.0, 0.0
+        # this is just in the middle of the ocean
+        if (center_x == 0.0) and (center_y == 0.0):
+            groups[group_id_string]["center_points"].append(None)
+        else:
             groups[group_id_string]["center_points"].append([center_x, center_y])
-        except:  # noqa
-            groups[group_id_string]["center_points"].append([])
+
+        # this is relevant for the tutorial
+        reference = feature.GetFieldAsDouble("reference")
+        screen = feature.GetFieldAsDouble("screen")
+        groups[group_id_string]["reference"].append(reference)
+        groups[group_id_string]["screen"].append(screen)
 
     return groups
 

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py

@@ -145,7 +145,11 @@ def create_groups(self):
         for group_id, item in raw_groups.items():
             group = Group(self, group_id)
             group.create_tasks(
-                item["feature_ids"], item["feature_geometries"], item["center_points"]
+                item["feature_ids"],
+                item["feature_geometries"],
+                item["center_points"],
+                item["reference"],
+                item["screen"],
             )
 
             # only append valid groups

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py

@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Dict, List, Optional, Union
 
 from osgeo import ogr
 
@@ -7,7 +7,13 @@
 
 class Task(BaseTask):
     def __init__(
-        self, group: object, featureId: int, featureGeometry: Dict, center: List
+        self,
+        group: object,
+        featureId: Union[int, str],
+        featureGeometry: Dict,
+        center: Optional[List[float]],
+        reference: Optional[int],
+        screen: Optional[int],
     ):
         """
         Parameters
@@ -21,13 +27,23 @@ def __init__(
         task_id = f"t{featureId}"
         super().__init__(group, taskId=task_id)
         self.geojson = featureGeometry
-        self.center = center
+
+        # only tasks that use Google tile map service need this
+        if center:
+            self.center = center
+
+        # only tasks that are part of a tutorial need this
+        if screen:
+            self.screen = screen
+            self.reference = reference
 
         # Remove projectId and groupId for tasks of Footprint project type
         del self.projectId
         del self.groupId
 
         # create wkt geometry from geojson
+        # this geometry will be stored in postgres
+        # it will be remove before storing the data in firebase
         poly = ogr.CreateGeometryFromJson(str(featureGeometry))
         wkt_geometry = poly.ExportToWkt()
         self.geometry = wkt_geometry