Merge pull request #348 from mapswipe/dev

Dev

Author: Hagellach37

mapswipe_workers/mapswipe_workers/base/base_project.py

@@ -1,17 +1,22 @@
-import logging
 import logging.config
 import os
-
 import sentry_sdk
 from xdg import XDG_DATA_HOME
-
 from mapswipe_workers.config import SENTRY_DSN
+from enum import Enum
 
 
 class CustomError(Exception):
     pass
 
 
+class MessageType(Enum):
+    SUCCESS = 1
+    FAIL = 2
+    NOTIFICATION_90 = 3
+    NOTIFICATION_100 = 4
+
+
 DATA_PATH = os.path.join(XDG_DATA_HOME, "mapswipe_workers")
 if not os.path.exists(DATA_PATH):
     os.makedirs(DATA_PATH)

mapswipe_workers/mapswipe_workers/definitions.py

@@ -8,7 +8,7 @@
 
 import click
 from mapswipe_workers import auth
-from mapswipe_workers.definitions import CustomError, logger, sentry
+from mapswipe_workers.definitions import CustomError, logger, sentry, MessageType
 from mapswipe_workers.firebase_to_postgres import (
     archive_project,
     delete_project,
@@ -27,7 +27,10 @@
 from mapswipe_workers.project_types.footprint.footprint_project import FootprintProject
 from mapswipe_workers.utils import user_management
 from mapswipe_workers.utils.create_directories import create_directories
-from mapswipe_workers.utils.slack_helper import send_slack_message
+from mapswipe_workers.utils.slack_helper import (
+    send_slack_message,
+    send_progress_notification,
+)
 
 
 class PythonLiteralOption(click.Option):
@@ -86,12 +89,12 @@ def run_create_projects():
             project.calc_required_results()
             # Save project and its groups and tasks to Firebase and Postgres.
             project.save_project()
-            send_slack_message("success", project_name, project.projectId)
+            send_slack_message(MessageType.SUCCESS, project_name, project.projectId)
             logger.info("Success: Project Creation ({0})".format(project_name))
         except CustomError:
             ref = fb_db.reference(f"v2/projectDrafts/{project_draft_id}")
             ref.set({})
-            send_slack_message("fail", project_name, project.projectId)
+            send_slack_message(MessageType.FAIL, project_name, project.projectId)
             logger.exception("Failed: Project Creation ({0}))".format(project_name))
             sentry.capture_exception()
         continue
@@ -103,6 +106,8 @@ def run_firebase_to_postgres() -> list:
     update_data.update_user_data()
     update_data.update_project_data()
     project_ids = transfer_results.transfer_results()
+    for project_id in project_ids:
+        send_progress_notification(project_id)
     return project_ids
 
 

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -27,7 +27,7 @@ def csv_to_geojson(filename: str, geometry_field: str = "geom"):
             outfile,
             filename,
             "-sql",
-            f'SELECT *, CAST({geometry_field} as geometry) FROM "{filename_without_path}"',
+            f'SELECT *, CAST({geometry_field} as geometry) FROM "{filename_without_path}"',  # noqa E501
         ],
         check=True,
     )
@@ -101,28 +101,60 @@ def add_metadata_to_geojson(filename: str, geometry_field: str = "geom"):
     logger.info(f"added metadata to {filename}.")
 
 
-def create_group_geom(group_data):
+def create_group_geom(group_data, shape="bounding_box"):
+    """Create bounding box or convex hull of input task geometries."""
+
     result_geom_collection = ogr.Geometry(ogr.wkbMultiPolygon)
     for result, data in group_data.items():
         result_geom = ogr.CreateGeometryFromWkt(data["wkt"])
         result_geom_collection.AddGeometry(result_geom)
 
-    group_geom = result_geom_collection.ConvexHull()
+    if shape == "convex_hull":
+        group_geom = result_geom_collection.ConvexHull()
+    elif shape == "bounding_box":
+        # Get Envelope
+        lon_left, lon_right, lat_top, lat_bottom = result_geom_collection.GetEnvelope()
+
+        # Create Geometry
+        ring = ogr.Geometry(ogr.wkbLinearRing)
+        ring.AddPoint(lon_left, lat_top)
+        ring.AddPoint(lon_right, lat_top)
+        ring.AddPoint(lon_right, lat_bottom)
+        ring.AddPoint(lon_left, lat_bottom)
+        ring.AddPoint(lon_left, lat_top)
+        # TODO: Make sure to return 2D geom, currently 3D with z = 0.0
+        group_geom = ogr.Geometry(ogr.wkbPolygon)
+        group_geom.AddGeometry(ring)
+
     return group_geom
 
 
 def create_geojson_file_from_dict(final_groups_dict, outfile):
-    # TODO: adapt input name
+    """Take output from generate stats and create TM geometries.
+
+    In order to create a GeoJSON file with a coordinate precision of 7
+    we take a small detour.
+    First, we create a GeoJSONSeq file.
+    This contains only the features.
+    Then we add these features to the final GeoJSON file.
+    The current shape of the output geometries is set to 'bounding_box'.
+    """
 
-    driver = ogr.GetDriverByName("GeoJSON")
+    driver = ogr.GetDriverByName("GeoJSONSeq")
     # define spatial Reference
     srs = osr.SpatialReference()
     srs.ImportFromEPSG(4326)
+    outfile_temp = outfile.replace(".geojson", "_temp.geojson")
+
+    if os.path.exists(outfile_temp):
+        driver.DeleteDataSource(outfile_temp)
+
     if os.path.exists(outfile):
         driver.DeleteDataSource(outfile)
-    dataSource = driver.CreateDataSource(outfile)
+
+    dataSource = driver.CreateDataSource(outfile_temp)
     # create layer
-    layer = dataSource.CreateLayer(outfile, srs, geom_type=ogr.wkbPolygon)
+    layer = dataSource.CreateLayer(outfile_temp, srs, geom_type=ogr.wkbPolygon,)
 
     # create fields
     field_id = ogr.FieldDefn("group_id", ogr.OFTInteger)
@@ -133,7 +165,8 @@ def create_geojson_file_from_dict(final_groups_dict, outfile):
     else:
         for group_id in final_groups_dict.keys():
             group_data = final_groups_dict[group_id]
-            group_geom = create_group_geom(group_data)
+            # create the final group geometry
+            group_geom = create_group_geom(group_data, "bounding_box")
             final_groups_dict[group_id]["group_geom"] = group_geom
             # init feature
 
@@ -163,21 +196,54 @@ def create_geojson_file_from_dict(final_groups_dict, outfile):
                 print(group_geom)
                 continue
 
+    # make sure to close layer and data source
     layer = None
-    logger.info("created outfile: %s." % outfile)
+    dataSource = None
+
+    # load the features from temp file
+    feature_collection = []
+    with open(outfile_temp, "r") as f:
+        for cnt, line in enumerate(f):
+            feature_collection.append(json.loads(line))
+
+    # create final geojson structure
+    geojson_structure = {
+        "type": "FeatureCollection",
+        "name": outfile,
+        "crs": {
+            "type": "name",
+            "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"},
+        },
+        "features": feature_collection,
+    }
+
+    # save to geojson
+    with open(outfile, "w") as json_file:
+        json.dump(geojson_structure, json_file)
+        logger.info("created outfile: %s." % outfile)
+
+    # remove temp file
+    if os.path.exists(outfile_temp):
+        driver.DeleteDataSource(outfile_temp)
 
 
 def create_geojson_file(geometries, outfile):
 
-    driver = ogr.GetDriverByName("GeoJSON")
+    driver = ogr.GetDriverByName("GeoJSONSeq")
     # define spatial Reference
     srs = osr.SpatialReference()
     srs.ImportFromEPSG(4326)
+    outfile_temp = outfile.replace(".geojson", "_temp.geojson")
+
+    if os.path.exists(outfile_temp):
+        driver.DeleteDataSource(outfile_temp)
+
     if os.path.exists(outfile):
         driver.DeleteDataSource(outfile)
-    dataSource = driver.CreateDataSource(outfile)
+
+    dataSource = driver.CreateDataSource(outfile_temp)
     # create layer
-    layer = dataSource.CreateLayer(outfile, srs, geom_type=ogr.wkbPolygon)
+    layer = dataSource.CreateLayer(outfile_temp, srs, geom_type=ogr.wkbPolygon,)
 
     # create fields
     field_id = ogr.FieldDefn("id", ogr.OFTInteger)
@@ -198,5 +264,33 @@ def create_geojson_file(geometries, outfile):
             # add feature to layer
             layer.CreateFeature(feature)
 
+    # make sure to close layer and data source
     layer = None
+    dataSource = None
+
+    # load the features from temp file
+    feature_collection = []
+    with open(outfile_temp, "r") as f:
+        for cnt, line in enumerate(f):
+            feature_collection.append(json.loads(line))
+
+    # create final geojson structure
+    geojson_structure = {
+        "type": "FeatureCollection",
+        "name": outfile,
+        "crs": {
+            "type": "name",
+            "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"},
+        },
+        "features": feature_collection,
+    }
+    # save to geojson
+    with open(outfile, "w") as json_file:
+        json.dump(geojson_structure, json_file)
+        logger.info("created outfile: %s." % outfile)
+
+    # remove temp file
+    if os.path.exists(outfile_temp):
+        driver.DeleteDataSource(outfile_temp)
+
     logger.info("created outfile: %s." % outfile)

mapswipe_workers/mapswipe_workers/utils/geojson_functions.py

@@ -1,15 +1,14 @@
-"""Initialize slack client with values provided by the config file"""
-
-import json
-
 import slack
-
 from mapswipe_workers.definitions import logger
+from mapswipe_workers import auth
 from mapswipe_workers.config import SLACK_CHANNEL, SLACK_TOKEN
+from mapswipe_workers.definitions import MessageType
 
 
-def send_slack_message(message_type: str, project_name: str, project_id: str = None):
-
+def send_slack_message(
+    message_type: MessageType, project_name: str, project_id: str = None
+):
+    """Initialize slack client with values provided in environment."""
     if SLACK_TOKEN is None or SLACK_CHANNEL is None:
         logger.info(
             "No configuration for Slack was found. "
@@ -19,22 +18,68 @@ def send_slack_message(message_type: str, project_name: str, project_id: str = N
 
     slack_client = slack.WebClient(token=SLACK_TOKEN)
 
-    if message_type == "success":
+    if message_type == MessageType.SUCCESS:
         message = (
             "### PROJECT CREATION SUCCESSFUL ###\n"
-            + "Project Name: {0}\n".format(project_name)
-            + "Project Id: {0}\n\n".format(project_id)
+            + f"Project Name: {project_name}\n"
+            + f"Project Id: {project_id}\n\n"
             + "Make sure to activate the project using the manager dashboard.\n"
             + "Happy Swiping. :)"
         )
-    elif message_type == "fail":
+        slack_client.chat_postMessage(channel=SLACK_CHANNEL, text=message)
+    elif message_type == MessageType.FAIL:
         message = (
             "### PROJECT CREATION FAILED ###\n"
-            + "Project Name: {0}\n".format(project_name)
+            + f"Project Name: {project_name}\n"
             + "Project draft is deleted."
         )
+        slack_client.chat_postMessage(channel=SLACK_CHANNEL, text=message)
+    elif message_type == MessageType.NOTIFICATION_90:
+        message = (
+            "### ALMOST THERE! PROJECT REACHED 90% ###\n"
+            + f"Project Name: {project_name}\n"
+            + f"Project Id: {project_id}\n\n"
+            + "Get your next projects ready."
+        )
+        slack_client.chat_postMessage(channel="mapswipe_managers", text=message)
+    elif message_type == MessageType.NOTIFICATION_100:
+        message = (
+            "### GREAT! PROJECT REACHED 100% ###\n"
+            + f"Project Name: {project_name}\n"
+            + f"Project Id: {project_id}\n\n"
+            + "You can set this project to 'finished' "
+            + "and activate another one."
+        )
+        slack_client.chat_postMessage(channel="mapswipe_managers", text=message)
     else:
         # TODO: Raise an Exception
         pass
 
-    slack_client.chat_postMessage(channel=SLACK_CHANNEL, text=message)
+
+def send_progress_notification(project_id: int):
+    """Send progress notification to project managers in Slack."""
+    fb_db = auth.firebaseDB()
+    progress = fb_db.reference(f"v2/projects/{project_id}/progress").get()
+
+    if progress >= 90:
+        project_name = fb_db.reference(f"v2/projects/{project_id}/name").get()
+        notification_90_sent = fb_db.reference(
+            f"v2/projects/{project_id}/notification_90_sent"
+        ).get()
+        notification_100_sent = fb_db.reference(
+            f"v2/projects/{project_id}/notification_100_sent"
+        ).get()
+        logger.info(
+            f"{project_id} - progress: {progress},"
+            f"notifications: {notification_90_sent} {notification_100_sent}"
+        )
+
+        if progress >= 90 and not notification_90_sent:
+            # send notification and set value in firebase
+            send_slack_message(MessageType.NOTIFICATION_90, project_name, project_id)
+            fb_db.reference(f"v2/projects/{project_id}/notification_90_sent").set(True)
+
+        if progress >= 100 and not notification_100_sent:
+            # send notification and set value in firebase
+            send_slack_message(MessageType.NOTIFICATION_100, project_name, project_id)
+            fb_db.reference(f"v2/projects/{project_id}/notification_100_sent").set(True)

mapswipe_workers/mapswipe_workers/utils/slack_helper.py

@@ -0,0 +1,38 @@
+from mapswipe_workers.definitions import DATA_PATH
+from mapswipe_workers import auth
+from mapswipe_workers.utils import geojson_functions
+import ogr
+
+
+def add_project_geometries_to_api():
+    """Load project geometries from postgres and save as geojson."""
+
+    # load from postgres
+    pg_db = auth.postgresDB()
+    sql_query = """
+        SELECT
+            project_id
+            ,ST_AsText(geom) as geom
+        FROM projects
+        """
+    data = pg_db.retr_query(sql_query)
+    print(len(data))
+
+    # save as geojson one by one
+    for project in data:
+        project_id = project[0]
+        wkt_geom = project[1]
+
+        outfile = (
+            f"{DATA_PATH}/api/project_geometries/project_geom_{project_id}.geojson"
+        )
+        try:
+            geometries = [ogr.CreateGeometryFromWkt(wkt_geom)]
+            geojson_functions.create_geojson_file(geometries, outfile)
+        except Exception:
+            print(f"got an error for {project_id}")
+            # just ignore if this fails
+            pass
+
+
+add_project_geometries_to_api()