Merge pull request #500 from mapswipe/arbitrary_geom_changes_2

Add more properties based on osm and ohsome data to geojson and push them into postgres

Author: ElJocho

mapswipe_workers/mapswipe_workers/definitions.py

@@ -13,6 +13,8 @@
 LOGGING_FILE_PATH = os.path.join(DATA_PATH, "mapswipe_workers.log")
 
 OHSOME_API_LINK = "https://api.ohsome.org/v1/"
+OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/"
+
 # number of geometries for project geometries
 MAX_INPUT_GEOMETRIES = 10
 

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py

@@ -8,29 +8,15 @@ class Group(BaseGroup):
     def __init__(self, project: object, groupId: int) -> None:
         super().__init__(project, groupId)
 
-    def create_tasks(
-        self,
-        feature_ids: List,
-        feature_geometries: List,
-        center_points: List,
-        reference: List,
-        screen: List,
-    ) -> None:
+    def create_tasks(self, feature_ids: List, features: List,) -> None:
         """Create tasks for a group
 
         feature_geometries is a list of geometries or feature in geojson format.
         These consist two keys: Coordinates and type.
         Coordinates of four two pair coordinates.
         Every coordinate pair is a vertex.
         """
-        for i in range(0, len(feature_ids)):
-            task = Task(
-                self,
-                feature_ids[i],
-                feature_geometries[i],
-                center_points[i],
-                reference[i],
-                screen[i],
-            )
+        for i, feature_id in enumerate(feature_ids):
+            task = Task(self, feature_id, features[i])
             self.tasks.append(task)
         self.numberOfTasks = len(self.tasks)

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py

@@ -1,28 +1,5 @@
-import argparse
 import json
 
-from osgeo import ogr
-
-########################################################################################
-parser = argparse.ArgumentParser(description="Process some integers.")
-parser.add_argument(
-    "-i",
-    "--input_file",
-    required=False,
-    default=None,
-    type=str,
-    help="the input file containning the geometries as geojson",
-)
-parser.add_argument(
-    "-g",
-    "--group_size",
-    required=False,
-    default=50,
-    type=int,
-    help="the size of each group",
-)
-########################################################################################
-
 
 def group_input_geometries(input_geometries_file, group_size):
     """
@@ -43,17 +20,15 @@ def group_input_geometries(input_geometries_file, group_size):
         "feature_geometries" per group with given group id key
     """
 
-    driver = ogr.GetDriverByName("GeoJSON")
-    datasource = driver.Open(input_geometries_file, 0)
-    layer = datasource.GetLayer()
-
+    with open(input_geometries_file, "r") as infile:
+        layer = json.load(infile)
     groups = {}
 
-    # we will simply, we will start with min group id = 100
+    # we will simply start with min group id = 100
     group_id = 100
     group_id_string = f"g{group_id}"
     feature_count = 0
-    for feature in layer:
+    for feature in layer["features"]:
         feature_count += 1
         # feature count starts at 1
         # assuming group size would be 10
@@ -65,55 +40,11 @@ def group_input_geometries(input_geometries_file, group_size):
         try:
             groups[group_id_string]
         except KeyError:
-            # todo: change this and call feature_geoms -> geojson
-            groups[group_id_string] = {
-                "feature_ids": [],
-                "feature_geometries": [],
-                "center_points": [],
-                "reference": [],
-                "screen": [],
-            }
+            groups[group_id_string] = {"feature_ids": [], "features": []}
 
         # we use a new id here based on the count
         # since we are not sure that GetFID returns unique values
         groups[group_id_string]["feature_ids"].append(feature_count)
-        groups[group_id_string]["feature_geometries"].append(
-            json.loads(feature.GetGeometryRef().ExportToJson())
-        )
-
-        # from gdal documentation GetFieldAsDouble():
-        # OFTInteger fields will be cast to double.
-        # Other field types, or errors will result in a return value of zero.
-        center_x = feature.GetFieldAsDouble("center_x")
-        center_y = feature.GetFieldAsDouble("center_y")
-
-        # check if center attribute has been provided in geojson
-        # normal tasks will never have a center of 0.0, 0.0
-        # this is just in the middle of the ocean
-        if (center_x == 0.0) and (center_y == 0.0):
-            groups[group_id_string]["center_points"].append(None)
-        else:
-            groups[group_id_string]["center_points"].append([center_x, center_y])
-
-        # this is relevant for the tutorial
-        reference = feature.GetFieldAsDouble("reference")
-        screen = feature.GetFieldAsDouble("screen")
-        groups[group_id_string]["reference"].append(reference)
-        groups[group_id_string]["screen"].append(screen)
+        groups[group_id_string]["features"].append(feature)
 
     return groups
-
-
-# todo: groups should have this output formatting
-# reference is correct answer for tutorial,
-# if not tutorial reference = None
-# {"g100":[{"task_id":1, "screen": 1, "reference":0,
-# "geojson":{"valid geojson FeatureCollection Object"}}]}
-########################################################################################
-if __name__ == "__main__":
-
-    args = parser.parse_args()
-
-    groups = group_input_geometries(args.input_file, args.group_size)
-
-    print(groups)

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py

@@ -46,14 +46,14 @@ def handle_input_type(self, raw_input_file: str):
             # write string to geom file
             ohsome_request = {"endpoint": "elements/geometry", "filter": self.filter}
 
-            result = ohsome(ohsome_request, self.geometry)
+            result = ohsome(ohsome_request, self.geometry, properties="tags, metadata")
             with open(raw_input_file, "w") as geom_file:
                 json.dump(result, geom_file)
         elif self.inputType == "TMId":
             logger.info("TMId detected")
             hot_tm_project_id = int(self.TMId)
             ohsome_request = {"endpoint": "elements/geometry", "filter": self.filter}
-            result = ohsome(ohsome_request, self.geometry)
+            result = ohsome(ohsome_request, self.geometry, properties="tags, metadata")
             result["properties"] = {}
             result["properties"]["hot_tm_project_id"] = hot_tm_project_id
             with open(raw_input_file, "w") as geom_file:
@@ -139,7 +139,8 @@ def validate_geometries(self):
         for feature in layer:
             feat_geom = feature.GetGeometryRef()
             geom_name = feat_geom.GetGeometryName()
-            fid = feature.GetFID
+            fid = feature.GetFID()
+
             if not feat_geom.IsValid():
                 layer.DeleteFeature(fid)
                 logger.warning(
@@ -194,13 +195,7 @@ def create_groups(self):
 
         for group_id, item in raw_groups.items():
             group = Group(self, group_id)
-            group.create_tasks(
-                item["feature_ids"],
-                item["feature_geometries"],
-                item["center_points"],
-                item["reference"],
-                item["screen"],
-            )
+            group.create_tasks(item["feature_ids"], item["features"])
 
             # only append valid groups
             if group.is_valid():

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Union
+from typing import Dict, Union
 
 from osgeo import ogr
 
@@ -7,43 +7,30 @@
 
 class Task(BaseTask):
     def __init__(
-        self,
-        group: object,
-        featureId: Union[int, str],
-        featureGeometry: Dict,
-        center: Optional[List[float]],
-        reference: Optional[int],
-        screen: Optional[int],
+        self, group: object, featureId: Union[int, str], feature: Dict,
     ):
         """
         Parameters
         ----------
-        feature_geometry: dict
-            The geometries or feature in geojson format.
-            It consist of two keys: Coordinates and type.
-            Coordinates of four two pair coordinates.
-            Every coordinate pair is a vertex.
+        feature: dict
+            a Feature in geojson format.
         """
         task_id = f"t{featureId}"
         super().__init__(group, taskId=task_id)
-        self.geojson = featureGeometry
-
-        # only tasks that use Google tile map service need this
-        if center:
-            self.center = center
-
+        self.geojson = feature["geometry"]
+        self.properties = feature["properties"]
         # only tasks that are part of a tutorial need this
-        if screen:
-            self.screen = screen
-            self.reference = reference
+        if "screen" in feature["properties"].keys():
+            self.screen = feature["properties"]["screen"]
+            self.reference = feature["properties"]["reference"]
 
         # Remove projectId and groupId for tasks of Footprint project type
         del self.projectId
         del self.groupId
 
         # create wkt geometry from geojson
         # this geometry will be stored in postgres
-        # it will be remove before storing the data in firebase
-        poly = ogr.CreateGeometryFromJson(str(featureGeometry))
+        # it will be removed before storing the data in firebase
+        poly = ogr.CreateGeometryFromJson(str(self.geojson))
         wkt_geometry = poly.ExportToWkt()
         self.geometry = wkt_geometry

mapswipe_workers/mapswipe_workers/project_types/base/project.py

@@ -121,8 +121,10 @@ def save_project(self):
             del group["tasks"]
             groups[group["groupId"]] = group
         del project["groups"]
+
         project.pop("inputGeometries", None)
         project.pop("validInputGeometries", None)
+
         # Convert Date object to ISO Datetime:
         # https://www.w3.org/TR/NOTE-datetime
         project["created"] = self.created.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
@@ -230,7 +232,9 @@ def save_to_firebase(self, project, groups, groupsOfTasks):
                 # since the tasks hold geometries their storage size
                 # can get quite big otherwise
                 if self.projectType in [ProjectType.FOOTPRINT.value]:
-                    # todo: remove extra information from geojson
+                    # removing properties from each task
+                    for task in tasks_list:
+                        task.pop("properties", None)
 
                     compressed_tasks = gzip_str.compress_tasks(tasks_list)
                     task_upload_dict[
@@ -587,14 +591,22 @@ def create_tasks_txt_file(self, groupsOfTasks):
                 # these common attributes don't need to be written
                 # to the project_type_specifics since they are
                 # already stored in separate columns
-                common_attributes = ["projectId", "groupId", "taskId", "geometry"]
+                common_attributes = [
+                    "projectId",
+                    "groupId",
+                    "taskId",
+                    "geometry",
+                    "geojson",
+                ]
 
                 for key in task.keys():
                     if key not in common_attributes:
                         output_dict["project_type_specifics"][key] = task[key]
                 output_dict["project_type_specifics"] = json.dumps(
                     output_dict["project_type_specifics"]
-                )
+                ).replace(
+                    "'", ""
+                )  # to prevent error: invalid token "'"
 
                 w.writerow(output_dict)
         tasks_txt_file.close()

mapswipe_workers/mapswipe_workers/project_types/base/task.py

@@ -10,7 +10,7 @@ class BaseTask(object):
 
     def __init__(self, group, taskId):
         """
-            The Constructor Method for a task instance
+        The Constructor Method for a task instance
 
         Parameters
         ----------