Merge branch 'feature/street' of https://github.com/mapswipe/python-mapswipe-workers into feature/street

Author: Gigaszi

.github/workflows/actions.yml

@@ -80,6 +80,7 @@ jobs:
         POSTGRES_DB: postgres
         OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }}
         DJANGO_SECRET_KEY: test-django-secret-key
+        MAPILLARY_API_KEY: test-mapillary-api-key
         COMPOSE_FILE: ../docker-compose.yaml:../docker-compose-ci.yaml
       run: |
         docker compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/

mapswipe_workers/mapswipe_workers/definitions.py

@@ -17,7 +17,7 @@
 OSMCHA_API_LINK = "https://osmcha.org/api/v1/"
 OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"]
 MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/"
-MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"]
+MAPILLARY_API_KEY = os.getenv("MAPILLARY_API_KEY")
 
 # number of geometries for project geometries
 MAX_INPUT_GEOMETRIES = 10

mapswipe_workers/mapswipe_workers/project_types/__init__.py

@@ -2,13 +2,13 @@
 from .arbitrary_geometry.footprint.project import FootprintProject
 from .arbitrary_geometry.footprint.tutorial import FootprintTutorial
 from .media_classification.project import MediaClassificationProject
+from .street.project import StreetProject
 from .tile_map_service.change_detection.project import ChangeDetectionProject
 from .tile_map_service.change_detection.tutorial import ChangeDetectionTutorial
 from .tile_map_service.classification.project import ClassificationProject
 from .tile_map_service.classification.tutorial import ClassificationTutorial
 from .tile_map_service.completeness.project import CompletenessProject
 from .tile_map_service.completeness.tutorial import CompletenessTutorial
-from .street.project import StreetProject
 
 __all__ = [
     "ClassificationProject",

mapswipe_workers/mapswipe_workers/project_types/street/project.py

@@ -1,11 +1,8 @@
-import json
-import os
-import urllib
 import math
-
-from osgeo import ogr
 from dataclasses import dataclass
-from mapswipe_workers.definitions import DATA_PATH, logger
+from typing import Dict, List
+
+from mapswipe_workers.definitions import logger
 from mapswipe_workers.firebase.firebase import Firebase
 from mapswipe_workers.firebase_to_postgres.transfer_results import (
     results_to_file,
@@ -15,16 +12,16 @@
 from mapswipe_workers.generate_stats.project_stats import (
     get_statistics_for_integer_result_project,
 )
+from mapswipe_workers.project_types.project import BaseGroup, BaseProject, BaseTask
+from mapswipe_workers.utils.process_mapillary import get_image_metadata
 from mapswipe_workers.utils.validate_input import (
-    check_if_layer_is_empty,
-    load_geojson_to_ogr,
     build_multipolygon_from_layer_geometries,
     check_if_layer_has_too_many_geometries,
-    save_geojson_to_file,
+    check_if_layer_is_empty,
+    load_geojson_to_ogr,
     multipolygon_to_wkt,
+    save_geojson_to_file,
 )
-from mapswipe_workers.project_types.project import BaseProject, BaseTask, BaseGroup
-from mapswipe_workers.utils.process_mapillary import get_image_metadata
 
 
 @dataclass

mapswipe_workers/mapswipe_workers/utils/process_mapillary.py

@@ -1,23 +1,22 @@
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 import mercantile
-import json
+import pandas as pd
 import requests
-import os
-import time
 from shapely import (
-    box,
-    Polygon,
-    MultiPolygon,
-    Point,
     LineString,
     MultiLineString,
+    MultiPolygon,
+    Point,
+    Polygon,
+    box,
     unary_union,
 )
 from shapely.geometry import shape
-import pandas as pd
 from vt2geojson import tools as vt2geojson_tools
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY
-from mapswipe_workers.definitions import logger
+
+from mapswipe_workers.definitions import MAPILLARY_API_KEY, MAPILLARY_API_LINK, logger
 from mapswipe_workers.utils.spatial_sampling import spatial_sampling
 
 
@@ -137,8 +136,8 @@ def coordinate_download(
                 downloaded_metadata[col] = None
 
         if (
-            downloaded_metadata.isna().all().all() == False
-            or downloaded_metadata.empty == True
+            downloaded_metadata.isna().all().all() is False
+            or downloaded_metadata.empty is True
         ):
             downloaded_metadata = downloaded_metadata[
                 downloaded_metadata["geometry"].apply(
@@ -240,8 +239,8 @@ def get_image_metadata(
     if sampling_threshold is not None:
         downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold)
     if (
-        downloaded_metadata.isna().all().all() == False
-        or downloaded_metadata.empty == False
+        downloaded_metadata.isna().all().all() is False
+        or downloaded_metadata.empty is False
     ):
         if len(downloaded_metadata) > 100000:
             err = (

mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py

@@ -1,26 +1,29 @@
 import numpy as np
 import pandas as pd
-from shapely import wkt
-from shapely.geometry import Point
 
 
 def distance_on_sphere(p1, p2):
     """
-    p1 and p2 are two lists that have two elements. They are numpy arrays of the long and lat
-    coordinates of the points in set1 and set2
+    p1 and p2 are two lists that have two elements. They are numpy arrays of the long
+    and lat coordinates of the points in set1 and set2
 
-    Calculate the distance between two points on the Earth's surface using the haversine formula.
+    Calculate the distance between two points on the Earth's surface using the
+    haversine formula.
 
     Args:
-        p1 (list): Array containing the longitude and latitude coordinates of points FROM which the distance to be calculated in degree
-        p2 (list): Array containing the longitude and latitude coordinates of points TO which the distance to be calculated in degree
+        p1 (list): Array containing the longitude and latitude coordinates of points
+        FROM which the distance to be calculated in degree
+        p2 (list): Array containing the longitude and latitude coordinates of points
+        TO which the distance to be calculated in degree
 
     Returns:
-        numpy.ndarray: Array containing the distances between the two points on the sphere in kilometers.
+        numpy.ndarray: Array containing the distances between the two points on the
+        sphere in kilometers.
 
-    This function computes the distance between two points on the Earth's surface using the haversine formula,
-    which takes into account the spherical shape of the Earth. The input arrays `p1` and `p2` should contain
-    longitude and latitude coordinates in degrees. The function returns an array containing the distances
+    This function computes the distance between two points on the Earth's surface
+    using the haversine formula, which takes into account the spherical shape of the
+    Earth. The input arrays `p1` and `p2` should contain longitude and latitude
+    coordinates in degrees. The function returns an array containing the distances
     between corresponding pairs of points.
     """
     earth_radius = 6371  # km
@@ -41,7 +44,7 @@ def distance_on_sphere(p1, p2):
     return distances
 
 
-"""-----------------------------------Filtering Points------------------------------------------------"""
+"""----------------------------Filtering Points-------------------------------"""
 
 
 def filter_points(df, threshold_distance):
@@ -56,10 +59,11 @@ def filter_points(df, threshold_distance):
         pandas.DataFrame: Filtered DataFrame containing selected points.
         float: Total road length calculated from the selected points.
 
-    This function filters points from a DataFrame based on the given threshold distance. It calculates
-    distances between consecutive points and accumulates them until the accumulated distance surpasses
-    the threshold distance. It then selects those points and constructs a new DataFrame. Additionally,
-    it manually checks the last point to include it if it satisfies the length condition. The function
+    This function filters points from a DataFrame based on the given threshold
+    distance. It calculates distances between consecutive points and accumulates them
+    until the accumulated distance surpasses the threshold distance. It then selects
+    those points and constructs a new DataFrame. Additionally, it manually checks the
+    last point to include it if it satisfies the length condition. The function
     returns the filtered DataFrame along with the calculated road length.
     """
     road_length = 0
@@ -83,7 +87,8 @@ def filter_points(df, threshold_distance):
             accumulated_distance = 0  # Reset accumulated distance
 
     to_be_returned_df = df[mask]
-    # since the last point has to be omitted in the vectorized distance calculation, it is being checked manually
+    # since the last point has to be omitted in the vectorized distance calculation,
+    # it is being checked manually
     p2 = to_be_returned_df.iloc[0]
     distance = distance_on_sphere(
         [float(p2["long"]), float(p2["lat"])], [long[-1], lat[-1]]
@@ -114,10 +119,10 @@ def spatial_sampling(df, interval_length):
         geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points.
         float: Total road length calculated from the selected points.
 
-    This function calculates the spacing between points in a GeoDataFrame by filtering points
-    based on the provided interval length. It first sorts the GeoDataFrame by timestamp and
-    then filters points using the filter_points function. The function returns the filtered
-    GeoDataFrame along with the total road length.
+    This function calculates the spacing between points in a GeoDataFrame by filtering
+    points based on the provided interval length. It first sorts the GeoDataFrame by
+    timestamp and then filters points using the filter_points function. The function
+    returns the filtered GeoDataFrame along with the total road length.
     """
     if len(df) == 1:
         return df

mapswipe_workers/requirements.txt

@@ -14,3 +14,6 @@ sentry-sdk==0.18.0
 six==1.15.0
 slackclient==2.9.2
 xdg==4.0.1
+shapely
+mercantile
+vt2geojson

mapswipe_workers/tests/integration/test_create_street_project.py

@@ -8,7 +8,7 @@
 from tests.integration import set_up, tear_down
 
 
-class TestCreateFootprintProject(unittest.TestCase):
+class TestCreateStreetProject(unittest.TestCase):
     def setUp(self):
         self.project_id = [
             set_up.create_test_project_draft("street", "street"),
@@ -33,7 +33,7 @@ def test_create_street_project(self):
             result = pg_db.retr_query(query, [element])[0][0]
             self.assertEqual(result, element)
 
-            # check if usernames made it to postgres
+            # check if tasks made it to postgres
             query = """
                 SELECT count(*)
                 FROM tasks
@@ -51,10 +51,11 @@ def test_create_street_project(self):
             result = ref.get(shallow=True)
             self.assertIsNotNone(result)
 
-            # Footprint projects have tasks in Firebase
+            # Street projects have tasks in Firebase
             ref = fb_db.reference(f"/v2/tasks/{element}")
             result = ref.get(shallow=True)
             self.assertIsNotNone(result)
 
+
 if __name__ == "__main__":
     unittest.main()