feat: add spatial sampling to street project

Author: Gigaszi

mapswipe_workers/mapswipe_workers/project_types/street/project.py

@@ -54,6 +54,7 @@ def __init__(self, project_draft):
             start_time=project_draft.get("startTimestamp", None),
             end_time=project_draft.get("endTimestamp", None),
             organization_id=project_draft.get("organizationId", None),
+            sampling_threshold=project_draft.get("samplingThreshold", None),
         )
 
 

mapswipe_workers/mapswipe_workers/utils/process_mapillary.py

@@ -18,6 +18,7 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY
 from mapswipe_workers.definitions import logger
+from mapswipe_workers.utils.spatial_sampling import spatial_sampling
 
 
 def create_tiles(polygon, level):
@@ -219,14 +220,21 @@ def get_image_metadata(
     organization_id: str = None,
     start_time: str = None,
     end_time: str = None,
+    sampling_threshold = None,
 ):
     aoi_polygon = geojson_to_polygon(aoi_geojson)
     downloaded_metadata, failed_tiles = coordinate_download(
         aoi_polygon, level, attempt_limit
     )
+    downloaded_metadata = downloaded_metadata[
+        downloaded_metadata['geometry'].apply(lambda geom: isinstance(geom, Point))
+    ]
+
     downloaded_metadata = filter_results(
         downloaded_metadata, is_pano, organization_id, start_time, end_time
     )
+    if sampling_threshold is not None:
+        downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold)
     if downloaded_metadata.isna().all().all() == False:
         return {
             "ids": downloaded_metadata["id"].tolist(),

mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py

@@ -58,11 +58,8 @@ def filter_points(df, threshold_distance):
     road_length = 0
     mask = np.zeros(len(df), dtype=bool)
     mask[0] = True
-    lat = np.array([wkt.loads(point).y for point in df['data']])
-    long = np.array([wkt.loads(point).x for point in df['data']])
-
-    df['lat'] = lat
-    df['long'] = long
+    lat = df["lat"].to_numpy()
+    long = df["long"].to_numpy()
 
 
     distances = distance_on_sphere([long[1:],lat[1:]],
@@ -92,7 +89,7 @@ def filter_points(df, threshold_distance):
     return to_be_returned_df
 
 
-def calculate_spacing(df, interval_length):
+def spatial_sampling(df, interval_length):
     """
     Calculate spacing between points in a GeoDataFrame.
 
@@ -109,9 +106,22 @@ def calculate_spacing(df, interval_length):
     then filters points using the filter_points function. The function returns the filtered
     GeoDataFrame along with the total road length.
     """
-    road_length = 0
     if len(df) == 1:
         return df
-    sorted_sub_df = df.sort_values(by=['timestamp'])
-    filtered_sorted_sub_df = filter_points(sorted_sub_df,interval_length)
-    return filtered_sorted_sub_df
+
+    df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None)
+    df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None)
+    sorted_df = df.sort_values(by=['captured_at'])
+
+    sampled_sequence_df = pd.DataFrame()
+
+    # loop through each sequence
+    for sequence in sorted_df['sequence_id'].unique():
+        sequence_df = sorted_df[sorted_df['sequence_id'] == sequence]
+
+        filtered_sorted_sub_df = filter_points(sequence_df,interval_length)
+        sampled_sequence_df = pd.concat([sampled_sequence_df,filtered_sorted_sub_df],axis=0)
+
+
+
+    return sampled_sequence_df

mapswipe_workers/tests/fixtures/mapillary_sequence.csv

@@ -1,4 +1,4 @@
-,data,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id
+,geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id
 9,POINT (38.995129466056824 -6.785243670271996),1453463352000.0,102506575322825.0,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ,
 12,POINT (38.9906769990921 -6.783315348346505),1453463400000.0,102506575322825.0,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ,
 14,POINT (39.00127172470093 -6.787981661065601),1453463294000.0,102506575322825.0,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ,

mapswipe_workers/tests/fixtures/projectDrafts/street.json

@@ -46,6 +46,6 @@
     "requestingOrganisation": "test",    
     "verificationNumber": 3,
     "groupSize": 25,
-    "isPano": false,
-    "startTimestamp": "2019-07-01T00:00:00.000Z"
+    "startTimestamp": "2019-07-01T00:00:00.000Z",
+    "samplingThreshold": 0.1
 }

mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json

@@ -46,9 +46,5 @@
     "requestingOrganisation": "test",    
     "verificationNumber": 3,
     "groupSize": 25,
-    "isPano": false,
-    "startTimestamp": "2019-07-01T00:00:00.000Z",
-    "endTimestamp": null,
-    "organisationId": "1",
     "customOptions": [{ "color": "", "label": "", "value": -999 }, { "color": "#008000", "label": "yes", "value": 1 }, { "color": "#FF0000", "label": "no", "value": 2 }, { "color": "#FFA500", "label": "maybe", "value": 3 }]
 }

mapswipe_workers/tests/unittests/test_process_mapillary.py

@@ -9,6 +9,7 @@
     MultiLineString,
     GeometryCollection,
 )
+from shapely import wkt
 import pandas as pd
 from unittest.mock import patch, MagicMock
 from mapswipe_workers.utils.process_mapillary import (
@@ -45,7 +46,9 @@ def setUpClass(cls):
             ),
             "r",
         ) as file:
-            cls.fixture_df = pd.read_csv(file)
+            df = pd.read_csv(file)
+            df['geometry'] = df['geometry'].apply(wkt.loads)
+            cls.fixture_df = df
 
     def setUp(self):
         self.level = 14

mapswipe_workers/tests/unittests/test_project_type_street.py

@@ -2,6 +2,7 @@
 import os
 import unittest
 from unittest.mock import patch
+from shapely import wkt
 import pandas as pd
 
 from mapswipe_workers.project_types import StreetProject
@@ -30,7 +31,10 @@ def setUp(self) -> None:
                 ),
                 "r",
             ) as file:
-                mock_get.return_value = (pd.read_csv(file), None)
+                df = pd.read_csv(file)
+                df['geometry'] = df['geometry'].apply(wkt.loads)
+
+                mock_get.return_value = (df, None)
                 self.project = StreetProject(project_draft)
 
     def test_init(self):

mapswipe_workers/tests/unittests/test_spatial_sampling.py

@@ -6,7 +6,7 @@
 from shapely import wkt
 from shapely.geometry import Point
 
-from mapswipe_workers.utils.spatial_sampling  import distance_on_sphere, filter_points, calculate_spacing
+from mapswipe_workers.utils.spatial_sampling  import distance_on_sphere, filter_points, spatial_sampling
 
 
 class TestDistanceCalculations(unittest.TestCase):
@@ -22,7 +22,10 @@ def setUpClass(cls):
                 ),
                 "r",
         ) as file:
-            cls.fixture_df = pd.read_csv(file)
+            df = pd.read_csv(file)
+            df['geometry'] = df['geometry'].apply(wkt.loads)
+
+            cls.fixture_df = df
 
     def test_distance_on_sphere(self):
         p1 = Point(-74.006, 40.7128)
@@ -35,7 +38,7 @@ def test_distance_on_sphere(self):
 
     def test_filter_points(self):
         data = {
-            "data": [
+            "geometry": [
                 "POINT (-74.006 40.7128)",
                 "POINT (-75.006 41.7128)",
                 "POINT (-76.006 42.7128)",
@@ -44,43 +47,47 @@ def test_filter_points(self):
         }
         df = pd.DataFrame(data)
 
+        df['geometry'] = df['geometry'].apply(wkt.loads)
+
+        df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None)
+        df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None)
         threshold_distance = 100
         filtered_df = filter_points(df, threshold_distance)
 
         self.assertIsInstance(filtered_df, pd.DataFrame)
         self.assertLessEqual(len(filtered_df), len(df))
 
 
-    def test_calculate_spacing(self):
+    def test_spatial_sampling_ordering(self):
         data = {
-            "data": [
+            "geometry": [
                 "POINT (-74.006 40.7128)",
                 "POINT (-75.006 41.7128)",
                 "POINT (-76.006 42.7128)",
                 "POINT (-77.006 43.7128)"
             ],
-            'timestamp': [1, 2, 3, 4]
+            'captured_at': [1, 2, 3, 4],
+            'sequence_id': ['1', '1', '1', '1']
         }
         df = pd.DataFrame(data)
-        gdf = pd.DataFrame(df)
+        df['geometry'] = df['geometry'].apply(wkt.loads)
 
-        interval_length = 100
-        filtered_gdf = calculate_spacing(gdf, interval_length)
+        interval_length = 0.1
+        filtered_gdf = spatial_sampling(df, interval_length)
 
-        self.assertTrue(filtered_gdf['timestamp'].is_monotonic_increasing)
+        self.assertTrue(filtered_gdf['captured_at'].is_monotonic_increasing)
 
 
-    def test_calculate_spacing_with_sequence(self):
-        threshold_distance = 5
-        filtered_df = filter_points(self.fixture_df, threshold_distance)
+    def test_spatial_sampling_with_sequence(self):
+        threshold_distance = 0.01
+        filtered_df = spatial_sampling(self.fixture_df, threshold_distance)
         self.assertIsInstance(filtered_df, pd.DataFrame)
         self.assertLess(len(filtered_df), len(self.fixture_df))
 
         filtered_df.reset_index(drop=True, inplace=True)
-
         for i in range(len(filtered_df) - 1):
-            geom1 = wkt.loads(filtered_df.loc[i, 'data'])
-            geom2 = wkt.loads(filtered_df.loc[i + 1, 'data'])
+            geom1 = filtered_df.loc[i, 'geometry']
+            geom2 = filtered_df.loc[i + 1, 'geometry']
 
             distance = geom1.distance(geom2)