feat: use filtering in street project

Author: Gigaszi

mapswipe_workers/mapswipe_workers/project_types/street/project.py

@@ -46,7 +46,15 @@ def __init__(self, project_draft):
         self.tasks: Dict[str, List[StreetTask]] = {}
 
         self.geometry = project_draft["geometry"]
-        ImageMetadata = get_image_metadata(self.geometry)
+
+        # TODO: validate inputs
+        ImageMetadata = get_image_metadata(
+            self.geometry,
+            is_pano=project_draft.get("isPano", None),
+            start_time=project_draft.get("startTimestamp", None),
+            end_time=project_draft.get("endTimestamp", None),
+            organization_id=project_draft.get("organizationId", None),
+        )
 
 
         self.imageIds = ImageMetadata["ids"]

mapswipe_workers/mapswipe_workers/utils/process_mapillary.py

@@ -17,6 +17,7 @@
 from vt2geojson import tools as vt2geojson_tools
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from mapswipe_workers.definitions import MAPILLARY_API_LINK, MAPILLARY_API_KEY
+from mapswipe_workers.definitions import logger
 
 
 def create_tiles(polygon, level):
@@ -128,6 +129,12 @@ def coordinate_download(
         failed_tiles = pd.DataFrame(failed_tiles, columns=tiles.columns).reset_index(
             drop=True
         )
+        target_columns = [
+            "id", "geometry", "captured_at", "is_pano", "compass_angle", "sequence", "organization_id"
+        ]
+        for col in target_columns:
+            if col not in downloaded_metadata.columns:
+                downloaded_metadata[col] = None
 
         return downloaded_metadata, failed_tiles
 
@@ -157,11 +164,11 @@ def geojson_to_polygon(geojson_data):
 
 def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None):
     df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms")
-    start_time = pd.Timestamp(start_time)
-
+    start_time = pd.to_datetime(start_time).tz_localize(None)
     if end_time is None:
-        end_time = pd.Timestamp.now()
-
+        end_time = pd.Timestamp.now().tz_localize(None)
+    else:
+        end_time = pd.to_datetime(end_time).tz_localize(None)
     filtered_df = df[
         (df["captured_at"] >= start_time) & (df["captured_at"] <= end_time)
     ]
@@ -177,12 +184,30 @@ def filter_results(
 ):
     df = results_df.copy()
     if is_pano is not None:
+        if df["is_pano"].isna().all():
+            logger.exception(
+                "No Mapillary Feature in the AoI has a 'is_pano' value."
+            )
+            return None
         df = df[df["is_pano"] == is_pano]
+
     if organization_id is not None:
+        if df["organization_id"].isna().all():
+            logger.exception(
+                "No Mapillary Feature in the AoI has an 'organization_id' value."
+            )
+            return None
         df = df[df["organization_id"] == organization_id]
+
     if start_time is not None:
+        if df["captured_at"].isna().all():
+            logger.exception(
+                "No Mapillary Feature in the AoI has a 'captured_at' value."
+            )
+            return None
         df = filter_by_timerange(df, start_time, end_time)
 
+
     return df
 
 
@@ -202,7 +227,10 @@ def get_image_metadata(
     downloaded_metadata = filter_results(
         downloaded_metadata, is_pano, organization_id, start_time, end_time
     )
-    return {
-        "ids": downloaded_metadata["id"].tolist(),
-        "geometries": downloaded_metadata["geometry"].tolist(),
-    }
+    if downloaded_metadata.isna().all().all() == False:
+        return {
+            "ids": downloaded_metadata["id"].tolist(),
+            "geometries": downloaded_metadata["geometry"].tolist(),
+        }
+    else:
+        raise ValueError("No Mapillary Features in the AoI match the filter criteria.")

mapswipe_workers/tests/fixtures/projectDrafts/street.json

@@ -47,7 +47,5 @@
     "verificationNumber": 3,
     "groupSize": 25,
     "isPano": false,
-    "startTimestamp": "2019-07-01T00:00:00.000Z",
-    "endTimestamp": null,
-    "organisationId": "1"
+    "startTimestamp": "2019-07-01T00:00:00.000Z"
 }

mapswipe_workers/tests/unittests/test_process_mapillary.py

@@ -18,12 +18,10 @@
     geojson_to_polygon,
     filter_by_timerange,
     filter_results,
+    get_image_metadata,
 )
 
 
-# Assuming create_tiles, download_and_process_tile, and coordinate_download are imported
-
-
 class TestTileGroupingFunctions(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -156,13 +154,11 @@ def test_geojson_to_polygon_contribution_geojson(self):
     )
     @patch("mapswipe_workers.utils.process_mapillary.requests.get")
     def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson):
-        # Mock the response from requests.get
         mock_response = MagicMock()
         mock_response.status_code = 200
         mock_response.content = b"mock vector tile data"  # Example mock data
         mock_get.return_value = mock_response
 
-        # Mock the return value of vt_bytes_to_geojson
         mock_vt2geojson.return_value = {
             "features": [
                 {
@@ -176,7 +172,6 @@ def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson):
 
         result, failed = download_and_process_tile(row)
 
-        # Assertions
         self.assertIsNone(failed)
         self.assertIsInstance(result, pd.DataFrame)
         self.assertEqual(len(result), 1)
@@ -269,6 +264,67 @@ def test_filter_time_range(self):
         )
         self.assertEqual(len(filtered_df), 3)
 
+    def test_filter_no_rows_after_filter(self):
+        filtered_df = filter_results(self.fixture_df, is_pano="False")
+        self.assertTrue(filtered_df.empty)
+
+    def test_filter_missing_columns(self):
+        columns_to_check = ["is_pano", "organization_id", "captured_at"]  # Add your column names here
+        for column in columns_to_check:
+            df_copy = self.fixture_df.copy()
+            df_copy[column] = None
+            if column == "captured_at":
+                column = "start_time"
+
+            result = filter_results(df_copy, **{column: True})
+            self.assertIsNone(result)
+
+    @patch("mapswipe_workers.utils.process_mapillary.coordinate_download")
+    def test_get_image_metadata(self, mock_coordinate_download):
+        mock_coordinate_download.return_value = (
+            self.fixture_df,
+            None,
+        )
+        result = get_image_metadata(self.fixture_data)
+        self.assertIsInstance(result, dict)
+        self.assertIn("ids", result)
+        self.assertIn("geometries", result)
+
+
+    @patch("mapswipe_workers.utils.process_mapillary.coordinate_download")
+    def test_get_image_metadata_filtering(self, mock_coordinate_download):
+        mock_coordinate_download.return_value = (
+            self.fixture_df,
+            None,
+        )
+
+        params = {
+            "is_pano": True,
+            "start_time": "2016-01-20 00:00:00",
+            "end_time": "2022-01-21 23:59:59",
+        }
+
+        result = get_image_metadata(self.fixture_data, **params)
+        self.assertIsInstance(result, dict)
+        self.assertIn("ids", result)
+        self.assertIn("geometries", result)
+
+
+    @patch("mapswipe_workers.utils.process_mapillary.coordinate_download")
+    def test_get_image_metadata_no_rows(self, mock_coordinate_download):
+        mock_coordinate_download.return_value = (
+            self.fixture_df,
+            None,
+        )
+
+        params = {
+            "is_pano": True,
+            "start_time": "1916-01-20 00:00:00",
+            "end_time": "1922-01-21 23:59:59",
+        }
+        with self.assertRaises(ValueError):
+            get_image_metadata(self.fixture_data, **params)
+
 
 if __name__ == "__main__":
     unittest.main()

mapswipe_workers/tests/unittests/test_project_type_street.py

@@ -2,6 +2,7 @@
 import os
 import unittest
 from unittest.mock import patch
+import pandas as pd
 
 from mapswipe_workers.project_types import StreetProject
 from tests import fixtures
@@ -16,7 +17,21 @@ def setUp(self) -> None:
             )
         )
         project_draft["projectDraftId"] = "foo"
-        self.project = StreetProject(project_draft)
+
+        with patch(
+            "mapswipe_workers.utils.process_mapillary.coordinate_download"
+        ) as mock_get:
+            with open(
+                os.path.join(
+                    os.path.dirname(os.path.abspath(__file__)),
+                    "..",
+                    "fixtures",
+                    "mapillary_response.csv",
+                ),
+                "r",
+            ) as file:
+                mock_get.return_value = (pd.read_csv(file), None)
+                self.project = StreetProject(project_draft)
 
     def test_init(self):
         self.assertEqual(self.project.geometry["type"], "FeatureCollection")
@@ -30,7 +45,6 @@ def test_create_tasks(self):
         self.project.create_groups()
         self.project.create_tasks()
         self.assertEqual(self.project.tasks["g0"][0].taskId, imageId)
-        # self.assertEqual(self.project.groups["g0"].numberOfTasks, 1)
 
 
 if __name__ == "__main__":