Merge branch 'dev' into improve-ressource-usage

Author: Gigaszi

manager-dashboard/app/views/NewProject/index.tsx

@@ -108,7 +108,8 @@ const defaultProjectFormValue: PartialProjectFormType = {
     // maxTasksPerUser: -1,
     inputType: PROJECT_INPUT_TYPE_UPLOAD,
     filter: FILTER_BUILDINGS,
-    isPano: false,
+    randomizeOrder: false,
+    panoOnly: false,
 };
 
 interface Props {
@@ -320,6 +321,7 @@ function NewProject(props: Props) {
 
             valuesToCopy.startTimestamp = valuesToCopy.dateRange?.startDate ?? null;
             valuesToCopy.endTimestamp = valuesToCopy.dateRange?.endDate ?? null;
+            valuesToCopy.isPano = valuesToCopy.panoOnly ? true : null;
 
             const storage = getStorage();
             const timestamp = (new Date()).getTime();
@@ -756,12 +758,19 @@ function NewProject(props: Props) {
                                 disabled={submissionPending || projectTypeEmpty}
                             />
                             <Checkbox
-                                name={'isPano' as const}
-                                value={value?.isPano}
+                                name={'panoOnly' as const}
+                                value={value?.panoOnly}
                                 label="Only use 360 degree panorama images."
                                 onChange={setFieldValue}
                                 disabled={submissionPending || projectTypeEmpty}
                             />
+                            <Checkbox
+                                name={'randomizeOrder' as const}
+                                value={value?.randomizeOrder}
+                                label="Randomize the order of images in the project."
+                                onChange={setFieldValue}
+                                disabled={submissionPending || projectTypeEmpty}
+                            />
                         </div>
                     </InputSection>
                 )}

manager-dashboard/app/views/NewProject/utils.ts

@@ -83,7 +83,9 @@ export interface ProjectFormType {
     endTimestamp?: string | null;
     organizationId?: number;
     creatorId?: number;
-    isPano?: boolean;
+    randomizeOrder?: boolean;
+    panoOnly?: boolean;
+    isPano?: boolean | null;
     samplingThreshold?: number;
 }
 
@@ -305,9 +307,15 @@ export const projectFormSchema: ProjectFormSchema = {
                     greaterThanCondition(0),
                 ],
             },
+            panoOnly: {
+                required: false,
+            },
             isPano: {
                 required: false,
             },
+            randomizeOrder: {
+                required: false,
+            },
         };
 
         baseSchema = addCondition(

mapswipe_workers/mapswipe_workers/project_types/street/project.py

@@ -51,6 +51,7 @@ def __init__(self, project_draft):
             start_time=project_draft.get("startTimestamp", None),
             end_time=project_draft.get("endTimestamp", None),
             organization_id=project_draft.get("organizationId", None),
+            randomize_order=project_draft.get("randomizeOrder", None),
             sampling_threshold=project_draft.get("samplingThreshold", None),
         )
 

mapswipe_workers/mapswipe_workers/utils/process_mapillary.py

@@ -19,9 +19,9 @@ def create_tiles(polygon, level):
     if isinstance(polygon, Polygon):
         polygon = MultiPolygon([polygon])
 
-    tiles = []
+    tiles = set()
     for i, poly in enumerate(polygon.geoms):
-        tiles.extend(list(mercantile.tiles(*poly.bounds, level)))
+        tiles.update(list(mercantile.tiles(*poly.bounds, level)))
 
     bbox_list = [mercantile.bounds(tile.x, tile.y, tile.z) for tile in tiles]
     bbox_polygons = [box(*bbox) for bbox in bbox_list]
@@ -212,6 +212,7 @@ def get_image_metadata(
     organization_id: str = None,
     start_time: str = None,
     end_time: str = None,
+    randomize_order=False,
     sampling_threshold=None,
 ):
     kwargs = {
@@ -231,6 +232,9 @@ def get_image_metadata(
     if sampling_threshold is not None:
         downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold)
 
+    if randomize_order is True:
+        downloaded_metadata = downloaded_metadata.sample(frac=1).reset_index(drop=True)
+
     total_images = len(downloaded_metadata)
     if total_images > 100000:
         raise ValueError(

mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py

@@ -141,9 +141,11 @@ def spatial_sampling(df, interval_length):
     for sequence in sorted_df["sequence_id"].unique():
         sequence_df = sorted_df[sorted_df["sequence_id"] == sequence]
 
-        filtered_sorted_sub_df = filter_points(sequence_df, interval_length)
-        sampled_sequence_df = pd.concat(
-            [sampled_sequence_df, filtered_sorted_sub_df], axis=0
-        )
+        if interval_length:
+            sequence_df = filter_points(sequence_df, interval_length)
+        sampled_sequence_df = pd.concat([sampled_sequence_df, sequence_df], axis=0)
+
+    # reverse order such that sequence are in direction of travel
+    sampled_sequence_df = sampled_sequence_df.iloc[::-1]
 
     return sampled_sequence_df

mapswipe_workers/tests/unittests/test_process_mapillary.py

@@ -48,9 +48,6 @@ def setUpClass(cls):
     def setUp(self):
         self.level = 14
         self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
-        self.test_multipolygon = MultiPolygon(
-            [self.test_polygon, Polygon([(2, 2), (3, 2), (3, 3), (2, 3)])]
-        )
         self.empty_polygon = Polygon()
         self.empty_geometry = GeometryCollection()
         self.row = pd.Series({"x": 1, "y": 1, "z": self.level})
@@ -61,9 +58,19 @@ def test_create_tiles_with_valid_polygon(self):
         self.assertFalse(tiles.empty)
 
     def test_create_tiles_with_multipolygon(self):
-        tiles = create_tiles(self.test_multipolygon, self.level)
+        polygon = Polygon(
+            [
+                (0.00000000, 0.00000000),
+                (0.000000001, 0.00000000),
+                (0.00000000, 0.000000001),
+                (0.00000000, 0.000000001),
+            ]
+        )
+        multipolygon = MultiPolygon([polygon, polygon])
+        tiles = create_tiles(multipolygon, self.level)
         self.assertIsInstance(tiles, pd.DataFrame)
         self.assertFalse(tiles.empty)
+        self.assertEqual(len(tiles), 1)
 
     def test_create_tiles_with_empty_polygon(self):
         tiles = create_tiles(self.empty_polygon, self.level)
@@ -312,12 +319,13 @@ def test_get_image_metadata_empty_response(self, mock_coordinate_download):
         with self.assertRaises(ValueError):
             get_image_metadata(self.fixture_data)
 
+    @patch("mapswipe_workers.utils.process_mapillary.filter_results")
     @patch("mapswipe_workers.utils.process_mapillary.coordinate_download")
-    def test_get_image_metadata_size_restriction(self, mock_coordinate_download):
-        mock_coordinate_download.return_value = pd.DataFrame(
-            {"geometry": range(1, 100002)}
-        )
-
+    def test_get_image_metadata_size_restriction(
+        self, mock_coordinate_download, mock_filter_results
+    ):
+        mock_df = pd.DataFrame({"id": range(1, 100002), "geometry": range(1, 100002)})
+        mock_coordinate_download.return_value = mock_df
         with self.assertRaises(ValueError):
             get_image_metadata(self.fixture_data)
 

mapswipe_workers/tests/unittests/test_spatial_sampling.py

@@ -1,29 +1,32 @@
 import os
-
 import unittest
+
 import numpy as np
 import pandas as pd
 from shapely import wkt
 from shapely.geometry import Point
 
-from mapswipe_workers.utils.spatial_sampling  import distance_on_sphere, filter_points, spatial_sampling
+from mapswipe_workers.utils.spatial_sampling import (
+    distance_on_sphere,
+    filter_points,
+    spatial_sampling,
+)
 
 
 class TestDistanceCalculations(unittest.TestCase):
-
     @classmethod
     def setUpClass(cls):
         with open(
-                os.path.join(
-                    os.path.dirname(os.path.abspath(__file__)),
-                    "..",
-                    "fixtures",
-                    "mapillary_sequence.csv",
-                ),
-                "r",
+            os.path.join(
+                os.path.dirname(os.path.abspath(__file__)),
+                "..",
+                "fixtures",
+                "mapillary_sequence.csv",
+            ),
+            "r",
         ) as file:
             df = pd.read_csv(file)
-            df['geometry'] = df['geometry'].apply(wkt.loads)
+            df["geometry"] = df["geometry"].apply(wkt.loads)
 
             cls.fixture_df = df
 
@@ -42,41 +45,43 @@ def test_filter_points(self):
                 "POINT (-74.006 40.7128)",
                 "POINT (-75.006 41.7128)",
                 "POINT (-76.006 42.7128)",
-                "POINT (-77.006 43.7128)"
+                "POINT (-77.006 43.7128)",
             ]
         }
         df = pd.DataFrame(data)
 
-        df['geometry'] = df['geometry'].apply(wkt.loads)
+        df["geometry"] = df["geometry"].apply(wkt.loads)
 
-        df['long'] = df['geometry'].apply(lambda geom: geom.x if geom.geom_type == 'Point' else None)
-        df['lat'] = df['geometry'].apply(lambda geom: geom.y if geom.geom_type == 'Point' else None)
+        df["long"] = df["geometry"].apply(
+            lambda geom: geom.x if geom.geom_type == "Point" else None
+        )
+        df["lat"] = df["geometry"].apply(
+            lambda geom: geom.y if geom.geom_type == "Point" else None
+        )
         threshold_distance = 100
         filtered_df = filter_points(df, threshold_distance)
 
         self.assertIsInstance(filtered_df, pd.DataFrame)
         self.assertLessEqual(len(filtered_df), len(df))
 
-
     def test_spatial_sampling_ordering(self):
         data = {
             "geometry": [
                 "POINT (-74.006 40.7128)",
                 "POINT (-75.006 41.7128)",
                 "POINT (-76.006 42.7128)",
-                "POINT (-77.006 43.7128)"
+                "POINT (-77.006 43.7128)",
             ],
-            'captured_at': [1, 2, 3, 4],
-            'sequence_id': ['1', '1', '1', '1']
+            "captured_at": [1, 2, 3, 4],
+            "sequence_id": ["1", "1", "1", "1"],
         }
         df = pd.DataFrame(data)
-        df['geometry'] = df['geometry'].apply(wkt.loads)
+        df["geometry"] = df["geometry"].apply(wkt.loads)
 
         interval_length = 0.1
         filtered_gdf = spatial_sampling(df, interval_length)
 
-        self.assertTrue(filtered_gdf['captured_at'].is_monotonic_increasing)
-
+        self.assertTrue(filtered_gdf["captured_at"].is_monotonic_decreasing)
 
     def test_spatial_sampling_with_sequence(self):
         threshold_distance = 0.01
@@ -86,15 +91,13 @@ def test_spatial_sampling_with_sequence(self):
 
         filtered_df.reset_index(drop=True, inplace=True)
         for i in range(len(filtered_df) - 1):
-            geom1 = filtered_df.loc[i, 'geometry']
-            geom2 = filtered_df.loc[i + 1, 'geometry']
+            geom1 = filtered_df.loc[i, "geometry"]
+            geom2 = filtered_df.loc[i + 1, "geometry"]
 
             distance = geom1.distance(geom2)
 
             self.assertLess(distance, threshold_distance)
 
 
-
-
 if __name__ == "__main__":
     unittest.main()