feat(project): Update mapillary api calls

Author: susilnem

apps/project/graphql/inputs/project_types/street.py

@@ -3,8 +3,8 @@
 from project_types.street import project as street_project
 
 
-@strawberry.experimental.pydantic.input(model=street_project.StreetMappilaryImageFilters, all_fields=True)
-class StreetMappilaryImageFiltersInput: ...
+@strawberry.experimental.pydantic.input(model=street_project.StreetMapilaryImageFilters, all_fields=True)
+class StreetMapilaryImageFiltersInput: ...
 
 
 @strawberry.experimental.pydantic.input(model=street_project.StreetProjectProperty, all_fields=True)

apps/project/graphql/types/project_types/street.py

@@ -3,8 +3,8 @@
 from project_types.street import project as street_project
 
 
-@strawberry.experimental.pydantic.type(model=street_project.StreetMappilaryImageFilters, all_fields=True)
-class StreetMappilaryImageFilters: ...
+@strawberry.experimental.pydantic.type(model=street_project.StreetMapilaryImageFilters, all_fields=True)
+class StreetMapilaryImageFilters: ...
 
 
 @strawberry.experimental.pydantic.type(model=street_project.StreetProjectProperty, all_fields=True)

apps/project/tests/mutation_test.py

@@ -1375,8 +1375,15 @@ def test_project_street(self, mock_requests):
                         "iconColor": "#FF0000",
                         "title": "Street Project Title",
                         "value": 1,
+                        "subOptions": [
+                            {
+                                "clientId": str(ULID()),
+                                "value": 1,
+                                "description": "Street sub option description",
+                            },
+                        ],
                     },
-                    "mapillaryImageFilters": {
+                    "mapilaryImageFilters": {
                         "isPano": True,
                         "creatorId": None,
                         "organizationId": None,
@@ -1396,9 +1403,7 @@ def test_project_street(self, mock_requests):
         assert latest_project.created_by_id == self.user.pk
         assert latest_project.modified_by_id == self.user.pk
         assert latest_project.image_id == int(image_asset["id"])
-        assert latest_project.project_type_specifics == {
-            "aoi_geometry": aoi_geometry_asset["id"],
-        }
+        assert latest_project.project_type_specifics is not None
 
         street_project.StreetProjectProperty.model_validate(
             latest_project.project_type_specifics,
@@ -1419,5 +1424,3 @@ def test_project_street(self, mock_requests):
 
         mock_requests.assert_called_once()
         mock_requests.assert_has_calls([call(int(project_id))])
-
-        process_project_task(int(project_id))

main/config.py

@@ -39,7 +39,7 @@ class Config:
     OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/"
 
     # NOTE: We get mapillary data from mapillary
-    MAPILLARY_API_LINK = "https://graph.mapillary.com/"
+    MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/"
     MAPILLARY_API_KEY = typing.cast("str", settings.MAPILLARY_API_KEY)
 
     FIREBASE_HELPER = typing.cast("FirebaseHelper", settings.FIREBASE_HELPER)

main/settings.py

@@ -113,7 +113,6 @@ def urlparse(value) -> ParseResult:
     MAP_IMAGE_ESRI_BETA_API_KEY=str,
     OSMCHA_API_KEY=str,  # os.environ["OSMCHA_API_KEY"]
     # Mapillary
-    MAPILLARY_API_LINK=str,
     MAPILLARY_API_KEY=str,  # os.environ["MAPILLARY_API_KEY"]
     # MAP_IMAGE_DIGITAL_GLOBE_API_KEY=str,
     # Firebase
@@ -535,8 +534,6 @@ def urlparse(value) -> ParseResult:
 OSMCHA_API_KEY = env("OSMCHA_API_KEY")
 
 # Mapillary
-
-MAPILLARY_API_LINK = env("MAPILLARY_API_LINK")
 MAPILLARY_API_KEY = env("MAPILLARY_API_KEY")
 
 # Firebase

project_types/store.py

@@ -166,4 +166,4 @@ def get_tutorial_type_handler(tutorial_type: ProjectTypeEnum) -> TutorialTypeHan
         case ProjectTypeEnum.VALIDATE_IMAGE:
             raise Exception("Validate Image tutorial is not yet supported")
         case ProjectTypeEnum.STREET:
-            raise Exception("Street tutorial is not yet supported")
+            return StreetTutorial

project_types/street/api_calls.py

@@ -2,18 +2,18 @@
 from collections.abc import Hashable
 from concurrent.futures import ProcessPoolExecutor
 from functools import partial
-from typing import Any
+from typing import Any, TypedDict
 from warnings import deprecated
 
 import mercantile
 import pandas as pd
 import requests
-from django.contrib.gis.geos import Point
 from geojson_pydantic import FeatureCollection
-from geojson_pydantic.geometries import MultiPolygon as GeoJsonMultiPolygon
-from geojson_pydantic.geometries import Polygon as GeojsonPolygon
+from geojson_pydantic.geometries import MultiPolygon as PydanticMultiPolygon
+from geojson_pydantic.geometries import Polygon as PydanticPolygon
 from pydantic import ValidationError
-from shapely import MultiPolygon, Polygon, box, unary_union
+from shapely import MultiPolygon, Point, Polygon, box, unary_union
+from shapely.geometry import shape
 from shapely.geometry.base import BaseGeometry
 from vt2geojson import tools as vt2geojson_tools
 
@@ -24,6 +24,11 @@
 logger = logging.getLogger(__name__)
 
 
+class StreetRawGroupItem(TypedDict):
+    ids: list[int]
+    geometries: list[Point]
+
+
 class StreetException(Exception):
     pass
 
@@ -69,13 +74,13 @@ def geojson_to_polygon(geojson_data: dict[str, Any]):
     except ValidationError as e:
         raise ValueError("Invalid GeoJSON FeatureCollection") from e
 
-    polygon_types = (GeojsonPolygon, GeoJsonMultiPolygon)
-    filtered_features = [feature for feature in fc.features if isinstance(feature.geometry, polygon_types)]
+    polygon_types = (PydanticPolygon, PydanticMultiPolygon)
+    geometries = [shape(feature.geometry) for feature in fc.features if isinstance(feature.geometry, polygon_types)]
 
-    if not filtered_features:
+    if not geometries:
         raise ValueError("No valid Polygon or MultiPolygon found in the GeoJSON FeatureCollection")
 
-    return unary_union(filtered_features)
+    return unary_union(geometries)
 
 
 def coordinate_download(
@@ -261,7 +266,7 @@ def get_image_metadata(
     end_time: str | None = None,
     randomize_order: bool = False,
     sampling_threshold: int | None = None,
-) -> dict[str, list[Any]]:
+) -> StreetRawGroupItem:
     kwargs = {
         "is_pano": is_pano,
         "creator_id": creator_id,
@@ -296,7 +301,7 @@ def get_image_metadata(
             f"Too many Images with selected filter options for the AoI: {total_images}",
         )
 
-    return {
-        "ids": downloaded_metadata["id"].tolist(),
-        "geometries": downloaded_metadata["geometry"].tolist(),
-    }
+    return StreetRawGroupItem(
+        ids=downloaded_metadata["id"].tolist(),
+        geometries=downloaded_metadata["geometry"].tolist(),
+    )

project_types/street/project.py

@@ -2,37 +2,27 @@
 import logging
 import math
 import typing
-from typing import Any, TypedDict
 
 from django.contrib.gis.geos import GEOSGeometry
 from django.core.files.base import ContentFile
-from geojson_pydantic import Feature
-from geojson_pydantic.geometries import MultiPolygon, Polygon
-from osgeo import ogr
 from pydantic import BaseModel, Field
 from pyfirebase_mapswipe import models as firebase_models
+from shapely import to_wkt
+from shapely.geometry.point import Point
 from ulid import ULID
 
 from apps.common.models import AssetMimetypeEnum, AssetTypeEnum
 from apps.project.models import Project, ProjectAsset, ProjectTask, ProjectTaskGroup, ProjectTypeEnum
 from main.bulk_managers import BulkCreateManager
 from project_types.base import project as base_project
-from project_types.street.api_calls import get_image_metadata
+from project_types.street.api_calls import StreetRawGroupItem, get_image_metadata
 from utils.common import create_json_dump
 from utils.custom_options.models import CustomOption
 
 logger = logging.getLogger(__name__)
 
 
-class ValidFeature(Feature[Polygon | MultiPolygon, dict[str, Any]]): ...
-
-
-class StreetRawGroupItem(TypedDict):
-    feature_ids: list[int]
-    features: list[ValidFeature]
-
-
-class StreetMappilaryImageFilters(BaseModel):
+class StreetMapilaryImageFilters(BaseModel):
     is_pano: bool | None = Field(
         default=None,
         description="Filter for images that are panoramas.",
@@ -66,7 +56,7 @@ class StreetMappilaryImageFilters(BaseModel):
 class StreetProjectProperty(base_project.BaseProjectProperty):
     aoi_geometry: typing.Annotated[str, Field(strict=True, pattern=r"^\d+$")] | None = None
     custom_options: list[CustomOption] | None = None
-    mappilary_image_filters: StreetMappilaryImageFilters
+    mapilary_image_filters: StreetMapilaryImageFilters
 
 
 class StreetTaskGroupProperty(base_project.BaseProjectTaskGroupProperty): ...
@@ -84,7 +74,7 @@ class StreetProject(
         StreetTaskGroupProperty,
         StreetTaskProperty,
         StreetRawGroupItem,
-        list[tuple[int, ValidFeature]],
+        list[tuple[int, Point]],
     ],
 ):
     project_property_class = StreetProjectProperty
@@ -111,77 +101,61 @@ def validate(self) -> StreetRawGroupItem:
         with aoi_asset.file.open() as aoi_file:
             aoi_geojson = json.loads(aoi_file.read())
 
-        mappilary_image_filters = self.project_type_specifics.mappilary_image_filters
+        mapilary_image_filters = self.project_type_specifics.mapilary_image_filters
 
-        image_metadata_data = get_image_metadata(
+        return get_image_metadata(
             aoi_geojson=aoi_geojson,
-            is_pano=mappilary_image_filters.is_pano,
-            creator_id=mappilary_image_filters.creator_id,
-            organization_id=mappilary_image_filters.organization_id,
-            start_time=mappilary_image_filters.start_time,
-            end_time=mappilary_image_filters.end_time,
-            randomize_order=mappilary_image_filters.randomize_order,
-            sampling_threshold=mappilary_image_filters.sampling_threshold,
-        )
-        return StreetRawGroupItem(
-            feature_ids=image_metadata_data["ids"],
-            features=image_metadata_data["geometries"],
+            is_pano=mapilary_image_filters.is_pano,
+            creator_id=mapilary_image_filters.creator_id,
+            organization_id=mapilary_image_filters.organization_id,
+            start_time=mapilary_image_filters.start_time,
+            end_time=mapilary_image_filters.end_time,
+            randomize_order=mapilary_image_filters.randomize_order,
+            sampling_threshold=mapilary_image_filters.sampling_threshold,
         )
 
     @typing.override
     def create_tasks(
         self,
         /,
         group: ProjectTaskGroup,
-        raw_group: list[tuple[int, ValidFeature]],
-        previous_tasks_count: int,
+        raw_group: list[tuple[int, Point]],
     ) -> int:
         """Create tasks for a group."""
         bulk_mgr = BulkCreateManager(chunk_size=1000)
-        tasks_count = previous_tasks_count
+        tasks_count = 0
 
         for f_id, feature in raw_group:
-            if feature.geometry is not None:
-                geom = ogr.CreateGeometryFromJson(
-                    feature.geometry.model_dump_json(),
-                )
-
-                if geom.GetCoordinateDimension() == 3:
-                    geom.FlattenTo2D()
-
-                geometry_str = geom.ExportToWkt()
-
-                bulk_mgr.add(
-                    ProjectTask(
-                        firebase_id=f"t{tasks_count + 1}",
-                        task_group_id=group.pk,
+            geometry_str = to_wkt(feature)
+
+            bulk_mgr.add(
+                ProjectTask(
+                    firebase_id=f"t{f_id}",
+                    task_group_id=group.pk,
+                    geometry=geometry_str,
+                    project_type_specifics=self.project_task_property_class(
+                        task_id=f"t{f_id}",
+                        group_id=f"g{group.pk}",
                         geometry=geometry_str,
-                        project_type_specifics=self.project_task_property_class(
-                            task_id=f"t{f_id}",
-                            group_id=f"g{group.pk}",
-                            geometry=geometry_str,
-                        ).model_dump(),
-                    ),
-                )
-                tasks_count += 1
+                    ).model_dump(),
+                ),
+            )
+            tasks_count += 1
 
         bulk_mgr.done()
         return tasks_count
 
     @typing.override
     def create_groups(self, resp: StreetRawGroupItem):
         self.project.update_processing_status(Project.ProcessingStatus.GENERATING_GROUPS_AND_TASKS, True)
-        number_of_groups = math.ceil(len(resp["feature_ids"]) / self.project.group_size)
-
-        # FIXME (susilnem): We can directly use the firebase_id instead of total_tasks_accumulated
-        total_tasks_accumulated: int = 0
+        number_of_groups = math.ceil(len(resp["ids"]) / self.project.group_size)
 
         for group_id in range(number_of_groups):
             start_index = group_id * self.project.group_size
             end_index = start_index + self.project.group_size
 
-            group_features = resp["features"][start_index:end_index]
-            group_feature_ids = resp["feature_ids"][start_index:end_index]
+            group_features = resp["geometries"][start_index:end_index]
+            group_feature_ids = resp["ids"][start_index:end_index]
             raw_group = list(zip(group_feature_ids, group_features, strict=True))
             new_group = ProjectTaskGroup.objects.create(
                 firebase_id=f"g{group_id}",
@@ -194,9 +168,8 @@ def create_groups(self, resp: StreetRawGroupItem):
             )
 
             # Create new tasks for this group
-            total_tasks = self.create_tasks(new_group, raw_group, total_tasks_accumulated)
+            total_tasks = self.create_tasks(new_group, raw_group)
             logger.info("Created %s tasks for group: %s", total_tasks, new_group.pk)
-            total_tasks_accumulated += total_tasks
 
     @typing.override
     def post_create_groups(self):
@@ -237,7 +210,7 @@ def get_feature(task: ProjectTask):
             file_size=file.size,
             type=AssetTypeEnum.OUTPUT,
             mimetype=AssetMimetypeEnum.GEOJSON,
-            # FIXME(tnagorra): Maybe create a internal user like mapswipe-bot
+            # FIXME(susilnem): Maybe create a internal user like mapswipe-bot
             created_by=self.project.modified_by,
             modified_by=self.project.modified_by,
         )

project_types/validate/project.py

@@ -307,7 +307,10 @@ def create_groups(self, resp: list[ValidFeature]):
             )
 
             # Create new tasks for this group
-            total_tasks = self.create_tasks(new_group, raw_group)
+            total_tasks = self.create_tasks(
+                group=new_group,
+                raw_group=raw_group,
+            )
             logger.info("Created %s tasks for group: %s", total_tasks, new_group.pk)
 
     @typing.override

schema.graphql

@@ -1893,7 +1893,7 @@ input StrFilterLookup {
   iRegex: String
 }
 
-type StreetMappilaryImageFilters {
+type StreetMapilaryImageFilters {
   """Filter for images that are panoramas."""
   isPano: Boolean
 
@@ -1916,7 +1916,7 @@ type StreetMappilaryImageFilters {
   samplingThreshold: Int
 }
 
-input StreetMappilaryImageFiltersInput {
+input StreetMapilaryImageFiltersInput {
   """Filter for images that are panoramas."""
   isPano: Boolean = null
 
@@ -1942,13 +1942,13 @@ input StreetMappilaryImageFiltersInput {
 input StreetProjectPropertyInput {
   aoiGeometry: String = null
   customOptions: [CustomOptionInput!] = null
-  mappilaryImageFilters: StreetMappilaryImageFiltersInput!
+  mapilaryImageFilters: StreetMapilaryImageFiltersInput!
 }
 
 type StreetProjectPropertyType {
   aoiGeometry: String
   customOptions: [ProjectCustomOption!]
-  mappilaryImageFilters: StreetMappilaryImageFilters!
+  mapilaryImageFilters: StreetMapilaryImageFilters!
 }
 
 type StreetTutorialTaskPropertyType {