feat(street): Tasks and groups for street project

- Add Tutorial setup for street project
- Add typing for the tutorial and project objects

Author: susilnem

apps/project/graphql/inputs/project_types/street.py

@@ -3,9 +3,5 @@
 from project_types.street import project as street_project
 
 
-@strawberry.experimental.pydantic.input(model=street_project.ValidateObjectSourceConfig, all_fields=True)
-class ValidateObjectSourceConfigInput: ...
-
-
-@strawberry.experimental.pydantic.input(model=street_project.ValidateProjectProperty, all_fields=True)
-class ValidateProjectPropertyInput: ...
+@strawberry.experimental.pydantic.input(model=street_project.StreetProjectProperty, all_fields=True)
+class StreetProjectPropertyInput: ...

apps/project/graphql/types/project_types/street.py

@@ -0,0 +1,11 @@
+import strawberry
+
+from project_types.street import project as street_project
+
+
+@strawberry.experimental.pydantic.type(model=street_project.StreetMappilaryImageFilters, all_fields=True)
+class StreetMappilaryImageFilters: ...
+
+
+@strawberry.experimental.pydantic.type(model=street_project.StreetProjectProperty, all_fields=True)
+class StreetProjectPropertyType: ...

apps/project/graphql/types/types.py

@@ -8,6 +8,7 @@
 from apps.project.models import Organization, Project, ProjectAsset, ProjectAssetInputTypeEnum
 from apps.tutorial.graphql.types.types import TutorialType
 from main.graphql.context import Info
+from project_types.street import project as street_project
 from project_types.tile_map_service.compare import project as compare_project
 from project_types.tile_map_service.completeness import project as completeness_project
 from project_types.tile_map_service.find import project as find_project
@@ -23,6 +24,7 @@
 from .project_types.compare import CompareProjectPropertyType
 from .project_types.completeness import CompletenessProjectPropertyType
 from .project_types.find import FindProjectPropertyType
+from .project_types.street import StreetProjectPropertyType
 from .project_types.validate import ValidateProjectPropertyType
 from .project_types.validate_image import ValidateImageProjectPropertyType
 
@@ -167,6 +169,7 @@ async def project_type_specifics(
         | ValidateProjectPropertyType
         | ValidateImageProjectPropertyType
         | CompletenessProjectPropertyType
+        | StreetProjectPropertyType
         | None
     ):
         data = project.project_type_specifics
@@ -188,4 +191,9 @@ async def project_type_specifics(
                 "CompletenessProjectPropertyType",
                 completeness_project.CompletenessProjectProperty.model_validate(data),
             )
+        if project.project_type_enum == Project.Type.STREET:
+            return typing.cast(
+                "StreetProjectPropertyType",
+                street_project.StreetProjectProperty.model_validate(data),
+            )
         typing.assert_never(project.project_type_enum)

apps/tutorial/graphql/types/project_types/street.py

@@ -0,0 +1,7 @@
+import strawberry
+
+from project_types.street import tutorial as street_tutorial
+
+
+@strawberry.experimental.pydantic.type(model=street_tutorial.StreetTutorialTaskProperty, all_fields=True)
+class StreetTutorialTaskPropertyType: ...

apps/tutorial/graphql/types/types.py

@@ -17,6 +17,7 @@
 
 import apps.project.graphql.types.asset_types  # noqa: F401  # isort: skip # type: ignore[reportUnusedImport]
 
+from project_types.street import tutorial as street_tutorial
 from project_types.tile_map_service.compare import tutorial as compare_tutorial
 from project_types.tile_map_service.completeness import tutorial as completeness_tutorial
 from project_types.tile_map_service.find import tutorial as find_tutorial
@@ -26,6 +27,7 @@
 from .project_types.compare import CompareTutorialTaskPropertyType
 from .project_types.completeness import CompletenessTutorialTaskPropertyType
 from .project_types.find import FindTutorialTaskPropertyType
+from .project_types.street import StreetTutorialTaskPropertyType
 from .project_types.validate import ValidateTutorialTaskPropertyType
 from .project_types.validate_image import ValidateImageTutorialTaskPropertyType
 
@@ -62,6 +64,7 @@ async def project_type_specifics(
         | ValidateTutorialTaskPropertyType
         | ValidateImageTutorialTaskPropertyType
         | CompletenessTutorialTaskPropertyType
+        | StreetTutorialTaskPropertyType
         | None
     ):
         data = task.project_type_specifics
@@ -91,6 +94,11 @@ async def project_type_specifics(
                 "CompletenessTutorialTaskPropertyType",
                 completeness_tutorial.CompletenessTutorialTaskProperty.model_validate(data),
             )
+        if project_type_enum == Project.Type.STREET:
+            return typing.cast(
+                "StreetTutorialTaskPropertyType",
+                street_tutorial.StreetTutorialTaskProperty.model_validate(data),
+            )
         typing.assert_never(project_type_enum)
 
 

project_types/store.py

@@ -1,6 +1,8 @@
 import typing
 
 from apps.project.models import ProjectTypeEnum
+from project_types.street.project import StreetProject, StreetProjectProperty
+from project_types.street.tutorial import StreetTutorial, StreetTutorialTaskProperty
 
 from .tile_map_service.compare.project import CompareProject, CompareProjectProperty
 from .tile_map_service.compare.tutorial import CompareTutorial, CompareTutorialTaskProperty
@@ -28,6 +30,8 @@ def get_tutorial_task_property(project_type: ProjectTypeEnum | None):
         return ("validate_image", ValidateImageTutorialTaskProperty)
     if project_type == ProjectTypeEnum.COMPLETENESS:
         return ("completeness", CompletenessTutorialTaskProperty)
+    if project_type == ProjectTypeEnum.STREET:
+        return ("street", StreetTutorialTaskProperty)
     typing.assert_never(project_type)
 
 
@@ -45,10 +49,14 @@ def get_project_property(project_type: ProjectTypeEnum | None):
         return ("validate_image", ValidateImageProjectProperty)
     if project_type == ProjectTypeEnum.COMPLETENESS:
         return ("completeness", CompletenessProjectProperty)
+    if project_type == ProjectTypeEnum.STREET:
+        return ("street", StreetProjectProperty)
     typing.assert_never(project_type)
 
 
-type ProjectTypeHandlers = type[CompareProject | ValidateProject | ValidateImageProject | FindProject | CompletenessProject]
+type ProjectTypeHandlers = type[
+    CompareProject | ValidateProject | ValidateImageProject | FindProject | CompletenessProject | StreetProject
+]
 
 
 @typing.overload
@@ -81,6 +89,12 @@ def get_project_type_handler(
 ) -> type[CompletenessProject]: ...
 
 
+@typing.overload
+def get_project_type_handler(
+    project_type: typing.Literal[ProjectTypeEnum.STREET],
+) -> type[StreetProject]: ...
+
+
 def get_project_type_handler(project_type: ProjectTypeEnum) -> ProjectTypeHandlers:
     match project_type:
         case ProjectTypeEnum.FIND:
@@ -93,10 +107,12 @@ def get_project_type_handler(project_type: ProjectTypeEnum) -> ProjectTypeHandle
             return CompletenessProject
         case ProjectTypeEnum.VALIDATE_IMAGE:
             return ValidateImageProject
+        case ProjectTypeEnum.STREET:
+            return StreetProject
 
 
 type TutorialTypeHandlers = type[
-    CompareTutorial | ValidateTutorial | FindTutorial | CompletenessTutorial | ValidateImageTutorial
+    CompareTutorial | ValidateTutorial | FindTutorial | CompletenessTutorial | ValidateImageTutorial | StreetTutorial
 ]
 
 
@@ -130,6 +146,13 @@ def get_tutorial_type_handler(
 ) -> type[ValidateImageTutorial]: ...
 
 
+# FIXME(susilnem): Handle street
+@typing.overload
+def get_tutorial_type_handler(
+    tutorial_type: typing.Literal[ProjectTypeEnum.STREET],
+) -> type[typing.Any]: ...
+
+
 def get_tutorial_type_handler(tutorial_type: ProjectTypeEnum) -> TutorialTypeHandlers:
     match tutorial_type:
         case ProjectTypeEnum.FIND:
@@ -141,4 +164,6 @@ def get_tutorial_type_handler(tutorial_type: ProjectTypeEnum) -> TutorialTypeHan
         case ProjectTypeEnum.COMPLETENESS:
             return CompletenessTutorial
         case ProjectTypeEnum.VALIDATE_IMAGE:
-            return ValidateImageTutorial
+            raise Exception("Validate Image tutorial is not yet supported")
+        case ProjectTypeEnum.STREET:
+            raise Exception("Street tutorial is not yet supported")

project_types/street/project.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import math
 import typing
 from typing import Any, TypedDict
 
@@ -27,45 +28,11 @@
 class ValidFeature(Feature[Polygon | MultiPolygon, dict[str, Any]]): ...
 
 
-class ValidateRawGroupItem(TypedDict):
+class StreetRawGroupItem(TypedDict):
     feature_ids: list[int]
     features: list[ValidFeature]
 
 
-def group_input_geometries(features: list[ValidFeature], group_size: int, tutorial: bool = False):
-    groups: dict[str, ValidateRawGroupItem] = {}
-
-    # we will simply start with min group id = 100
-    group_id = 100
-    group_id_string = f"g{group_id}"
-    for feature_count, feature in enumerate(features):
-        feature_id = feature_count + 1
-        if feature_id % (group_size + 1) == 0:
-            group_id += 1
-            group_id_string = f"g{group_id}"
-
-        try:
-            groups[group_id_string]
-        except KeyError:
-            new_feature_group: ValidateRawGroupItem = {"feature_ids": [], "features": []}
-            groups[group_id_string] = new_feature_group
-
-        # we use a new id here based on the count
-        # since we are not sure that GetFID returns unique values
-        if not tutorial:
-            groups[group_id_string]["feature_ids"].append(feature_id)
-        # In the tutorial the feature id is defined by the "screen" attribute.
-        # We do this so that we can sort by the feature id later and
-        # get the screens displayed in the right order on the app.
-        elif feature.properties is not None:
-            groups[group_id_string]["feature_ids"].append(
-                feature.properties["screen"],
-            )
-        groups[group_id_string]["features"].append(feature)
-
-    return groups
-
-
 class StreetMappilaryImageFilters(BaseModel):
     is_pano: bool | None = Field(
         default=None,
@@ -97,7 +64,7 @@ class StreetMappilaryImageFilters(BaseModel):
     )
 
 
-class StreetProperty(base_project.BaseProjectProperty):
+class StreetProjectProperty(base_project.BaseProjectProperty):
     aoi_geometry: custom_fields.PydanticId
     custom_options: list[CustomOption] | None = None
     mappilary_image_filters: StreetMappilaryImageFilters
@@ -114,14 +81,14 @@ class StreetTaskProperty(base_project.BaseProjectTaskProperty):
 
 class StreetProject(
     base_project.BaseProject[
-        StreetProperty,
+        StreetProjectProperty,
         StreetTaskGroupProperty,
         StreetTaskProperty,
-        list[ValidFeature],
-        ValidateRawGroupItem,
+        StreetRawGroupItem,
+        list[tuple[int, ValidFeature]],
     ],
 ):
-    project_property_class = StreetProperty
+    project_property_class = StreetProjectProperty
     project_task_group_property_class = StreetTaskGroupProperty
     project_task_property_class = StreetTaskProperty
 
@@ -131,7 +98,7 @@ def __init__(self, project: Project):
             assert project.project_type == ProjectTypeEnum.STREET, f"{type(self)} is defined for STREET"
 
     @typing.override
-    def validate(self) -> list[ValidFeature]:
+    def validate(self) -> StreetRawGroupItem:
         """Validate project before creating groups"""
         self.project.update_processing_status(Project.ProcessingStatus.VALIDATING_GEOMETRY, True)
 
@@ -147,7 +114,7 @@ def validate(self) -> list[ValidFeature]:
 
         mappilary_image_filters = self.project_type_specifics.mappilary_image_filters
 
-        image_ids, geometries = get_image_metadata(
+        image_metadata_data = get_image_metadata(
             aoi_geojson=aoi_geojson,
             is_pano=mappilary_image_filters.is_pano,
             creator_id=mappilary_image_filters.creator_id,
@@ -157,19 +124,24 @@ def validate(self) -> list[ValidFeature]:
             randomize_order=mappilary_image_filters.randomize_order,
             sampling_threshold=mappilary_image_filters.sampling_threshold,
         )
+        return StreetRawGroupItem(
+            feature_ids=image_metadata_data["ids"],
+            features=image_metadata_data["geometries"],
+        )
 
     @typing.override
-    def create_tasks(self, group: ProjectTaskGroup, raw_group: ValidateRawGroupItem) -> int:
+    def create_tasks(
+        self,
+        /,
+        group: ProjectTaskGroup,
+        raw_group: list[tuple[int, ValidFeature]],
+        previous_tasks_count: int,
+    ) -> int:
         """Create tasks for a group."""
         bulk_mgr = BulkCreateManager(chunk_size=1000)
+        tasks_count = previous_tasks_count
 
-        tasks_count = 0
-        features = raw_group["features"]
-        f_ids = raw_group["feature_ids"]
-
-        for i, f_id in enumerate(f_ids):
-            feature = features[i]
-
+        for f_id, feature in raw_group:
             if feature.geometry is not None:
                 geom = ogr.CreateGeometryFromJson(
                     feature.geometry.model_dump_json(),
@@ -198,13 +170,21 @@ def create_tasks(self, group: ProjectTaskGroup, raw_group: ValidateRawGroupItem)
         return tasks_count
 
     @typing.override
-    def create_groups(self, resp: list[ValidFeature]):
+    def create_groups(self, resp: StreetRawGroupItem):
         self.project.update_processing_status(Project.ProcessingStatus.GENERATING_GROUPS_AND_TASKS, True)
-        raw_groups = group_input_geometries(resp, self.project.group_size)
+        number_of_groups = math.ceil(len(resp["feature_ids"]) / self.project.group_size)
+
+        total_tasks_accumulated: int = 0
+
+        for group_id in range(number_of_groups):
+            start_index = group_id * self.project.group_size
+            end_index = start_index + self.project.group_size
 
-        for group_key, raw_group in raw_groups.items():
+            group_features = resp["features"][start_index:end_index]
+            group_feature_ids = resp["feature_ids"][start_index:end_index]
+            raw_group = list(zip(group_feature_ids, group_features, strict=True))
             new_group = ProjectTaskGroup.objects.create(
-                firebase_id=group_key,
+                firebase_id=f"g{group_id}",
                 project_id=self.project.pk,
                 number_of_tasks=0,
                 progress=0,
@@ -214,8 +194,9 @@ def create_groups(self, resp: list[ValidFeature]):
             )
 
             # Create new tasks for this group
-            total_tasks = self.create_tasks(new_group, raw_group)
+            total_tasks = self.create_tasks(new_group, raw_group, total_tasks_accumulated)
             logger.info("Created %s tasks for group: %s", total_tasks, new_group.pk)
+            total_tasks_accumulated += total_tasks
 
     @typing.override
     def post_create_groups(self):

project_types/street/tutorial.py

@@ -0,0 +1,21 @@
+from apps.tutorial.models import Tutorial
+from project_types.base import tutorial as base_tutorial
+from project_types.street.project import StreetProjectProperty
+
+
+class StreetTutorialTaskProperty(base_tutorial.BaseTutorialTaskProperty):
+    identifier: int
+    object_geometry: str
+
+
+class StreetTutorial(
+    base_tutorial.BaseTutorial[
+        StreetProjectProperty,
+        StreetTutorialTaskProperty,
+    ],
+):
+    project_property_class = StreetProjectProperty
+    tutorial_task_property_class = StreetTutorialTaskProperty
+
+    def __init__(self, tutorial: Tutorial):
+        super().__init__(tutorial)

project_types/tile_map_service/base/project.py

@@ -151,11 +151,16 @@ def get_feature(task: ProjectTask):
         # TODO(thenav56): Calculate: total_area, time_spent_max_allowed
 
     @typing.override
-    def create_tasks(self, group: ProjectTaskGroup, raw_group: tile_grouping.RawGroup) -> int:
+    def create_tasks(
+        self,
+        group: ProjectTaskGroup,
+        raw_group: tile_grouping.RawGroup,
+        previous_tasks_count: int = 0,
+    ) -> int:
         """Create tasks for a group."""
         bulk_mgr = BulkCreateManager(chunk_size=1000)
 
-        tasks_count = 0
+        tasks_count = previous_tasks_count
         for tile_x in range(raw_group["xMin"], raw_group["xMax"] + 1):
             for tile_y in range(raw_group["yMin"], raw_group["yMax"] + 1):
                 geometry = tile_functions.geometry_from_tile_coords(