fix(exports): fix exports for find project

tnagorra · tnagorra · commit c69e8f44d3a6 · 2025-10-10T16:40:47.000+05:45
- use label instead of value for client type in results export
- use project firebase_id instead of id in history export
- read tile_x, tile_y, tile_z instead of task_x, task_y, task_z in hot tm exports
- skip flattening hot tm exports
- add url in tile map service project tasks
- fix calculation of required_count, progress
diff --git a/apps/mapping/models.py b/apps/mapping/models.py
@@ -26,6 +26,17 @@ def get_client_type(cls, value: str) -> "MappingSessionClientTypeEnum":
             "web": cls.WEB,
         }.get(value, cls.UNKNOWN)
 
+    @classmethod
+    def get_client_type_label_sql(cls, field: str) -> str:
+        return f"""
+            CASE {field}
+                WHEN {cls.MOBILE_ANDROID.value} THEN 'android'
+                WHEN {cls.MOBILE_IOS.value} THEN 'ios'
+                WHEN {cls.WEB.value} THEN 'web'
+            ELSE 'unknown'
+            END
+        """
+
 
 class MappingSession(models.Model):
     """Model representing a mapping session where a contributor user worked on a specific project task group."""
diff --git a/apps/project/exports/mapping_results.py b/apps/project/exports/mapping_results.py
@@ -8,6 +8,7 @@
 from apps.contributor.models import ContributorUser
 from apps.mapping.models import (
     MappingSession,
+    MappingSessionClientTypeEnum,
     MappingSessionResult,
 )
 from apps.project.models import Project, ProjectTask, ProjectTaskGroup
@@ -19,7 +20,6 @@
 
 
 def generate_mapping_results(*, destination_filename: Path, project: Project) -> pd.DataFrame:
-    # TODO: client_type IS ENUM -- CONVERT TO VALUE?
     sql_query = sql.SQL(f"""
         COPY (
             SELECT
@@ -38,7 +38,9 @@ def generate_mapping_results(*, destination_filename: Path, project: Project) ->
                 MS.{fd_name(MappingSession.start_time)} as start_time,
                 MS.{fd_name(MappingSession.end_time)} as end_time,
                 MS.{fd_name(MappingSession.app_version)} as app_version,
-                MS.{fd_name(MappingSession.client_type)} as client_type,
+                (
+                    {MappingSessionClientTypeEnum.get_client_type_label_sql(f"MS.{fd_name(MappingSession.client_type)}")}
+                ) as client_type,
                 MSR.{fd_name(MappingSessionResult.result)} as result,
                 -- the username for users which login to MapSwipe with their
                 -- OSM account is not defined or ''.
diff --git a/apps/project/exports/project_stats_by_date.py b/apps/project/exports/project_stats_by_date.py
@@ -177,6 +177,6 @@ def get_project_history(
 
     # merge contributors and progress
     project_history_df = progress_by_date_df.merge(contributors_by_date_df, left_on="day", right_on="day")
-    project_history_df["project_id"] = project.id
+    project_history_df["project_id"] = project.firebase_id
     project_history_df.to_csv(destination_filename)
     return project_history_df
diff --git a/apps/project/exports/project_tasks.py b/apps/project/exports/project_tasks.py
@@ -44,7 +44,7 @@ def generate_project_tasks(
                 PTG.{fd_name(ProjectTaskGroup.firebase_id)} as group_id,
                 PT.{fd_name(ProjectTask.firebase_id)} as task_id,
                 -- Metadata
-                ST_AsText({fd_name(ProjectTask.geometry)}) AS geom,
+                ST_AsText(ST_Multi({fd_name(ProjectTask.geometry)})) AS geom,
                 '{project.project_type_specifics.get("zoom_level")}' as tile_z,
                 -- NOTE: Existing tile_x and tile_y are passed from project_type_specifics now
                 -- NOTE: this is destructured by normalize_project_type_specifics(write_sql_to_gzipped_csv)
diff --git a/apps/project/exports/tasking_manager_geometries.py b/apps/project/exports/tasking_manager_geometries.py
@@ -59,9 +59,9 @@ def _get_row_value[T: int | float](
 
             task_id = row[1]
 
-            task_x = _get_row_value(column_index_map, row, "task_x")
-            task_y = _get_row_value(column_index_map, row, "task_y")
-            task_z = _get_row_value(column_index_map, row, "task_z")
+            task_x = _get_row_value(column_index_map, row, "tile_x")
+            task_y = _get_row_value(column_index_map, row, "tile_y")
+            task_z = _get_row_value(column_index_map, row, "tile_z")
 
             # TODO: Add no_count here and use later
             project_data.append(
@@ -108,6 +108,7 @@ def _get_row_value[T: int | float](
                         task_x,
                         task_y,
                         task_z,
+                        skip_flatten=True,
                     ),
                 },
             )
diff --git a/apps/project/tests/e2e_create_project_tile_map_service_test.py b/apps/project/tests/e2e_create_project_tile_map_service_test.py
diff --git a/apps/project/tests/export_test.py b/apps/project/tests/export_test.py
@@ -95,6 +95,7 @@ def setUpClass(cls):
                 project_type_specifics=FindProjectTaskProperty(
                     tile_x=1,
                     tile_y=2,
+                    url="https://some-service.com/14/1/2/",
                 ).model_dump(),
             )
         ]
diff --git a/apps/project/tests/mutation_test.py b/apps/project/tests/mutation_test.py
@@ -1393,7 +1393,7 @@ class TaskGroupType(typing.TypedDict):
             {
                 "firebase_id": "g101",
                 "number_of_tasks": 18,
-                "required_count": 18 * 10,
+                "required_count": 10,
                 "total_area": 210.10735845202447,
                 "project_type_specifics": {
                     "x_max": 24152,
@@ -1405,7 +1405,7 @@ class TaskGroupType(typing.TypedDict):
             {
                 "firebase_id": "g102",
                 "number_of_tasks": 24,
-                "required_count": 24 * 10,
+                "required_count": 10,
                 "total_area": 280.2915392364502,
                 "project_type_specifics": {
                     "x_max": 24153,
@@ -1417,7 +1417,7 @@ class TaskGroupType(typing.TypedDict):
             {
                 "firebase_id": "g103",
                 "number_of_tasks": 24,
-                "required_count": 24 * 10,
+                "required_count": 10,
                 "total_area": 280.4398676951218,
                 "project_type_specifics": {
                     "x_max": 24153,
@@ -1429,7 +1429,7 @@ class TaskGroupType(typing.TypedDict):
             {
                 "firebase_id": "g104",
                 "number_of_tasks": 6,
-                "required_count": 6 * 10,
+                "required_count": 10,
                 "total_area": 70.14703242812156,
                 "project_type_specifics": {
                     "x_max": 24150,
@@ -1445,34 +1445,44 @@ class TaskGroupType(typing.TypedDict):
                 "project_type_specifics": {
                     "tile_x": 24147,
                     "tile_y": 13753,
+                    "url": "https://hi-there/24147/13753/15",
+                    "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24147/13753.jpg?connectId=dummy-maxar-standard",
                 },
             },
             {
                 "firebase_id": "15-24147-13754",
                 "project_type_specifics": {
                     "tile_x": 24147,
                     "tile_y": 13754,
+                    "url": "https://hi-there/24147/13754/15",
+                    "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24147/13754.jpg?connectId=dummy-maxar-standard",
                 },
             },
             {
                 "firebase_id": "15-24147-13755",
                 "project_type_specifics": {
                     "tile_x": 24147,
                     "tile_y": 13755,
+                    "url": "https://hi-there/24147/13755/15",
+                    "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24147/13755.jpg?connectId=dummy-maxar-standard",
                 },
             },
             {
                 "firebase_id": "15-24148-13753",
                 "project_type_specifics": {
                     "tile_x": 24148,
                     "tile_y": 13753,
+                    "url": "https://hi-there/24148/13753/15",
+                    "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24148/13753.jpg?connectId=dummy-maxar-standard",
                 },
             },
             {
                 "firebase_id": "15-24148-13754",
                 "project_type_specifics": {
                     "tile_x": 24148,
                     "tile_y": 13754,
+                    "url": "https://hi-there/24148/13754/15",
+                    "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24148/13754.jpg?connectId=dummy-maxar-standard",
                 },
             },
         ]
@@ -1482,7 +1492,7 @@ class TaskGroupType(typing.TypedDict):
         project_task_qs = ProjectTask.objects.filter(task_group__project=latest_project)
 
         assert {
-            "required_results": sum(task_group["required_count"] for task_group in expected_task_groups),
+            "required_results": (18 + 24 + 24 + 6) * 10,
             "tasks_groups_count": project_task_group_qs.count(),
             "tasks_groups": list(
                 project_task_group_qs.order_by("id").values(
diff --git a/assets b/assets
@@ -1 +1 @@
-Subproject commit c6bfc5c8bf01663f30a2e1d193440f8f175447c0
+Subproject commit 4334d89b716ac16fa633654f781e1a856bcee296
diff --git a/project_types/base/project.py b/project_types/base/project.py
@@ -142,13 +142,13 @@ def analyze_groups(self):
         )
         # NOTE: After number_of_tasks is calculated
         project_task_groups_qs.update(
-            required_count=models.F("number_of_tasks") * self.project.verification_number,
+            required_count=self.project.verification_number,
             time_spent_max_allowed=(models.F("number_of_tasks") * self.get_max_time_spend_percentile()),
         )
 
         self.project.required_results = (
             ProjectTaskGroup.objects.filter(project_id=self.project.pk).aggregate(
-                required_results=models.Sum("required_count"),
+                required_results=models.Sum("number_of_tasks") * self.project.verification_number,
             )
         )["required_results"] or 0
 
diff --git a/project_types/tile_map_service/base/project.py b/project_types/tile_map_service/base/project.py
@@ -51,6 +51,8 @@ class TileMapServiceProjectTaskGroupProperty(base_project.BaseProjectTaskGroupPr
 class TileMapServiceProjectTaskProperty(base_project.BaseProjectTaskProperty):
     tile_x: int
     tile_y: int
+    # NOTE: We added URL as it's used directly when creating exports
+    url: str
 
 
 class TileMapServiceBaseProject[
@@ -132,6 +134,17 @@ def get_feature(task: ProjectTask):
         self.project.project_type_specific_output_asset = asset
         self.project.save(update_fields=("project_type_specific_output_asset",))
 
+    def get_task_specifics_for_db(self, tile_x: int, tile_y: int) -> TileMapServiceProjectTaskProperty:
+        return self.project_task_property_class(
+            tile_x=tile_x,
+            tile_y=tile_y,
+            url=self.project_type_specifics.tile_server_property.generate_url(
+                tile_x,
+                tile_y,
+                self.project_type_specifics.zoom_level,
+            ),
+        )
+
     @typing.override
     def create_tasks(
         self,
@@ -155,10 +168,7 @@ def create_tasks(
                         task_group_id=group.pk,
                         geometry=geometry,
                         project_type_specifics=clean_up_none_keys(
-                            self.project_task_property_class(
-                                tile_x=tile_x,
-                                tile_y=tile_y,
-                            ).model_dump(),
+                            self.get_task_specifics_for_db(tile_x, tile_y).model_dump(),
                         ),
                     ),
                 )
diff --git a/project_types/tile_map_service/compare/project.py b/project_types/tile_map_service/compare/project.py
@@ -14,7 +14,9 @@ class CompareProjectProperty(base_project.TileMapServiceProjectProperty):
 class CompareProjectTaskGroupProperty(base_project.TileMapServiceProjectTaskGroupProperty): ...
 
 
-class CompareProjectTaskProperty(base_project.TileMapServiceProjectTaskProperty): ...
+class CompareProjectTaskProperty(base_project.TileMapServiceProjectTaskProperty):
+    # NOTE: We added URL as it's used directly when creating exports
+    url_b: str
 
 
 class CompareProject(
@@ -37,6 +39,23 @@ def __init__(self, project: Project):
     def get_max_time_spend_percentile(self) -> float:
         return 11.2
 
+    @typing.override
+    def get_task_specifics_for_db(self, tile_x: int, tile_y: int) -> CompareProjectTaskProperty:
+        return self.project_task_property_class(
+            tile_x=tile_x,
+            tile_y=tile_y,
+            url=self.project_type_specifics.tile_server_property.generate_url(
+                tile_x,
+                tile_y,
+                self.project_type_specifics.zoom_level,
+            ),
+            url_b=self.project_type_specifics.tile_server_b_property.generate_url(
+                tile_x,
+                tile_y,
+                self.project_type_specifics.zoom_level,
+            ),
+        )
+
     # FIREBASE
 
     @typing.override
@@ -46,24 +65,14 @@ def skip_tasks_on_firebase(self) -> bool:
     @typing.override
     def get_task_specifics_for_firebase(self, task: ProjectTask) -> firebase_models.FbMappingTaskCompareCreateOnlyInput:
         task_specifics = self.project_task_property_class.model_validate(task.project_type_specifics)
-        tsp = self.project_type_specifics.tile_server_property
-        tsp_b = self.project_type_specifics.tile_server_b_property
 
         return firebase_models.FbMappingTaskCompareCreateOnlyInput(
             groupId=str(task.task_group.firebase_id),
             taskId=task.firebase_id,
             taskX=task_specifics.tile_x,
             taskY=task_specifics.tile_y,
-            url=tsp.generate_url(
-                task_specifics.tile_x,
-                task_specifics.tile_y,
-                self.project_type_specifics.zoom_level,
-            ),
-            urlB=tsp_b.generate_url(
-                task_specifics.tile_x,
-                task_specifics.tile_y,
-                self.project_type_specifics.zoom_level,
-            ),
+            url=task_specifics.url,
+            urlB=task_specifics.url_b,
         )
 
     @typing.override
diff --git a/utils/geo/tile_functions.py b/utils/geo/tile_functions.py
@@ -91,7 +91,7 @@ def quad_key_to_bing_url(quad_key: str, api_key: str):
 
 
 # FIXME(tnagorra): Add typings for osgeo
-def geometry_from_tile_coords(tile_x: float, tile_y: float, zoom: int) -> str:
+def geometry_from_tile_coords(tile_x: float, tile_y: float, zoom: int, *, skip_flatten: bool = False) -> str:
     """Compute the polygon geometry of a tile map service tile."""
     # Calculate lat, lon of upper left corner of tile
     pixel_x = tile_x * 256
@@ -113,7 +113,7 @@ def geometry_from_tile_coords(tile_x: float, tile_y: float, zoom: int) -> str:
     poly = ogr.Geometry(ogr.wkbPolygon)
     poly.AddGeometry(ring)
 
-    if poly.GetCoordinateDimension() == 3:
+    if not skip_flatten and poly.GetCoordinateDimension() == 3:
         poly.FlattenTo2D()
 
     return poly.ExportToWkt()

Original file line number	Diff line number	Diff line change
`@@ -95,6 +95,7 @@ def setUpClass(cls):`
`95`	`95`	`project_type_specifics=FindProjectTaskProperty(`
`96`	`96`	`tile_x=1,`
`97`	`97`	`tile_y=2,`
	`98`	`+ url="https://some-service.com/14/1/2/",`
`98`	`99`	`).model_dump(),`
`99`	`100`	`)`
`100`	`101`	`]`
Original file line number	Diff line number	Diff line change
`@@ -142,13 +142,13 @@ def analyze_groups(self):`
`142`	`142`	`)`
`143`	`143`	`# NOTE: After number_of_tasks is calculated`
`144`	`144`	`project_task_groups_qs.update(`
`145`		`- required_count=models.F("number_of_tasks") * self.project.verification_number,`
	`145`	`+ required_count=self.project.verification_number,`
`146`	`146`	`time_spent_max_allowed=(models.F("number_of_tasks") * self.get_max_time_spend_percentile()),`
`147`	`147`	`)`
`148`	`148`
`149`	`149`	`self.project.required_results = (`
`150`	`150`	`ProjectTaskGroup.objects.filter(project_id=self.project.pk).aggregate(`
`151`		`- required_results=models.Sum("required_count"),`
	`151`	`+ required_results=models.Sum("number_of_tasks") * self.project.verification_number,`
`152`	`152`	`)`
`153`	`153`	`)["required_results"] or 0`
`154`	`154`