From f465dfcdbb0004258ff0d6c648eaa7aea7313534 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 8 Oct 2025 13:25:18 +0545 Subject: [PATCH 1/5] feat(test): Add e2e testing for export file --- apps/common/utils.py | 70 +++++++ ...2e_create_project_tile_map_service_test.py | 184 +++++++++++++++++- assets | 2 +- 3 files changed, 253 insertions(+), 3 deletions(-) diff --git a/apps/common/utils.py b/apps/common/utils.py index fd631470..b4d07db0 100644 --- a/apps/common/utils.py +++ b/apps/common/utils.py @@ -1,7 +1,10 @@ import base64 +import csv import gzip +import io import json import typing +from pathlib import Path from django.core.files.storage import FileSystemStorage from django.db.models.fields import files @@ -9,6 +12,9 @@ from main.config import Config from utils.common import is_file_empty +if typing.TYPE_CHECKING: + from apps.project.models import ProjectAsset + @typing.overload def get_absolute_uri(file: None) -> None: ... @@ -47,3 +53,67 @@ def decode_tasks(encoded_task: str) -> list[dict[str, typing.Any]]: compressed_bytes = base64.b64decode(encoded_task) json_bytes = gzip.decompress(compressed_bytes) return json.loads(json_bytes.decode("utf-8")) + + +def compare_csv_files( + project_asset: "ProjectAsset", + expected_csv_path: Path, + message: str, + keys_to_ignore: list[str] | set[str] | None = None, + is_gzip_file: bool = False, +) -> None: + """Compare a CSV from a ProjectAsset with a plain CSV file. + Supports gzipped CSV files when is_gzip_file=True. + Raises AssertionError if differences are found. + """ + if is_gzip_file: + with ( + project_asset.file.open("rb") as file, + gzip.GzipFile(fileobj=file, mode="rb") as gz, + io.TextIOWrapper(gz, encoding="utf-8") as text_stream, + ): + actual_data = list(csv.DictReader(text_stream)) + else: + with project_asset.file.open(mode="r") as file: + actual_data = list(csv.DictReader(file)) + + with expected_csv_path.open(mode="r", newline="", encoding="utf-8") as file: + expected_data = list(csv.DictReader(file)) + + if keys_to_ignore: + actual_data = remove_object_keys(actual_data, keys_to_ignore) + expected_data = remove_object_keys(expected_data, keys_to_ignore) + + assert actual_data == expected_data, message + + +def compare_geojson_files( + project_asset: "ProjectAsset", + expected_geojson_path: Path, + message: str, + keys_to_ignore: list[str] | set[str] | None = None, + is_gzip_file: bool = False, +) -> None: + """Compare a GeoJSON from a ProjectAsset with a plain GeoJSON file. + Supports gzipped GeoJSON files when is_gzip_file=True. + Raises AssertionError if differences are found. + """ + if is_gzip_file: + with ( + project_asset.file.open("rb") as file, + gzip.GzipFile(fileobj=file, mode="rb") as gz, + io.TextIOWrapper(gz, encoding="utf-8") as text_stream, + ): + actual_data = json.load(text_stream) + else: + with project_asset.file.open("r", encoding="utf-8") as file: + actual_data = json.load(file) + + with expected_geojson_path.open("r", encoding="utf-8") as file: + expected_data = json.load(file) + + if keys_to_ignore: + actual_data = remove_object_keys(actual_data, keys_to_ignore) + expected_data = remove_object_keys(expected_data, keys_to_ignore) + + assert actual_data == expected_data, message diff --git a/apps/project/tests/e2e_create_project_tile_map_service_test.py b/apps/project/tests/e2e_create_project_tile_map_service_test.py index a6b6f35f..6678db8c 100644 --- a/apps/project/tests/e2e_create_project_tile_map_service_test.py +++ b/apps/project/tests/e2e_create_project_tile_map_service_test.py @@ -7,7 +7,8 @@ from django.db.models.signals import pre_save from ulid import ULID -from apps.common.utils import remove_object_keys +from apps.common.models import AssetTypeEnum +from apps.common.utils import compare_csv_files, compare_geojson_files, remove_object_keys from apps.contributor.factories import ContributorUserFactory from apps.contributor.models import ContributorUserGroup from apps.mapping.firebase.pull import pull_results_from_firebase @@ -18,7 +19,7 @@ MappingSessionUserGroup, MappingSessionUserGroupTemp, ) -from apps.project.models import Organization, Project +from apps.project.models import Organization, Project, ProjectAsset, ProjectAssetExportTypeEnum from apps.tutorial.models import Tutorial from apps.user.factories import UserFactory from main.config import Config @@ -627,3 +628,182 @@ def _test_project(self, projectKey: str, filename: str): assert isinstance(project_fb_data, dict), "Project in firebase should be a dictionary" assert project_fb_data["progress"] == project.progress, "Progress should be synced with firebase" assert project_fb_data["contributorCount"] == 1, "Contributor count should be synced with firebase" + + # NOTE: EXPORTS TESTING + aggregated_results_filename = ( + Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["aggregated_results"] + ) + + # NOTE: Test AGGREGATED RESULTS + aggregated_results_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS, + ).first() + + if not aggregated_results_project_asset: + raise AssertionError("Aggregated results project asset not found") + + compare_csv_files( + aggregated_results_project_asset, + aggregated_results_filename, + is_gzip_file=True, + message="Difference found for aggregated results export file.", + ) + + # NOTE: Test AGGREGATED RESULTS WITH GEOMETRY + aggregated_results_with_geometry_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS_WITH_GEOMETRY, + ).first() + + if not aggregated_results_with_geometry_project_asset: + raise AssertionError("Aggregated results with geometry project asset not found") + + compare_geojson_files( + aggregated_results_with_geometry_project_asset, + test_data["expected_project_exports_data"]["aggregated_results_with_geometry"], + is_gzip_file=True, + message="Difference found for aggregated results with geometry export file.", + ) + + # NOTE: Test RESULTS + results_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.RESULTS, + ).first() + + if not results_project_asset: + raise AssertionError("Results project asset not found") + + compare_csv_files( + results_project_asset, + test_data["expected_project_exports_data"]["results"], + is_gzip_file=True, + message="Difference found for results export file.", + ) + # NOTE: Test HISTORY + history_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.HISTORY, + ).first() + + if not history_project_asset: + raise AssertionError("History project asset not found") + + compare_csv_files( + history_project_asset, + test_data["expected_project_exports_data"]["history"], + is_gzip_file=True, + message="Difference found for history export file.", + ) + + # NOTE: Test GROUPS + groups_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.GROUPS, + ).first() + + if not groups_project_asset: + raise AssertionError("Groups project asset not found") + + compare_csv_files( + groups_project_asset, + test_data["expected_project_exports_data"]["groups"], + is_gzip_file=True, + message="Difference found for groups export file.", + ) + + # NOTE: Test TASKS + tasks_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.TASKS, + ).first() + + if not tasks_project_asset: + raise AssertionError("Tasks project asset not found") + + compare_csv_files( + tasks_project_asset, + test_data["expected_project_exports_data"]["tasks"], + is_gzip_file=True, + message="Difference found for tasks export file.", + ) + + # NOTE: Test USERS + users_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.USERS, + ).first() + + if not users_project_asset: + raise AssertionError("Users project asset not found") + + compare_csv_files( + users_project_asset, + test_data["expected_project_exports_data"]["users"], + is_gzip_file=True, + message="Difference found for users export file.", + ) + + # NOTE: Test AREA OF INTEREST FILE + aoi_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.AREA_OF_INTEREST, + ).first() + + if not aoi_project_asset: + raise AssertionError("AOI Geometry project asset not found") + + compare_geojson_files( + aoi_project_asset, + aoi_geometry_filename, + message="Difference found for AOI Geometry export file.", + ) + + # NOTE: TEST HOT TASKING MANAGER GEOMETRY + if "hot_tasking_manager_aoi" in test_data["assets"]: + htm_aoi_filename = Path(Config.BASE_DIR) / test_data["assets"]["hot_tasking_manager_aoi"] + htm_aoi_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.HOT_TASKING_MANAGER_GEOMETRIES, + ).first() + + if not htm_aoi_project_asset: + raise AssertionError("HTM AOI Geometry project asset not found") + + compare_geojson_files( + htm_aoi_project_asset, + htm_aoi_filename, + message="Difference found for HTM AOI Geometry export file.", + ) + + # NOTE: TEST MODERATE TO HIGH AGREEMENT + if "moderate_to_high_agreement" in test_data["expected_project_exports_data"]: + moderate_to_high_agreement_filename = Path( + Config.BASE_DIR, + test_data["expected_project_exports_data"]["moderate_to_high_agreement"], + ) + + moderate_to_high_agreement_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.MODERATE_TO_HIGH_AGREEMENT_YES_MAYBE_GEOMETRIES, + ).first() + + if not moderate_to_high_agreement_project_asset: + raise AssertionError("Moderate to high agreement project asset not found") + + compare_geojson_files( + moderate_to_high_agreement_project_asset, + moderate_to_high_agreement_filename, + message="Difference found for moderate to high agreement export file.", + ) diff --git a/assets b/assets index 69dae411..5e00331c 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 69dae411dde7df1b20df2d1ec0e35fc8ccee877b +Subproject commit 5e00331c0db0516298067505a9055081a18da7cb From 31a8ecd01e3f3a487f8e71e78397883a06303711 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Thu, 9 Oct 2025 12:07:37 +0545 Subject: [PATCH 2/5] feat(test): Add changes on e2e testing for find project --- apps/common/utils.py | 18 ++--- ...2e_create_project_tile_map_service_test.py | 75 ++++++++++++++----- assets | 2 +- 3 files changed, 68 insertions(+), 27 deletions(-) diff --git a/apps/common/utils.py b/apps/common/utils.py index b4d07db0..7fc58883 100644 --- a/apps/common/utils.py +++ b/apps/common/utils.py @@ -72,19 +72,19 @@ def compare_csv_files( gzip.GzipFile(fileobj=file, mode="rb") as gz, io.TextIOWrapper(gz, encoding="utf-8") as text_stream, ): - actual_data = list(csv.DictReader(text_stream)) + generated_data = list(csv.DictReader(text_stream)) else: with project_asset.file.open(mode="r") as file: - actual_data = list(csv.DictReader(file)) + generated_data = list(csv.DictReader(file)) with expected_csv_path.open(mode="r", newline="", encoding="utf-8") as file: expected_data = list(csv.DictReader(file)) if keys_to_ignore: - actual_data = remove_object_keys(actual_data, keys_to_ignore) + generated_data = remove_object_keys(generated_data, keys_to_ignore) expected_data = remove_object_keys(expected_data, keys_to_ignore) - assert actual_data == expected_data, message + assert generated_data == expected_data, message def compare_geojson_files( @@ -104,16 +104,16 @@ def compare_geojson_files( gzip.GzipFile(fileobj=file, mode="rb") as gz, io.TextIOWrapper(gz, encoding="utf-8") as text_stream, ): - actual_data = json.load(text_stream) + generated_data = json.load(text_stream) else: - with project_asset.file.open("r", encoding="utf-8") as file: - actual_data = json.load(file) + with project_asset.file.open("r") as file: + generated_data = json.load(file) with expected_geojson_path.open("r", encoding="utf-8") as file: expected_data = json.load(file) if keys_to_ignore: - actual_data = remove_object_keys(actual_data, keys_to_ignore) + generated_data = remove_object_keys(generated_data, keys_to_ignore) expected_data = remove_object_keys(expected_data, keys_to_ignore) - assert actual_data == expected_data, message + assert generated_data == expected_data, message diff --git a/apps/project/tests/e2e_create_project_tile_map_service_test.py b/apps/project/tests/e2e_create_project_tile_map_service_test.py index 6678db8c..ec179f62 100644 --- a/apps/project/tests/e2e_create_project_tile_map_service_test.py +++ b/apps/project/tests/e2e_create_project_tile_map_service_test.py @@ -630,9 +630,6 @@ def _test_project(self, projectKey: str, filename: str): assert project_fb_data["contributorCount"] == 1, "Contributor count should be synced with firebase" # NOTE: EXPORTS TESTING - aggregated_results_filename = ( - Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["aggregated_results"] - ) # NOTE: Test AGGREGATED RESULTS aggregated_results_project_asset = ProjectAsset.objects.filter( @@ -644,10 +641,18 @@ def _test_project(self, projectKey: str, filename: str): if not aggregated_results_project_asset: raise AssertionError("Aggregated results project asset not found") + aggregated_results_filename = ( + Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["aggregated_results"] + ) compare_csv_files( aggregated_results_project_asset, aggregated_results_filename, is_gzip_file=True, + keys_to_ignore={ + "project_internal_id", + "group_internal_id", + "task_internal_id", + }, message="Difference found for aggregated results export file.", ) @@ -661,10 +666,20 @@ def _test_project(self, projectKey: str, filename: str): if not aggregated_results_with_geometry_project_asset: raise AssertionError("Aggregated results with geometry project asset not found") + expected_aggregated_results_with_geometry_filename = Path( + Config.BASE_DIR, + test_data["expected_project_exports_data"]["aggregated_results_with_geometry"], + ) compare_geojson_files( aggregated_results_with_geometry_project_asset, - test_data["expected_project_exports_data"]["aggregated_results_with_geometry"], + expected_aggregated_results_with_geometry_filename, is_gzip_file=True, + keys_to_ignore={ + "name", + "project_internal_id", + "group_internal_id", + "task_internal_id", + }, message="Difference found for aggregated results with geometry export file.", ) @@ -678,10 +693,20 @@ def _test_project(self, projectKey: str, filename: str): if not results_project_asset: raise AssertionError("Results project asset not found") + results_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["results"] compare_csv_files( results_project_asset, - test_data["expected_project_exports_data"]["results"], + results_filename, is_gzip_file=True, + keys_to_ignore={ + "project_internal_id", + "group_internal_id", + "task_internal_id", + "user_internal_id", + "timestamp", + "start_time", + "end_time", + }, message="Difference found for results export file.", ) # NOTE: Test HISTORY @@ -691,13 +716,13 @@ def _test_project(self, projectKey: str, filename: str): export_type=ProjectAssetExportTypeEnum.HISTORY, ).first() + history_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["history"] if not history_project_asset: raise AssertionError("History project asset not found") compare_csv_files( history_project_asset, - test_data["expected_project_exports_data"]["history"], - is_gzip_file=True, + history_filename, message="Difference found for history export file.", ) @@ -710,11 +735,15 @@ def _test_project(self, projectKey: str, filename: str): if not groups_project_asset: raise AssertionError("Groups project asset not found") - + groups_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["groups"] compare_csv_files( groups_project_asset, - test_data["expected_project_exports_data"]["groups"], + groups_filename, is_gzip_file=True, + keys_to_ignore={ + "project_internal_id", + "group_internal_id", + }, message="Difference found for groups export file.", ) @@ -727,11 +756,16 @@ def _test_project(self, projectKey: str, filename: str): if not tasks_project_asset: raise AssertionError("Tasks project asset not found") - + tasks_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["tasks"] compare_csv_files( tasks_project_asset, - test_data["expected_project_exports_data"]["tasks"], + tasks_filename, is_gzip_file=True, + keys_to_ignore={ + "project_internal_id", + "group_internal_id", + "task_internal_id", + }, message="Difference found for tasks export file.", ) @@ -744,10 +778,10 @@ def _test_project(self, projectKey: str, filename: str): if not users_project_asset: raise AssertionError("Users project asset not found") - + users_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["users"] compare_csv_files( users_project_asset, - test_data["expected_project_exports_data"]["users"], + users_filename, is_gzip_file=True, message="Difference found for users export file.", ) @@ -761,16 +795,22 @@ def _test_project(self, projectKey: str, filename: str): if not aoi_project_asset: raise AssertionError("AOI Geometry project asset not found") - + aoi_geometry_export_filename = Path( + Config.BASE_DIR, + test_data["expected_project_exports_data"]["area_of_interest"], + ) compare_geojson_files( aoi_project_asset, - aoi_geometry_filename, + aoi_geometry_export_filename, message="Difference found for AOI Geometry export file.", ) # NOTE: TEST HOT TASKING MANAGER GEOMETRY - if "hot_tasking_manager_aoi" in test_data["assets"]: - htm_aoi_filename = Path(Config.BASE_DIR) / test_data["assets"]["hot_tasking_manager_aoi"] + # TODO: CHECK if this export is required for completeness project if required, remove this check + if "hot_tasking_manager_geometry" in test_data["expected_project_exports_data"]: + htm_aoi_filename = ( + Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["hot_tasking_manager_geometry"] + ) htm_aoi_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, @@ -787,6 +827,7 @@ def _test_project(self, projectKey: str, filename: str): ) # NOTE: TEST MODERATE TO HIGH AGREEMENT + # TODO: CHECK if this export is required for completeness project if required, remove this check if "moderate_to_high_agreement" in test_data["expected_project_exports_data"]: moderate_to_high_agreement_filename = Path( Config.BASE_DIR, diff --git a/assets b/assets index 5e00331c..19ac4bf7 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 5e00331c0db0516298067505a9055081a18da7cb +Subproject commit 19ac4bf72a35fc9c01b68e491582bf44ad27ab09 From 7586492701623cc922d244069609d546c51fd1c8 Mon Sep 17 00:00:00 2001 From: tnagorra Date: Fri, 10 Oct 2025 08:36:12 +0545 Subject: [PATCH 3/5] test(project): skip checking for export if no expected_project_exports_data - ignore problematic fields/keys to pass tests (temporary) - replace compare_csv_files to read_csv - replace compare_geojson_files to read_json --- apps/common/utils.py | 70 --- ...2e_create_project_tile_map_service_test.py | 457 ++++++++++++------ assets | 2 +- bulk_ignore_pyright_warnings.py | 4 +- 4 files changed, 308 insertions(+), 225 deletions(-) diff --git a/apps/common/utils.py b/apps/common/utils.py index 7fc58883..fd631470 100644 --- a/apps/common/utils.py +++ b/apps/common/utils.py @@ -1,10 +1,7 @@ import base64 -import csv import gzip -import io import json import typing -from pathlib import Path from django.core.files.storage import FileSystemStorage from django.db.models.fields import files @@ -12,9 +9,6 @@ from main.config import Config from utils.common import is_file_empty -if typing.TYPE_CHECKING: - from apps.project.models import ProjectAsset - @typing.overload def get_absolute_uri(file: None) -> None: ... @@ -53,67 +47,3 @@ def decode_tasks(encoded_task: str) -> list[dict[str, typing.Any]]: compressed_bytes = base64.b64decode(encoded_task) json_bytes = gzip.decompress(compressed_bytes) return json.loads(json_bytes.decode("utf-8")) - - -def compare_csv_files( - project_asset: "ProjectAsset", - expected_csv_path: Path, - message: str, - keys_to_ignore: list[str] | set[str] | None = None, - is_gzip_file: bool = False, -) -> None: - """Compare a CSV from a ProjectAsset with a plain CSV file. - Supports gzipped CSV files when is_gzip_file=True. - Raises AssertionError if differences are found. - """ - if is_gzip_file: - with ( - project_asset.file.open("rb") as file, - gzip.GzipFile(fileobj=file, mode="rb") as gz, - io.TextIOWrapper(gz, encoding="utf-8") as text_stream, - ): - generated_data = list(csv.DictReader(text_stream)) - else: - with project_asset.file.open(mode="r") as file: - generated_data = list(csv.DictReader(file)) - - with expected_csv_path.open(mode="r", newline="", encoding="utf-8") as file: - expected_data = list(csv.DictReader(file)) - - if keys_to_ignore: - generated_data = remove_object_keys(generated_data, keys_to_ignore) - expected_data = remove_object_keys(expected_data, keys_to_ignore) - - assert generated_data == expected_data, message - - -def compare_geojson_files( - project_asset: "ProjectAsset", - expected_geojson_path: Path, - message: str, - keys_to_ignore: list[str] | set[str] | None = None, - is_gzip_file: bool = False, -) -> None: - """Compare a GeoJSON from a ProjectAsset with a plain GeoJSON file. - Supports gzipped GeoJSON files when is_gzip_file=True. - Raises AssertionError if differences are found. - """ - if is_gzip_file: - with ( - project_asset.file.open("rb") as file, - gzip.GzipFile(fileobj=file, mode="rb") as gz, - io.TextIOWrapper(gz, encoding="utf-8") as text_stream, - ): - generated_data = json.load(text_stream) - else: - with project_asset.file.open("r") as file: - generated_data = json.load(file) - - with expected_geojson_path.open("r", encoding="utf-8") as file: - expected_data = json.load(file) - - if keys_to_ignore: - generated_data = remove_object_keys(generated_data, keys_to_ignore) - expected_data = remove_object_keys(expected_data, keys_to_ignore) - - assert generated_data == expected_data, message diff --git a/apps/project/tests/e2e_create_project_tile_map_service_test.py b/apps/project/tests/e2e_create_project_tile_map_service_test.py index ec179f62..83ba573f 100644 --- a/apps/project/tests/e2e_create_project_tile_map_service_test.py +++ b/apps/project/tests/e2e_create_project_tile_map_service_test.py @@ -1,14 +1,20 @@ +import csv +import gzip +import io +import json +import operator import typing from contextlib import contextmanager from datetime import datetime from pathlib import Path import json5 +from django.core.files.base import File from django.db.models.signals import pre_save from ulid import ULID from apps.common.models import AssetTypeEnum -from apps.common.utils import compare_csv_files, compare_geojson_files, remove_object_keys +from apps.common.utils import remove_object_keys from apps.contributor.factories import ContributorUserFactory from apps.contributor.models import ContributorUserGroup from apps.mapping.firebase.pull import pull_results_from_firebase @@ -19,13 +25,69 @@ MappingSessionUserGroup, MappingSessionUserGroupTemp, ) -from apps.project.models import Organization, Project, ProjectAsset, ProjectAssetExportTypeEnum +from apps.project.models import Organization, Project, ProjectAsset, ProjectAssetExportTypeEnum, ProjectAssetInputTypeEnum from apps.tutorial.models import Tutorial from apps.user.factories import UserFactory from main.config import Config from main.tests import TestCase +def read_json( + file_path: Path | File, + *, + compressed: bool = False, + ignore_fields: set[str] | None = None, +): + if compressed: + with ( + file_path.open("rb") as file, + gzip.GzipFile(fileobj=file, mode="rb") as gz, + io.TextIOWrapper(gz, encoding="utf-8") as text_stream, + ): + data = json.load(text_stream) + elif isinstance(file_path, Path): + with file_path.open(mode="r", encoding="utf-8") as file: + data = json.load(file) + else: + with file_path.open(mode="r") as file: + data = json.load(file) + + if ignore_fields: + data = remove_object_keys(data, ignore_fields) + + return data + + +def read_csv( + file_path: Path | File, + *, + compressed: bool = False, + ignore_columns: set[str] | None = None, + sort_column: typing.Callable[[typing.Any], typing.Any] | None = None, +): + if compressed: + with ( + file_path.open("rb") as file, + gzip.GzipFile(fileobj=file, mode="rb") as gz, + io.TextIOWrapper(gz, encoding="utf-8") as text_stream, + ): + data = list(csv.DictReader(text_stream)) + elif isinstance(file_path, Path): + with file_path.open(mode="r", encoding="utf-8") as file: + data = list(csv.DictReader(file)) + else: + with file_path.open(mode="r") as file: + data = list(csv.DictReader(file)) + + if sort_column: + data.sort(key=sort_column) + + if ignore_columns: + data = remove_object_keys(data, ignore_columns) + + return data + + @contextmanager def create_override(): def pre_save_override(sender: typing.Any, instance: typing.Any, **kwargs): # type: ignore[reportMissingParameterType] @@ -629,222 +691,311 @@ def _test_project(self, projectKey: str, filename: str): assert project_fb_data["progress"] == project.progress, "Progress should be synced with firebase" assert project_fb_data["contributorCount"] == 1, "Contributor count should be synced with firebase" - # NOTE: EXPORTS TESTING + if not test_data.get("expected_project_exports_data"): + return + + # Check aggregated results export - # NOTE: Test AGGREGATED RESULTS aggregated_results_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS, ).first() - - if not aggregated_results_project_asset: - raise AssertionError("Aggregated results project asset not found") - - aggregated_results_filename = ( - Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["aggregated_results"] + assert aggregated_results_project_asset is not None, "Aggregated results project asset not found" + + expected_aggregated_results = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["aggregated_results"]), + ignore_columns={ + "geom", # FIXME: Previously MULTIPOLYGON now it's POLYGON + "url", # FIXME: Does not exist in actual csv + "taskX", # FIXME: Does not exist in actual csv + "taskY", # FIXME: Does not exist in actual csv + }, ) - compare_csv_files( - aggregated_results_project_asset, - aggregated_results_filename, - is_gzip_file=True, - keys_to_ignore={ - "project_internal_id", - "group_internal_id", - "task_internal_id", + actual_aggregated_results = read_csv( + aggregated_results_project_asset.file, + compressed=True, + ignore_columns={ + "project_internal_id", # FIXME: remove this + "group_internal_id", # FIXME: remove this + "task_internal_id", # FIXME: remove this + "geom", # FIXME: Previously MULTIPOLYGON now it's POLYGON }, - message="Difference found for aggregated results export file.", ) + assert expected_aggregated_results == actual_aggregated_results, ( + "Difference found for aggregated results export file." + ) + + # Check aggregated results with geometry export - # NOTE: Test AGGREGATED RESULTS WITH GEOMETRY aggregated_results_with_geometry_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS_WITH_GEOMETRY, ).first() + assert aggregated_results_with_geometry_project_asset is not None, ( + "Aggregated results with geometry project asset not found" + ) - if not aggregated_results_with_geometry_project_asset: - raise AssertionError("Aggregated results with geometry project asset not found") - - expected_aggregated_results_with_geometry_filename = Path( - Config.BASE_DIR, - test_data["expected_project_exports_data"]["aggregated_results_with_geometry"], + expected_aggregated_results_with_geometry = read_json( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["aggregated_results_with_geometry"]), + ignore_fields={ + "url", # FIXME: Does not exist in actual csv + "taskX", # FIXME: Does not exist in actual csv + "taskY", # FIXME: Does not exist in actual csv + "name", # FIXME: Previously "tmp", now "tmp" + random_str + "geometry", # FIXME: Previously MultiPolygon now it's Polygon + }, ) - compare_geojson_files( - aggregated_results_with_geometry_project_asset, - expected_aggregated_results_with_geometry_filename, - is_gzip_file=True, - keys_to_ignore={ - "name", - "project_internal_id", - "group_internal_id", - "task_internal_id", + actual_aggregated_results_with_geometry = read_json( + aggregated_results_with_geometry_project_asset.file, + compressed=True, + ignore_fields={ + "name", # FIXME: Previously "tmp", now "tmp" + random_str + "geometry", # FIXME: Previously MultiPolygon now it's Polygon + "group_internal_id", # FIXME: Remove this + "project_internal_id", # FIXME: Remove this + "task_internal_id", # FIXME: Remove this }, - message="Difference found for aggregated results with geometry export file.", + ) + assert expected_aggregated_results_with_geometry == actual_aggregated_results_with_geometry, ( + "Difference found for aggregated results with geometry export file." ) - # NOTE: Test RESULTS + # Check results export + results_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, export_type=ProjectAssetExportTypeEnum.RESULTS, ).first() - - if not results_project_asset: - raise AssertionError("Results project asset not found") - - results_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["results"] - compare_csv_files( - results_project_asset, - results_filename, - is_gzip_file=True, - keys_to_ignore={ - "project_internal_id", - "group_internal_id", - "task_internal_id", - "user_internal_id", - "timestamp", - "start_time", - "end_time", + assert results_project_asset is not None, "Results project asset not found" + + expected_results = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["results"]), + sort_column=operator.itemgetter("task_id"), + ignore_columns={ + "timestamp", # FIXME: +00 added in our system + "start_time", # FIXME: +00 added in our system + "end_time", # FIXME: +00 added in our system + "client_type", # FIXME: web in previous, 3 in current + "", # FIXME: Not sure what this is at all }, - message="Difference found for results export file.", ) - # NOTE: Test HISTORY + actual_results = read_csv( + results_project_asset.file, + sort_column=operator.itemgetter("task_id"), + ignore_columns={ + "", # FIXME: Not sure what this is at all + "timestamp", # FIXME: +00 added in our system + "start_time", # FIXME: +00 added in our system + "end_time", # FIXME: +00 added in our system + "client_type", # FIXME: web in previous, 3 in current + "task_internal_id", # FIXME: remove this + "user_internal_id", # FIXME: remove this + "group_internal_id", # FIXME: remove this + "project_internal_id", # FIXME: remove this + }, + compressed=True, + ) + assert expected_results == actual_results, "Difference found for results export file." + + # Check history export + history_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, export_type=ProjectAssetExportTypeEnum.HISTORY, ).first() - - history_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["history"] - if not history_project_asset: - raise AssertionError("History project asset not found") - - compare_csv_files( - history_project_asset, - history_filename, - message="Difference found for history export file.", + assert history_project_asset is not None, "History project asset not found" + + expected_history = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["history"]), + ignore_columns={ + "cum_progress", # FIXME: Not correct + "progress", # FIXME: Not correct + "project_id", # FIXME: Previously firebase_id now number + }, + ) + actual_history = read_csv( + history_project_asset.file, + ignore_columns={ + "cum_progress", # FIXME: Not correct + "progress", # FIXME: Not correct + "project_id", # FIXME: Previously firebase_id now number + }, ) + assert expected_history == actual_history, "Difference found for history export file." + + # Check groups export - # NOTE: Test GROUPS groups_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, export_type=ProjectAssetExportTypeEnum.GROUPS, ).first() - - if not groups_project_asset: - raise AssertionError("Groups project asset not found") - groups_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["groups"] - compare_csv_files( - groups_project_asset, - groups_filename, - is_gzip_file=True, - keys_to_ignore={ - "project_internal_id", - "group_internal_id", + assert groups_project_asset is not None, "Groups project asset not found" + + expected_groups = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["groups"]), + ignore_columns={ + "total_area", # FIXME: previously empty, now real value + "time_spent_max_allowed", # FIXME: previously empty, now real value + "required_count", # FIXME: previously looks like verification count + "number_of_users_required", # FIXME: previously looks like verification count + "xMax", # FIXME: previously camelcase + "xMin", # FIXME: previously camelcase + "yMax", # FIXME: previously camelcase + "yMin", # FIXME: previously camelcase }, - message="Difference found for groups export file.", ) + actual_groups = read_csv( + groups_project_asset.file, + compressed=True, + ignore_columns={ + "total_area", # FIXME: previously empty, now real value + "time_spent_max_allowed", # FIXME: previously empty, now real value + "project_internal_id", # FIXME: remove this + "group_internal_id", # FIXME: remove this + "required_count", # FIXME: previously looks like verification count + "number_of_users_required", # FIXME: previously looks like verification count + "x_max", # FIXME: previously camelcase + "x_min", # FIXME: previously camelcase + "y_max", # FIXME: previously camelcase + "y_min", # FIXME: previously camelcase + }, + ) + assert expected_groups == actual_groups, "Difference found for groups export file." - # NOTE: Test TASKS + # Check tasks export tasks_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, export_type=ProjectAssetExportTypeEnum.TASKS, ).first() - - if not tasks_project_asset: - raise AssertionError("Tasks project asset not found") - tasks_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["tasks"] - compare_csv_files( - tasks_project_asset, - tasks_filename, - is_gzip_file=True, - keys_to_ignore={ - "project_internal_id", - "group_internal_id", - "task_internal_id", + assert tasks_project_asset is not None, "Tasks project asset not found" + + expected_tasks = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["tasks"]), + sort_column=operator.itemgetter("task_id"), + ignore_columns={ + "", # FIXME: Not sure what this is at all + "geom", # FIXME: Previously MULTIPOLYGON now it's POLYGON + "url", # FIXME: Does not exist in actual csv + "taskX", # FIXME: Does not exist in actual csv + "taskY", # FIXME: Does not exist in actual csv }, - message="Difference found for tasks export file.", ) + actual_tasks = read_csv( + tasks_project_asset.file, + compressed=True, + sort_column=operator.itemgetter("task_id"), + ignore_columns={ + "", # FIXME: Not sure what this is at all + "project_internal_id", # FIXME: remove this + "group_internal_id", # FIXME: remove this + "task_internal_id", # FIXME: remove this + "geom", # FIXME: Previously MULTIPOLYGON now it's POLYGON + }, + ) + assert expected_tasks == actual_tasks, "Difference found for tasks export file." - # NOTE: Test USERS + # Check users export users_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, export_type=ProjectAssetExportTypeEnum.USERS, ).first() + assert users_project_asset is not None, "Users project asset not found" - if not users_project_asset: - raise AssertionError("Users project asset not found") - users_filename = Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["users"] - compare_csv_files( - users_project_asset, - users_filename, - is_gzip_file=True, - message="Difference found for users export file.", + expected_users = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["users"]), + ignore_columns={ + "groups_completed", # FIXME: Why is this different? + "total_contributions", # FIXME: Why is this different? + }, + ) + actual_users = read_csv( + users_project_asset.file, + compressed=True, + ignore_columns={ + "groups_completed", # FIXME: Why is this different? + "total_contributions", # FIXME: Why is this different? + }, ) + assert expected_users == actual_users, "Difference found for users export file." + + # NOTE: Check aoi export - # NOTE: Test AREA OF INTEREST FILE aoi_project_asset = ProjectAsset.objects.filter( project=project, - type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.AREA_OF_INTEREST, + type=AssetTypeEnum.INPUT, + input_type=ProjectAssetInputTypeEnum.AOI_GEOMETRY, ).first() - - if not aoi_project_asset: - raise AssertionError("AOI Geometry project asset not found") - aoi_geometry_export_filename = Path( - Config.BASE_DIR, - test_data["expected_project_exports_data"]["area_of_interest"], + assert aoi_project_asset is not None, "AOI Geometry project asset not found" + + expected_aoi = read_json( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["area_of_interest"]), + ignore_fields={ + "crs", # FIXME: not defined on uploaded geojson + "name", # FIXME: not defined on uploaded geojson + "properties", # FIXME: empty in uploaded geojson + "coordinates", # FIXME: precision has changed + }, ) - compare_geojson_files( - aoi_project_asset, - aoi_geometry_export_filename, - message="Difference found for AOI Geometry export file.", + actual_aoi = read_json( + aoi_project_asset.file, + ignore_fields={ + "properties", # FIXME: empty in uploaded geojson + "coordinates", # FIXME: precision has changed + }, ) + assert expected_aoi == actual_aoi, "Difference found for AOI geometry export file." - # NOTE: TEST HOT TASKING MANAGER GEOMETRY - # TODO: CHECK if this export is required for completeness project if required, remove this check - if "hot_tasking_manager_geometry" in test_data["expected_project_exports_data"]: - htm_aoi_filename = ( - Path(Config.BASE_DIR) / test_data["expected_project_exports_data"]["hot_tasking_manager_geometry"] - ) - htm_aoi_project_asset = ProjectAsset.objects.filter( - project=project, - type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.HOT_TASKING_MANAGER_GEOMETRIES, - ).first() - - if not htm_aoi_project_asset: - raise AssertionError("HTM AOI Geometry project asset not found") - - compare_geojson_files( - htm_aoi_project_asset, - htm_aoi_filename, - message="Difference found for HTM AOI Geometry export file.", - ) + # Check for moderate to high agreement export - # NOTE: TEST MODERATE TO HIGH AGREEMENT - # TODO: CHECK if this export is required for completeness project if required, remove this check - if "moderate_to_high_agreement" in test_data["expected_project_exports_data"]: - moderate_to_high_agreement_filename = Path( - Config.BASE_DIR, - test_data["expected_project_exports_data"]["moderate_to_high_agreement"], - ) + agreement_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.MODERATE_TO_HIGH_AGREEMENT_YES_MAYBE_GEOMETRIES, + ).first() + assert agreement_project_asset is not None, "Moderate to high agreement project asset not found" - moderate_to_high_agreement_project_asset = ProjectAsset.objects.filter( - project=project, - type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.MODERATE_TO_HIGH_AGREEMENT_YES_MAYBE_GEOMETRIES, - ).first() + expected_agreement = read_json( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["moderate_to_high_agreement"]), + ignore_fields={ + "name", # FIXME: previously full path, not just filename + }, + ) + actual_agreement = read_json( + agreement_project_asset.file, + ignore_fields={ + "name", # FIXME: previously full path, not just filename + }, + ) + assert expected_agreement == actual_agreement, "Difference found for moderate to high agreement export file." - if not moderate_to_high_agreement_project_asset: - raise AssertionError("Moderate to high agreement project asset not found") + # Check hot tasking manager geometry export - compare_geojson_files( - moderate_to_high_agreement_project_asset, - moderate_to_high_agreement_filename, - message="Difference found for moderate to high agreement export file.", - ) + hot_aoi_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.HOT_TASKING_MANAGER_GEOMETRIES, + ).first() + assert hot_aoi_project_asset is not None, "HTM AOI Geometry project asset not found" + + expected_hot_aoi = read_json( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["hot_tasking_manager_geometry"]), + ignore_fields={ + "name", # FIXME: previously full path, not just filename + }, + ) + expected_hot_aoi["features"].sort(key=lambda x: x["properties"]["group_id"]) # type: ignore[reportArgumentType, reportCallIssue] + actual_hot_aoi = read_json( + hot_aoi_project_asset.file, + ignore_fields={ + "name", # FIXME: previously full path, not just filename + }, + ) + actual_hot_aoi["features"].sort(key=lambda x: x["properties"]["group_id"]) # type: ignore[reportArgumentType, reportCallIssue] + assert expected_hot_aoi == actual_hot_aoi, "Difference found for HOT TM AOI geometry export file." diff --git a/assets b/assets index 19ac4bf7..c6bfc5c8 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 19ac4bf72a35fc9c01b68e491582bf44ad27ab09 +Subproject commit c6bfc5c8bf01663f30a2e1d193440f8f175447c0 diff --git a/bulk_ignore_pyright_warnings.py b/bulk_ignore_pyright_warnings.py index e1340229..53ad7970 100644 --- a/bulk_ignore_pyright_warnings.py +++ b/bulk_ignore_pyright_warnings.py @@ -19,6 +19,8 @@ # - We need to use the same version of python as specified on precommit # - We look for pyproject.toml on the current working directory +SEVERITY = "warning" + def run_pyright(path: str) -> dict: # type: ignore[reportMissingTypeArgument] """Run pyright and return JSON output.""" @@ -46,7 +48,7 @@ def apply_specific_ignores(pyright_output: dict): # type: ignore[reportMissingT if "file" not in diag or "range" not in diag or "rule" not in diag or "severity" not in diag: print("Error: diagnostics should define file, range and rule", diag) continue - if diag["severity"] != "warning": + if diag["severity"] != SEVERITY: continue file_path = diag["file"] From a3861458f93b4c0b7572cb9b09a857d7248e63f3 Mon Sep 17 00:00:00 2001 From: tnagorra Date: Fri, 10 Oct 2025 16:16:43 +0545 Subject: [PATCH 4/5] fix(exports): fix exports for find project - use label instead of value for client type in results export - use project firebase_id instead of id in history export - read tile_x, tile_y, tile_z instead of task_x, task_y, task_z in hot tm exports - skip flattening hot tm exports - add url in tile map service project tasks - fix calculation of required_count, progress --- apps/mapping/models.py | 11 + apps/project/exports/mapping_results.py | 6 +- apps/project/exports/project_stats_by_date.py | 2 +- apps/project/exports/project_tasks.py | 3 +- .../exports/tasking_manager_geometries.py | 9 +- ...2e_create_project_tile_map_service_test.py | 310 +++++++----------- apps/project/tests/export_test.py | 1 + apps/project/tests/mutation_test.py | 20 +- assets | 2 +- project_types/base/project.py | 4 +- .../tile_map_service/base/project.py | 18 +- .../tile_map_service/compare/project.py | 35 +- utils/geo/tile_functions.py | 4 +- 13 files changed, 205 insertions(+), 220 deletions(-) diff --git a/apps/mapping/models.py b/apps/mapping/models.py index b94f837f..80d7d02d 100644 --- a/apps/mapping/models.py +++ b/apps/mapping/models.py @@ -26,6 +26,17 @@ def get_client_type(cls, value: str) -> "MappingSessionClientTypeEnum": "web": cls.WEB, }.get(value, cls.UNKNOWN) + @classmethod + def get_client_type_label_sql(cls, field: str) -> str: + return f""" + CASE {field} + WHEN {cls.MOBILE_ANDROID.value} THEN 'android' + WHEN {cls.MOBILE_IOS.value} THEN 'ios' + WHEN {cls.WEB.value} THEN 'web' + ELSE 'unknown' + END + """ + class MappingSession(models.Model): """Model representing a mapping session where a contributor user worked on a specific project task group.""" diff --git a/apps/project/exports/mapping_results.py b/apps/project/exports/mapping_results.py index 49fb9f4a..cdad664f 100644 --- a/apps/project/exports/mapping_results.py +++ b/apps/project/exports/mapping_results.py @@ -8,6 +8,7 @@ from apps.contributor.models import ContributorUser from apps.mapping.models import ( MappingSession, + MappingSessionClientTypeEnum, MappingSessionResult, ) from apps.project.models import Project, ProjectTask, ProjectTaskGroup @@ -19,7 +20,6 @@ def generate_mapping_results(*, destination_filename: Path, project: Project) -> pd.DataFrame: - # TODO: client_type IS ENUM -- CONVERT TO VALUE? sql_query = sql.SQL(f""" COPY ( SELECT @@ -38,7 +38,9 @@ def generate_mapping_results(*, destination_filename: Path, project: Project) -> MS.{fd_name(MappingSession.start_time)} as start_time, MS.{fd_name(MappingSession.end_time)} as end_time, MS.{fd_name(MappingSession.app_version)} as app_version, - MS.{fd_name(MappingSession.client_type)} as client_type, + ( + {MappingSessionClientTypeEnum.get_client_type_label_sql(f"MS.{fd_name(MappingSession.client_type)}")} + ) as client_type, MSR.{fd_name(MappingSessionResult.result)} as result, -- the username for users which login to MapSwipe with their -- OSM account is not defined or ''. diff --git a/apps/project/exports/project_stats_by_date.py b/apps/project/exports/project_stats_by_date.py index 65300342..3c5c0bde 100644 --- a/apps/project/exports/project_stats_by_date.py +++ b/apps/project/exports/project_stats_by_date.py @@ -177,6 +177,6 @@ def get_project_history( # merge contributors and progress project_history_df = progress_by_date_df.merge(contributors_by_date_df, left_on="day", right_on="day") - project_history_df["project_id"] = project.id + project_history_df["project_id"] = project.firebase_id project_history_df.to_csv(destination_filename) return project_history_df diff --git a/apps/project/exports/project_tasks.py b/apps/project/exports/project_tasks.py index acf6444f..83f45970 100644 --- a/apps/project/exports/project_tasks.py +++ b/apps/project/exports/project_tasks.py @@ -44,7 +44,8 @@ def generate_project_tasks( PTG.{fd_name(ProjectTaskGroup.firebase_id)} as group_id, PT.{fd_name(ProjectTask.firebase_id)} as task_id, -- Metadata - ST_AsText({fd_name(ProjectTask.geometry)}) AS geom, + -- NOTE: Using ST_Multi only to make the exports backward compatible with previous exports + ST_AsText(ST_Multi({fd_name(ProjectTask.geometry)})) AS geom, '{project.project_type_specifics.get("zoom_level")}' as tile_z, -- NOTE: Existing tile_x and tile_y are passed from project_type_specifics now -- NOTE: this is destructured by normalize_project_type_specifics(write_sql_to_gzipped_csv) diff --git a/apps/project/exports/tasking_manager_geometries.py b/apps/project/exports/tasking_manager_geometries.py index 853a53fc..07057eab 100644 --- a/apps/project/exports/tasking_manager_geometries.py +++ b/apps/project/exports/tasking_manager_geometries.py @@ -59,9 +59,10 @@ def _get_row_value[T: int | float]( task_id = row[1] - task_x = _get_row_value(column_index_map, row, "task_x") - task_y = _get_row_value(column_index_map, row, "task_y") - task_z = _get_row_value(column_index_map, row, "task_z") + # TODO: rename all task_N to tile_N + task_x = _get_row_value(column_index_map, row, "tile_x") + task_y = _get_row_value(column_index_map, row, "tile_y") + task_z = _get_row_value(column_index_map, row, "tile_z") # TODO: Add no_count here and use later project_data.append( @@ -108,6 +109,8 @@ def _get_row_value[T: int | float]( task_x, task_y, task_z, + # NOTE: We do not flatten to 2D only for backwards compatibility + skip_flatten=True, ), }, ) diff --git a/apps/project/tests/e2e_create_project_tile_map_service_test.py b/apps/project/tests/e2e_create_project_tile_map_service_test.py index 83ba573f..8007c280 100644 --- a/apps/project/tests/e2e_create_project_tile_map_service_test.py +++ b/apps/project/tests/e2e_create_project_tile_map_service_test.py @@ -694,76 +694,62 @@ def _test_project(self, projectKey: str, filename: str): if not test_data.get("expected_project_exports_data"): return - # Check aggregated results export - - aggregated_results_project_asset = ProjectAsset.objects.filter( + # Check groups export + groups_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS, + export_type=ProjectAssetExportTypeEnum.GROUPS, ).first() - assert aggregated_results_project_asset is not None, "Aggregated results project asset not found" + assert groups_project_asset is not None, "Groups project asset not found" - expected_aggregated_results = read_csv( - Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["aggregated_results"]), + expected_groups = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["groups"]), ignore_columns={ - "geom", # FIXME: Previously MULTIPOLYGON now it's POLYGON - "url", # FIXME: Does not exist in actual csv - "taskX", # FIXME: Does not exist in actual csv - "taskY", # FIXME: Does not exist in actual csv + "total_area", # NOTE: previously empty, now real value + "time_spent_max_allowed", # NOTE: previously empty, now real value }, ) - actual_aggregated_results = read_csv( - aggregated_results_project_asset.file, + actual_groups = read_csv( + groups_project_asset.file, compressed=True, ignore_columns={ - "project_internal_id", # FIXME: remove this - "group_internal_id", # FIXME: remove this - "task_internal_id", # FIXME: remove this - "geom", # FIXME: Previously MULTIPOLYGON now it's POLYGON + "total_area", # NOTE: previously empty, now real value + "time_spent_max_allowed", # NOTE: previously empty, now real value + "project_internal_id", # NOTE: added for referencing + "group_internal_id", # NOTE: added for referencing }, ) - assert expected_aggregated_results == actual_aggregated_results, ( - "Difference found for aggregated results export file." - ) - - # Check aggregated results with geometry export + assert expected_groups == actual_groups, "Difference found for groups export file." - aggregated_results_with_geometry_project_asset = ProjectAsset.objects.filter( + # Check tasks export + tasks_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS_WITH_GEOMETRY, + export_type=ProjectAssetExportTypeEnum.TASKS, ).first() - assert aggregated_results_with_geometry_project_asset is not None, ( - "Aggregated results with geometry project asset not found" - ) + assert tasks_project_asset is not None, "Tasks project asset not found" - expected_aggregated_results_with_geometry = read_json( - Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["aggregated_results_with_geometry"]), - ignore_fields={ - "url", # FIXME: Does not exist in actual csv - "taskX", # FIXME: Does not exist in actual csv - "taskY", # FIXME: Does not exist in actual csv - "name", # FIXME: Previously "tmp", now "tmp" + random_str - "geometry", # FIXME: Previously MultiPolygon now it's Polygon + expected_tasks = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["tasks"]), + sort_column=operator.itemgetter("task_id"), + ignore_columns={ + "", # NOTE: dataframe index }, ) - actual_aggregated_results_with_geometry = read_json( - aggregated_results_with_geometry_project_asset.file, + actual_tasks = read_csv( + tasks_project_asset.file, compressed=True, - ignore_fields={ - "name", # FIXME: Previously "tmp", now "tmp" + random_str - "geometry", # FIXME: Previously MultiPolygon now it's Polygon - "group_internal_id", # FIXME: Remove this - "project_internal_id", # FIXME: Remove this - "task_internal_id", # FIXME: Remove this + sort_column=operator.itemgetter("task_id"), + ignore_columns={ + "", # NOTE: dataframe index + "project_internal_id", # NOTE: added for referencing + "group_internal_id", # NOTE: added for referencing + "task_internal_id", # NOTE: added for referencing }, ) - assert expected_aggregated_results_with_geometry == actual_aggregated_results_with_geometry, ( - "Difference found for aggregated results with geometry export file." - ) + assert expected_tasks == actual_tasks, "Difference found for tasks export file." # Check results export - results_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, @@ -775,130 +761,118 @@ def _test_project(self, projectKey: str, filename: str): Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["results"]), sort_column=operator.itemgetter("task_id"), ignore_columns={ - "timestamp", # FIXME: +00 added in our system - "start_time", # FIXME: +00 added in our system - "end_time", # FIXME: +00 added in our system - "client_type", # FIXME: web in previous, 3 in current - "", # FIXME: Not sure what this is at all + "", # NOTE: dataframe index }, ) actual_results = read_csv( results_project_asset.file, sort_column=operator.itemgetter("task_id"), ignore_columns={ - "", # FIXME: Not sure what this is at all - "timestamp", # FIXME: +00 added in our system - "start_time", # FIXME: +00 added in our system - "end_time", # FIXME: +00 added in our system - "client_type", # FIXME: web in previous, 3 in current - "task_internal_id", # FIXME: remove this - "user_internal_id", # FIXME: remove this - "group_internal_id", # FIXME: remove this - "project_internal_id", # FIXME: remove this + "", # NOTE: dataframe index + "task_internal_id", # NOTE: added for referencing + "user_internal_id", # NOTE: added for referencing + "group_internal_id", # NOTE: added for referencing + "project_internal_id", # NOTE: added for referencing }, compressed=True, ) assert expected_results == actual_results, "Difference found for results export file." - # Check history export - - history_project_asset = ProjectAsset.objects.filter( + # Check aoi export + aoi_project_asset = ProjectAsset.objects.filter( project=project, - type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.HISTORY, + type=AssetTypeEnum.INPUT, + input_type=ProjectAssetInputTypeEnum.AOI_GEOMETRY, ).first() - assert history_project_asset is not None, "History project asset not found" + assert aoi_project_asset is not None, "AOI Geometry project asset not found" - expected_history = read_csv( - Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["history"]), - ignore_columns={ - "cum_progress", # FIXME: Not correct - "progress", # FIXME: Not correct - "project_id", # FIXME: Previously firebase_id now number + expected_aoi = read_json( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["area_of_interest"]), + ignore_fields={ + "crs", # NOTE: crs has almost no data + "name", # NOTE: previously system file path + "properties", # FIXME: previously has id (index) + "coordinates", # FIXME: precision has changed }, ) - actual_history = read_csv( - history_project_asset.file, - ignore_columns={ - "cum_progress", # FIXME: Not correct - "progress", # FIXME: Not correct - "project_id", # FIXME: Previously firebase_id now number + actual_aoi = read_json( + aoi_project_asset.file, + ignore_fields={ + "properties", # FIXME: previously has id (index) + "coordinates", # FIXME: precision has changed }, ) - assert expected_history == actual_history, "Difference found for history export file." - - # Check groups export + assert expected_aoi == actual_aoi, "Difference found for AOI geometry export file." - groups_project_asset = ProjectAsset.objects.filter( + # Check aggregated results export + aggregated_results_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.GROUPS, + export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS, ).first() - assert groups_project_asset is not None, "Groups project asset not found" + assert aggregated_results_project_asset is not None, "Aggregated results project asset not found" - expected_groups = read_csv( - Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["groups"]), - ignore_columns={ - "total_area", # FIXME: previously empty, now real value - "time_spent_max_allowed", # FIXME: previously empty, now real value - "required_count", # FIXME: previously looks like verification count - "number_of_users_required", # FIXME: previously looks like verification count - "xMax", # FIXME: previously camelcase - "xMin", # FIXME: previously camelcase - "yMax", # FIXME: previously camelcase - "yMin", # FIXME: previously camelcase - }, + expected_aggregated_results = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["aggregated_results"]), ) - actual_groups = read_csv( - groups_project_asset.file, + actual_aggregated_results = read_csv( + aggregated_results_project_asset.file, compressed=True, ignore_columns={ - "total_area", # FIXME: previously empty, now real value - "time_spent_max_allowed", # FIXME: previously empty, now real value - "project_internal_id", # FIXME: remove this - "group_internal_id", # FIXME: remove this - "required_count", # FIXME: previously looks like verification count - "number_of_users_required", # FIXME: previously looks like verification count - "x_max", # FIXME: previously camelcase - "x_min", # FIXME: previously camelcase - "y_max", # FIXME: previously camelcase - "y_min", # FIXME: previously camelcase + "project_internal_id", # NOTE: added for referencing + "group_internal_id", # NOTE: added for referencing + "task_internal_id", # NOTE: added for referencing }, ) - assert expected_groups == actual_groups, "Difference found for groups export file." + assert expected_aggregated_results == actual_aggregated_results, ( + "Difference found for aggregated results export file." + ) - # Check tasks export - tasks_project_asset = ProjectAsset.objects.filter( + # Check aggregated results with geometry export + aggregated_results_with_geometry_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.TASKS, + export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS_WITH_GEOMETRY, ).first() - assert tasks_project_asset is not None, "Tasks project asset not found" + assert aggregated_results_with_geometry_project_asset is not None, ( + "Aggregated results with geometry project asset not found" + ) - expected_tasks = read_csv( - Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["tasks"]), - sort_column=operator.itemgetter("task_id"), - ignore_columns={ - "", # FIXME: Not sure what this is at all - "geom", # FIXME: Previously MULTIPOLYGON now it's POLYGON - "url", # FIXME: Does not exist in actual csv - "taskX", # FIXME: Does not exist in actual csv - "taskY", # FIXME: Does not exist in actual csv + expected_aggregated_results_with_geometry = read_json( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["aggregated_results_with_geometry"]), + ignore_fields={ + "name", # NOTE: Previously "tmp", now "tmp" + random_str }, ) - actual_tasks = read_csv( - tasks_project_asset.file, + actual_aggregated_results_with_geometry = read_json( + aggregated_results_with_geometry_project_asset.file, compressed=True, - sort_column=operator.itemgetter("task_id"), - ignore_columns={ - "", # FIXME: Not sure what this is at all - "project_internal_id", # FIXME: remove this - "group_internal_id", # FIXME: remove this - "task_internal_id", # FIXME: remove this - "geom", # FIXME: Previously MULTIPOLYGON now it's POLYGON + ignore_fields={ + "name", # NOTE: Previously "tmp", now "tmp" + random_str + "project_internal_id", # NOTE: added for referencing + "group_internal_id", # NOTE: added for referencing + "task_internal_id", # NOTE: added for referencing }, ) - assert expected_tasks == actual_tasks, "Difference found for tasks export file." + assert expected_aggregated_results_with_geometry == actual_aggregated_results_with_geometry, ( + "Difference found for aggregated results with geometry export file." + ) + + # Check history export + history_project_asset = ProjectAsset.objects.filter( + project=project, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.HISTORY, + ).first() + assert history_project_asset is not None, "History project asset not found" + + expected_history = read_csv( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["history"]), + ) + actual_history = read_csv( + history_project_asset.file, + ) + assert expected_history == actual_history, "Difference found for history export file." # Check users export users_project_asset = ProjectAsset.objects.filter( @@ -910,50 +884,39 @@ def _test_project(self, projectKey: str, filename: str): expected_users = read_csv( Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["users"]), - ignore_columns={ - "groups_completed", # FIXME: Why is this different? - "total_contributions", # FIXME: Why is this different? - }, ) actual_users = read_csv( users_project_asset.file, compressed=True, - ignore_columns={ - "groups_completed", # FIXME: Why is this different? - "total_contributions", # FIXME: Why is this different? - }, ) assert expected_users == actual_users, "Difference found for users export file." - # NOTE: Check aoi export - - aoi_project_asset = ProjectAsset.objects.filter( + # Check hot tasking manager geometry export + hot_aoi_project_asset = ProjectAsset.objects.filter( project=project, - type=AssetTypeEnum.INPUT, - input_type=ProjectAssetInputTypeEnum.AOI_GEOMETRY, + type=AssetTypeEnum.EXPORT, + export_type=ProjectAssetExportTypeEnum.HOT_TASKING_MANAGER_GEOMETRIES, ).first() - assert aoi_project_asset is not None, "AOI Geometry project asset not found" + assert hot_aoi_project_asset is not None, "HOT TM AOI Geometry project asset not found" - expected_aoi = read_json( - Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["area_of_interest"]), + expected_hot_aoi = read_json( + Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["hot_tasking_manager_geometry"]), ignore_fields={ - "crs", # FIXME: not defined on uploaded geojson - "name", # FIXME: not defined on uploaded geojson - "properties", # FIXME: empty in uploaded geojson - "coordinates", # FIXME: precision has changed + "name", # NOTE: previously full path, not just filename }, ) - actual_aoi = read_json( - aoi_project_asset.file, + expected_hot_aoi["features"].sort(key=lambda x: x["properties"]["group_id"]) # type: ignore[reportArgumentType, reportCallIssue] + actual_hot_aoi = read_json( + hot_aoi_project_asset.file, ignore_fields={ - "properties", # FIXME: empty in uploaded geojson - "coordinates", # FIXME: precision has changed + "name", # NOTE: previously full path, not just filename }, ) - assert expected_aoi == actual_aoi, "Difference found for AOI geometry export file." - # Check for moderate to high agreement export + actual_hot_aoi["features"].sort(key=lambda x: x["properties"]["group_id"]) # type: ignore[reportArgumentType, reportCallIssue] + assert expected_hot_aoi == actual_hot_aoi, "Difference found for HOT TM AOI geometry export file." + # Check for moderate to high agreement export agreement_project_asset = ProjectAsset.objects.filter( project=project, type=AssetTypeEnum.EXPORT, @@ -964,38 +927,13 @@ def _test_project(self, projectKey: str, filename: str): expected_agreement = read_json( Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["moderate_to_high_agreement"]), ignore_fields={ - "name", # FIXME: previously full path, not just filename + "name", # NOTE: previously full path, not just filename }, ) actual_agreement = read_json( agreement_project_asset.file, ignore_fields={ - "name", # FIXME: previously full path, not just filename + "name", # NOTE: previously full path, not just filename }, ) assert expected_agreement == actual_agreement, "Difference found for moderate to high agreement export file." - - # Check hot tasking manager geometry export - - hot_aoi_project_asset = ProjectAsset.objects.filter( - project=project, - type=AssetTypeEnum.EXPORT, - export_type=ProjectAssetExportTypeEnum.HOT_TASKING_MANAGER_GEOMETRIES, - ).first() - assert hot_aoi_project_asset is not None, "HTM AOI Geometry project asset not found" - - expected_hot_aoi = read_json( - Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["hot_tasking_manager_geometry"]), - ignore_fields={ - "name", # FIXME: previously full path, not just filename - }, - ) - expected_hot_aoi["features"].sort(key=lambda x: x["properties"]["group_id"]) # type: ignore[reportArgumentType, reportCallIssue] - actual_hot_aoi = read_json( - hot_aoi_project_asset.file, - ignore_fields={ - "name", # FIXME: previously full path, not just filename - }, - ) - actual_hot_aoi["features"].sort(key=lambda x: x["properties"]["group_id"]) # type: ignore[reportArgumentType, reportCallIssue] - assert expected_hot_aoi == actual_hot_aoi, "Difference found for HOT TM AOI geometry export file." diff --git a/apps/project/tests/export_test.py b/apps/project/tests/export_test.py index 978ed7b0..9c2c0244 100644 --- a/apps/project/tests/export_test.py +++ b/apps/project/tests/export_test.py @@ -95,6 +95,7 @@ def setUpClass(cls): project_type_specifics=FindProjectTaskProperty( tile_x=1, tile_y=2, + url="https://some-service.com/14/1/2/", ).model_dump(), ) ] diff --git a/apps/project/tests/mutation_test.py b/apps/project/tests/mutation_test.py index 8009cebd..0a24aa37 100644 --- a/apps/project/tests/mutation_test.py +++ b/apps/project/tests/mutation_test.py @@ -1393,7 +1393,7 @@ class TaskGroupType(typing.TypedDict): { "firebase_id": "g101", "number_of_tasks": 18, - "required_count": 18 * 10, + "required_count": 10, "total_area": 210.10735845202447, "project_type_specifics": { "x_max": 24152, @@ -1405,7 +1405,7 @@ class TaskGroupType(typing.TypedDict): { "firebase_id": "g102", "number_of_tasks": 24, - "required_count": 24 * 10, + "required_count": 10, "total_area": 280.2915392364502, "project_type_specifics": { "x_max": 24153, @@ -1417,7 +1417,7 @@ class TaskGroupType(typing.TypedDict): { "firebase_id": "g103", "number_of_tasks": 24, - "required_count": 24 * 10, + "required_count": 10, "total_area": 280.4398676951218, "project_type_specifics": { "x_max": 24153, @@ -1429,7 +1429,7 @@ class TaskGroupType(typing.TypedDict): { "firebase_id": "g104", "number_of_tasks": 6, - "required_count": 6 * 10, + "required_count": 10, "total_area": 70.14703242812156, "project_type_specifics": { "x_max": 24150, @@ -1445,6 +1445,8 @@ class TaskGroupType(typing.TypedDict): "project_type_specifics": { "tile_x": 24147, "tile_y": 13753, + "url": "https://hi-there/24147/13753/15", + "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24147/13753.jpg?connectId=dummy-maxar-standard", }, }, { @@ -1452,6 +1454,8 @@ class TaskGroupType(typing.TypedDict): "project_type_specifics": { "tile_x": 24147, "tile_y": 13754, + "url": "https://hi-there/24147/13754/15", + "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24147/13754.jpg?connectId=dummy-maxar-standard", }, }, { @@ -1459,6 +1463,8 @@ class TaskGroupType(typing.TypedDict): "project_type_specifics": { "tile_x": 24147, "tile_y": 13755, + "url": "https://hi-there/24147/13755/15", + "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24147/13755.jpg?connectId=dummy-maxar-standard", }, }, { @@ -1466,6 +1472,8 @@ class TaskGroupType(typing.TypedDict): "project_type_specifics": { "tile_x": 24148, "tile_y": 13753, + "url": "https://hi-there/24148/13753/15", + "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24148/13753.jpg?connectId=dummy-maxar-standard", }, }, { @@ -1473,6 +1481,8 @@ class TaskGroupType(typing.TypedDict): "project_type_specifics": { "tile_x": 24148, "tile_y": 13754, + "url": "https://hi-there/24148/13754/15", + "url_b": "https://services.digitalglobe.com/earthservice/tmsaccess/tms/1.0.0/DigitalGlobe%3AImageryTileService@EPSG%3A3857@jpg/15/24148/13754.jpg?connectId=dummy-maxar-standard", }, }, ] @@ -1482,7 +1492,7 @@ class TaskGroupType(typing.TypedDict): project_task_qs = ProjectTask.objects.filter(task_group__project=latest_project) assert { - "required_results": sum(task_group["required_count"] for task_group in expected_task_groups), + "required_results": (18 + 24 + 24 + 6) * 10, "tasks_groups_count": project_task_group_qs.count(), "tasks_groups": list( project_task_group_qs.order_by("id").values( diff --git a/assets b/assets index c6bfc5c8..6771f901 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit c6bfc5c8bf01663f30a2e1d193440f8f175447c0 +Subproject commit 6771f9018fc67d39c11277924a0e2aec14149db8 diff --git a/project_types/base/project.py b/project_types/base/project.py index 27f91d44..d5369065 100644 --- a/project_types/base/project.py +++ b/project_types/base/project.py @@ -142,13 +142,13 @@ def analyze_groups(self): ) # NOTE: After number_of_tasks is calculated project_task_groups_qs.update( - required_count=models.F("number_of_tasks") * self.project.verification_number, + required_count=self.project.verification_number, time_spent_max_allowed=(models.F("number_of_tasks") * self.get_max_time_spend_percentile()), ) self.project.required_results = ( ProjectTaskGroup.objects.filter(project_id=self.project.pk).aggregate( - required_results=models.Sum("required_count"), + required_results=models.Sum("number_of_tasks") * self.project.verification_number, ) )["required_results"] or 0 diff --git a/project_types/tile_map_service/base/project.py b/project_types/tile_map_service/base/project.py index 4335d92c..b0aea5eb 100644 --- a/project_types/tile_map_service/base/project.py +++ b/project_types/tile_map_service/base/project.py @@ -51,6 +51,8 @@ class TileMapServiceProjectTaskGroupProperty(base_project.BaseProjectTaskGroupPr class TileMapServiceProjectTaskProperty(base_project.BaseProjectTaskProperty): tile_x: int tile_y: int + # NOTE: We added URL as it's used directly when creating exports + url: str class TileMapServiceBaseProject[ @@ -132,6 +134,17 @@ def get_feature(task: ProjectTask): self.project.project_type_specific_output_asset = asset self.project.save(update_fields=("project_type_specific_output_asset",)) + def get_task_specifics_for_db(self, tile_x: int, tile_y: int) -> TileMapServiceProjectTaskProperty: + return self.project_task_property_class( + tile_x=tile_x, + tile_y=tile_y, + url=self.project_type_specifics.tile_server_property.generate_url( + tile_x, + tile_y, + self.project_type_specifics.zoom_level, + ), + ) + @typing.override def create_tasks( self, @@ -155,10 +168,7 @@ def create_tasks( task_group_id=group.pk, geometry=geometry, project_type_specifics=clean_up_none_keys( - self.project_task_property_class( - tile_x=tile_x, - tile_y=tile_y, - ).model_dump(), + self.get_task_specifics_for_db(tile_x, tile_y).model_dump(), ), ), ) diff --git a/project_types/tile_map_service/compare/project.py b/project_types/tile_map_service/compare/project.py index 208f0a13..2ba959ed 100644 --- a/project_types/tile_map_service/compare/project.py +++ b/project_types/tile_map_service/compare/project.py @@ -14,7 +14,9 @@ class CompareProjectProperty(base_project.TileMapServiceProjectProperty): class CompareProjectTaskGroupProperty(base_project.TileMapServiceProjectTaskGroupProperty): ... -class CompareProjectTaskProperty(base_project.TileMapServiceProjectTaskProperty): ... +class CompareProjectTaskProperty(base_project.TileMapServiceProjectTaskProperty): + # NOTE: We added URL as it's used directly when creating exports + url_b: str class CompareProject( @@ -37,6 +39,23 @@ def __init__(self, project: Project): def get_max_time_spend_percentile(self) -> float: return 11.2 + @typing.override + def get_task_specifics_for_db(self, tile_x: int, tile_y: int) -> CompareProjectTaskProperty: + return self.project_task_property_class( + tile_x=tile_x, + tile_y=tile_y, + url=self.project_type_specifics.tile_server_property.generate_url( + tile_x, + tile_y, + self.project_type_specifics.zoom_level, + ), + url_b=self.project_type_specifics.tile_server_b_property.generate_url( + tile_x, + tile_y, + self.project_type_specifics.zoom_level, + ), + ) + # FIREBASE @typing.override @@ -46,24 +65,14 @@ def skip_tasks_on_firebase(self) -> bool: @typing.override def get_task_specifics_for_firebase(self, task: ProjectTask) -> firebase_models.FbMappingTaskCompareCreateOnlyInput: task_specifics = self.project_task_property_class.model_validate(task.project_type_specifics) - tsp = self.project_type_specifics.tile_server_property - tsp_b = self.project_type_specifics.tile_server_b_property return firebase_models.FbMappingTaskCompareCreateOnlyInput( groupId=str(task.task_group.firebase_id), taskId=task.firebase_id, taskX=task_specifics.tile_x, taskY=task_specifics.tile_y, - url=tsp.generate_url( - task_specifics.tile_x, - task_specifics.tile_y, - self.project_type_specifics.zoom_level, - ), - urlB=tsp_b.generate_url( - task_specifics.tile_x, - task_specifics.tile_y, - self.project_type_specifics.zoom_level, - ), + url=task_specifics.url, + urlB=task_specifics.url_b, ) @typing.override diff --git a/utils/geo/tile_functions.py b/utils/geo/tile_functions.py index 86db6052..14fdaa40 100644 --- a/utils/geo/tile_functions.py +++ b/utils/geo/tile_functions.py @@ -91,7 +91,7 @@ def quad_key_to_bing_url(quad_key: str, api_key: str): # FIXME(tnagorra): Add typings for osgeo -def geometry_from_tile_coords(tile_x: float, tile_y: float, zoom: int) -> str: +def geometry_from_tile_coords(tile_x: float, tile_y: float, zoom: int, *, skip_flatten: bool = False) -> str: """Compute the polygon geometry of a tile map service tile.""" # Calculate lat, lon of upper left corner of tile pixel_x = tile_x * 256 @@ -113,7 +113,7 @@ def geometry_from_tile_coords(tile_x: float, tile_y: float, zoom: int) -> str: poly = ogr.Geometry(ogr.wkbPolygon) poly.AddGeometry(ring) - if poly.GetCoordinateDimension() == 3: + if not skip_flatten and poly.GetCoordinateDimension() == 3: poly.FlattenTo2D() return poly.ExportToWkt() From fcd7cd5e4c0f33875ec1df44c08fbcb5a91308e0 Mon Sep 17 00:00:00 2001 From: tnagorra Date: Fri, 10 Oct 2025 16:45:26 +0545 Subject: [PATCH 5/5] fix(slack): update status of slack progress notification --- apps/project/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/project/tasks.py b/apps/project/tasks.py index 25da94e3..6e084d2c 100644 --- a/apps/project/tasks.py +++ b/apps/project/tasks.py @@ -104,4 +104,4 @@ def send_slack_message_for_project(project_id: int, action: Literal["progress-ch update_base_slack_message(client=mapslack, project=project, ts=base_slack_message_ts) if action == "progress-change": project.slack_progress_notifications = project.progress - project.save(update_fields=["slack_message_notifications"]) + project.save(update_fields=["slack_progress_notifications"])