Skip to content

Commit a74cb57

Browse files
susilnemsusilnem
committed
chore(test): Update version for shapely and use wkt dump
- Update export data for street - Use wkt dump instead of to_wkt - Update version of shapely - Add export test for street project
1 parent d9d53af commit a74cb57

File tree

5 files changed

+967
-733
lines changed

5 files changed

+967
-733
lines changed

‎apps/project/tests/e2e_create_street_project_test.py‎

Lines changed: 232 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import logging
2+
import operator
23
import typing
34
from contextlib import contextmanager
45
from datetime import datetime
@@ -9,6 +10,7 @@
910
from django.db.models.signals import pre_save
1011
from ulid import ULID
1112

13+
from apps.common.models import AssetTypeEnum
1214
from apps.common.utils import decode_tasks, remove_object_keys
1315
from apps.contributor.factories import ContributorUserFactory
1416
from apps.contributor.models import ContributorUserGroup
@@ -20,7 +22,14 @@
2022
MappingSessionUserGroup,
2123
MappingSessionUserGroupTemp,
2224
)
23-
from apps.project.models import Organization, Project
25+
from apps.project.models import (
26+
Organization,
27+
Project,
28+
ProjectAsset,
29+
ProjectAssetExportTypeEnum,
30+
ProjectAssetInputTypeEnum,
31+
)
32+
from apps.project.tests.e2e_create_project_tile_map_service_test import read_csv, read_json
2433
from apps.tutorial.models import Tutorial
2534
from apps.user.factories import UserFactory
2635
from main.config import Config
@@ -618,3 +627,225 @@ def _test_project(self, filename: str):
618627

619628
project.refresh_from_db()
620629
assert project.progress == test_data["expected_pulled_results_data"]["progress"]
630+
631+
# Check if progress and contributorCount synced to firebase
632+
project_fb_data = project_fb_ref.get()
633+
assert project_fb_data is not None, "Project in firebase is None"
634+
assert isinstance(project_fb_data, dict), "Project in firebase should be a dictionary"
635+
assert project_fb_data["progress"] == project.progress, "Progress should be synced with firebase"
636+
assert project_fb_data["contributorCount"] == 1, "Contributor count should be synced with firebase"
637+
638+
# Check groups export
639+
groups_project_asset = ProjectAsset.objects.filter(
640+
project=project,
641+
type=AssetTypeEnum.EXPORT,
642+
export_type=ProjectAssetExportTypeEnum.GROUPS,
643+
).first()
644+
assert groups_project_asset is not None, "Groups project asset not found"
645+
646+
expected_groups = read_csv(
647+
Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["groups"]),
648+
ignore_columns={
649+
"total_area", # NOTE: previously empty, now real value
650+
"time_spent_max_allowed", # NOTE: previously empty, now real value
651+
},
652+
)
653+
actual_groups = read_csv(
654+
groups_project_asset.file,
655+
compressed=True,
656+
ignore_columns={
657+
"total_area", # NOTE: previously empty, now real value
658+
"time_spent_max_allowed", # NOTE: previously empty, now real value
659+
"project_internal_id", # NOTE: added for referencing
660+
"group_internal_id", # NOTE: added for referencing
661+
},
662+
)
663+
assert expected_groups == actual_groups, "Difference found for groups export file."
664+
665+
# Check tasks export
666+
tasks_project_asset = ProjectAsset.objects.filter(
667+
project=project,
668+
type=AssetTypeEnum.EXPORT,
669+
export_type=ProjectAssetExportTypeEnum.TASKS,
670+
).first()
671+
assert tasks_project_asset is not None, "Tasks project asset not found"
672+
673+
expected_tasks = read_csv(
674+
Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["tasks"]),
675+
ignore_columns={
676+
"", # NOTE: dataframe index
677+
"urlB", # FIXME: old system contains this field
678+
"tile_x", # FIXME: old system contains this field
679+
"tile_y", # FIXME: old system contains this field
680+
},
681+
)
682+
actual_tasks = read_csv(
683+
tasks_project_asset.file,
684+
compressed=True,
685+
ignore_columns={
686+
"", # NOTE: dataframe index
687+
"project_internal_id", # NOTE: added for referencing
688+
"group_internal_id", # NOTE: added for referencing
689+
"task_internal_id", # NOTE: added for referencing
690+
},
691+
)
692+
assert expected_tasks == actual_tasks, "Difference found for tasks export file."
693+
694+
# Check results export
695+
results_project_asset = ProjectAsset.objects.filter(
696+
project=project,
697+
type=AssetTypeEnum.EXPORT,
698+
export_type=ProjectAssetExportTypeEnum.RESULTS,
699+
).first()
700+
assert results_project_asset is not None, "Results project asset not found"
701+
702+
expected_results = read_csv(
703+
Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["results"]),
704+
sort_column=operator.itemgetter("task_id"),
705+
ignore_columns={
706+
"", # NOTE: dataframe index
707+
},
708+
)
709+
actual_results = read_csv(
710+
results_project_asset.file,
711+
sort_column=operator.itemgetter("task_id"),
712+
ignore_columns={
713+
"", # NOTE: dataframe index
714+
"task_internal_id", # NOTE: added for referencing
715+
"user_internal_id", # NOTE: added for referencing
716+
"group_internal_id", # NOTE: added for referencing
717+
"project_internal_id", # NOTE: added for referencing
718+
},
719+
compressed=True,
720+
)
721+
assert expected_results == actual_results, "Difference found for results export file."
722+
723+
# Check aoi export
724+
aoi_project_asset = ProjectAsset.objects.filter(
725+
project=project,
726+
type=AssetTypeEnum.INPUT,
727+
input_type=ProjectAssetInputTypeEnum.AOI_GEOMETRY,
728+
).first()
729+
assert aoi_project_asset is not None, "AOI Geometry project asset not found"
730+
731+
expected_aoi = read_json(
732+
Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["area_of_interest"]),
733+
ignore_fields={
734+
"crs", # NOTE: crs has almost no data
735+
"name", # NOTE: previously system file path
736+
"properties", # FIXME: previously has id (index)
737+
"coordinates", # FIXME: precision has changed
738+
},
739+
)
740+
actual_aoi = read_json(
741+
aoi_project_asset.file,
742+
ignore_fields={
743+
"properties", # FIXME: previously has id (index)
744+
"coordinates", # FIXME: precision has changed
745+
},
746+
)
747+
assert expected_aoi == actual_aoi, "Difference found for AOI geometry export file."
748+
749+
# Check aggregated results export
750+
aggregated_results_project_asset = ProjectAsset.objects.filter(
751+
project=project,
752+
type=AssetTypeEnum.EXPORT,
753+
export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS,
754+
).first()
755+
assert aggregated_results_project_asset is not None, "Aggregated results project asset not found"
756+
757+
expected_aggregated_results = read_csv(
758+
Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["aggregated_results"]),
759+
ignore_columns={
760+
"urlB", # FIXME: old system contains this field
761+
"tile_x", # FIXME: old system contains this field
762+
"tile_y", # FIXME: old system contains this field
763+
"3_count", # FIXME: old system contains this field
764+
"3_share", # FIXME: old system contains this field
765+
},
766+
)
767+
actual_aggregated_results = read_csv(
768+
aggregated_results_project_asset.file,
769+
compressed=True,
770+
ignore_columns={
771+
"project_internal_id", # NOTE: added for referencing
772+
"group_internal_id", # NOTE: added for referencing
773+
"task_internal_id", # NOTE: added for referencing
774+
"task_id.1", # FIXME: This column is extra
775+
"group_id.1", # FIXME: This column is extra
776+
},
777+
)
778+
779+
assert expected_aggregated_results == actual_aggregated_results, (
780+
"Difference found for aggregated results export file."
781+
)
782+
783+
# Check aggregated results with geometry export
784+
aggregated_results_with_geometry_project_asset = ProjectAsset.objects.filter(
785+
project=project,
786+
type=AssetTypeEnum.EXPORT,
787+
export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS_WITH_GEOMETRY,
788+
).first()
789+
assert aggregated_results_with_geometry_project_asset is not None, (
790+
"Aggregated results with geometry project asset not found"
791+
)
792+
793+
expected_aggregated_results_with_geometry = read_json(
794+
Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["aggregated_results_with_geometry"]),
795+
ignore_fields={
796+
"name", # NOTE: Previously "tmp", now "tmp" + random_str
797+
"urlB", # FIXME: old system contains this field
798+
"tile_x", # FIXME: old system contains this field
799+
"tile_y", # FIXME: old system contains this field
800+
"3_count", # FIXME: old system contains this field
801+
"3_share", # FIXME: old system contains this field
802+
},
803+
)
804+
actual_aggregated_results_with_geometry = read_json(
805+
aggregated_results_with_geometry_project_asset.file,
806+
compressed=True,
807+
ignore_fields={
808+
"name", # NOTE: Previously "tmp", now "tmp" + random_str
809+
"project_internal_id", # NOTE: added for referencing
810+
"group_internal_id", # NOTE: added for referencing
811+
"task_internal_id", # NOTE: added for referencing
812+
"task_id.1", # FIXME: This column is extra
813+
"group_id.1", # FIXME: This column is extra
814+
},
815+
)
816+
assert expected_aggregated_results_with_geometry == actual_aggregated_results_with_geometry, (
817+
"Difference found for aggregated results with geometry export file."
818+
)
819+
820+
# Check history export
821+
history_project_asset = ProjectAsset.objects.filter(
822+
project=project,
823+
type=AssetTypeEnum.EXPORT,
824+
export_type=ProjectAssetExportTypeEnum.HISTORY,
825+
).first()
826+
assert history_project_asset is not None, "History project asset not found"
827+
828+
expected_history = read_csv(
829+
Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["history"]),
830+
)
831+
actual_history = read_csv(
832+
history_project_asset.file,
833+
)
834+
assert expected_history == actual_history, "Difference found for history export file."
835+
836+
# Check users export
837+
users_project_asset = ProjectAsset.objects.filter(
838+
project=project,
839+
type=AssetTypeEnum.EXPORT,
840+
export_type=ProjectAssetExportTypeEnum.USERS,
841+
).first()
842+
assert users_project_asset is not None, "Users project asset not found"
843+
844+
expected_users = read_csv(
845+
Path(Config.BASE_DIR, test_data["expected_project_exports_data"]["users"]),
846+
)
847+
actual_users = read_csv(
848+
users_project_asset.file,
849+
compressed=True,
850+
)
851+
assert expected_users == actual_users, "Difference found for users export file."

‎assets‎

Submodule assets updated 15 files

‎project_types/street/project.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from django.core.files.base import ContentFile
88
from pydantic import BaseModel
99
from pyfirebase_mapswipe import models as firebase_models
10-
from shapely import to_wkt
10+
from shapely import wkt
1111
from ulid import ULID
1212

1313
from apps.common.models import AssetMimetypeEnum, AssetTypeEnum
@@ -124,16 +124,16 @@ def create_tasks(
124124
features = list(zip(raw_group["feature_ids"], raw_group["features"], strict=True))
125125

126126
for f_id, feature in features:
127-
geometry_str = to_wkt(feature)
127+
geometry_str = wkt.dumps(feature)
128128

129129
bulk_mgr.add(
130130
ProjectTask(
131-
firebase_id=f"{f_id}",
131+
firebase_id=str(f_id),
132132
task_group_id=group.pk,
133133
geometry=geometry_str,
134134
project_type_specifics=self.project_task_property_class(
135-
task_id=f"{f_id}",
136-
group_id=f"g{group.pk}",
135+
task_id=str(f_id),
136+
group_id=group.firebase_id,
137137
).model_dump(),
138138
),
139139
)

‎pyproject.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,9 @@ dependencies = [
4444
"firebase-admin==6.9.0",
4545
"pyfirebase-mapswipe",
4646
"python-magic>=0.4.27",
47+
"shapely>=2.1.2",
4748
"mercantile>=1.2.1",
48-
"vt2geojson",
49+
"vt2geojson>=0.2.1",
4950
"pandas~=2.3.1",
5051
"slack-sdk>=3.36.0",
5152
]

0 commit comments

Comments
 (0)