Skip to content

Commit d6a1a95

Browse files
susilnemsusilnem
committed
feat(test): Add e2e testing for export file
1 parent 3d08e88 commit d6a1a95

File tree

3 files changed

+253
-3
lines changed

3 files changed

+253
-3
lines changed

‎apps/common/utils.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,20 @@
11
import base64
2+
import csv
23
import gzip
4+
import io
35
import json
46
import typing
7+
from pathlib import Path
58

69
from django.core.files.storage import FileSystemStorage
710
from django.db.models.fields import files
811

912
from main.config import Config
1013
from utils.common import is_file_empty
1114

15+
if typing.TYPE_CHECKING:
16+
from apps.project.models import ProjectAsset
17+
1218

1319
@typing.overload
1420
def get_absolute_uri(file: None) -> None: ...
@@ -47,3 +53,67 @@ def decode_tasks(encoded_task: str) -> list[dict[str, typing.Any]]:
4753
compressed_bytes = base64.b64decode(encoded_task)
4854
json_bytes = gzip.decompress(compressed_bytes)
4955
return json.loads(json_bytes.decode("utf-8"))
56+
57+
58+
def compare_csv_files(
59+
project_asset: "ProjectAsset",
60+
expected_csv_path: Path,
61+
message: str,
62+
keys_to_ignore: list[str] | set[str] | None = None,
63+
is_gzip_file: bool = False,
64+
) -> None:
65+
"""Compare a CSV from a ProjectAsset with a plain CSV file.
66+
Supports gzipped CSV files when is_gzip_file=True.
67+
Raises AssertionError if differences are found.
68+
"""
69+
if is_gzip_file:
70+
with (
71+
project_asset.file.open("rb") as file,
72+
gzip.GzipFile(fileobj=file, mode="rb") as gz,
73+
io.TextIOWrapper(gz, encoding="utf-8") as text_stream,
74+
):
75+
actual_data = list(csv.DictReader(text_stream))
76+
else:
77+
with project_asset.file.open(mode="r") as file:
78+
actual_data = list(csv.DictReader(file))
79+
80+
with expected_csv_path.open(mode="r", newline="", encoding="utf-8") as file:
81+
expected_data = list(csv.DictReader(file))
82+
83+
if keys_to_ignore:
84+
actual_data = remove_object_keys(actual_data, keys_to_ignore)
85+
expected_data = remove_object_keys(expected_data, keys_to_ignore)
86+
87+
assert actual_data == expected_data, message
88+
89+
90+
def compare_geojson_files(
91+
project_asset: "ProjectAsset",
92+
expected_geojson_path: Path,
93+
message: str,
94+
keys_to_ignore: list[str] | set[str] | None = None,
95+
is_gzip_file: bool = False,
96+
) -> None:
97+
"""Compare a GeoJSON from a ProjectAsset with a plain GeoJSON file.
98+
Supports gzipped GeoJSON files when is_gzip_file=True.
99+
Raises AssertionError if differences are found.
100+
"""
101+
if is_gzip_file:
102+
with (
103+
project_asset.file.open("rb") as file,
104+
gzip.GzipFile(fileobj=file, mode="rb") as gz,
105+
io.TextIOWrapper(gz, encoding="utf-8") as text_stream,
106+
):
107+
actual_data = json.load(text_stream)
108+
else:
109+
with project_asset.file.open("r", encoding="utf-8") as file:
110+
actual_data = json.load(file)
111+
112+
with expected_geojson_path.open("r", encoding="utf-8") as file:
113+
expected_data = json.load(file)
114+
115+
if keys_to_ignore:
116+
actual_data = remove_object_keys(actual_data, keys_to_ignore)
117+
expected_data = remove_object_keys(expected_data, keys_to_ignore)
118+
119+
assert actual_data == expected_data, message

‎apps/project/tests/e2e_create_project_tile_map_service_test.py‎

Lines changed: 182 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
from django.db.models.signals import pre_save
99
from ulid import ULID
1010

11-
from apps.common.utils import remove_object_keys
11+
from apps.common.models import AssetTypeEnum
12+
from apps.common.utils import compare_csv_files, compare_geojson_files, remove_object_keys
1213
from apps.contributor.factories import ContributorUserFactory
1314
from apps.contributor.models import ContributorUserGroup
1415
from apps.mapping.firebase.pull import pull_results_from_firebase
@@ -19,7 +20,7 @@
1920
MappingSessionUserGroup,
2021
MappingSessionUserGroupTemp,
2122
)
22-
from apps.project.models import Organization, Project
23+
from apps.project.models import Organization, Project, ProjectAsset, ProjectAssetExportTypeEnum
2324
from apps.tutorial.models import Tutorial
2425
from apps.user.factories import UserFactory
2526
from main.tests import TestCase
@@ -619,3 +620,182 @@ def _test_project(self, projectKey: str, filename: str):
619620

620621
project.refresh_from_db()
621622
assert project.progress == test_data["expected_pulled_results_data"]["progress"]
623+
624+
# NOTE: EXPORTS TESTING
625+
aggregated_results_filename = (
626+
Path(settings.BASE_DIR) / test_data["expected_project_exports_data"]["aggregated_results"]
627+
)
628+
629+
# NOTE: Test AGGREGATED RESULTS
630+
aggregated_results_project_asset = ProjectAsset.objects.filter(
631+
project=project,
632+
type=AssetTypeEnum.EXPORT,
633+
export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS,
634+
).first()
635+
636+
if not aggregated_results_project_asset:
637+
raise AssertionError("Aggregated results project asset not found")
638+
639+
compare_csv_files(
640+
aggregated_results_project_asset,
641+
aggregated_results_filename,
642+
is_gzip_file=True,
643+
message="Difference found for aggregated results export file.",
644+
)
645+
646+
# NOTE: Test AGGREGATED RESULTS WITH GEOMETRY
647+
aggregated_results_with_geometry_project_asset = ProjectAsset.objects.filter(
648+
project=project,
649+
type=AssetTypeEnum.EXPORT,
650+
export_type=ProjectAssetExportTypeEnum.AGGREGATED_RESULTS_WITH_GEOMETRY,
651+
).first()
652+
653+
if not aggregated_results_with_geometry_project_asset:
654+
raise AssertionError("Aggregated results with geometry project asset not found")
655+
656+
compare_geojson_files(
657+
aggregated_results_with_geometry_project_asset,
658+
test_data["expected_project_exports_data"]["aggregated_results_with_geometry"],
659+
is_gzip_file=True,
660+
message="Difference found for aggregated results with geometry export file.",
661+
)
662+
663+
# NOTE: Test RESULTS
664+
results_project_asset = ProjectAsset.objects.filter(
665+
project=project,
666+
type=AssetTypeEnum.EXPORT,
667+
export_type=ProjectAssetExportTypeEnum.RESULTS,
668+
).first()
669+
670+
if not results_project_asset:
671+
raise AssertionError("Results project asset not found")
672+
673+
compare_csv_files(
674+
results_project_asset,
675+
test_data["expected_project_exports_data"]["results"],
676+
is_gzip_file=True,
677+
message="Difference found for results export file.",
678+
)
679+
# NOTE: Test HISTORY
680+
history_project_asset = ProjectAsset.objects.filter(
681+
project=project,
682+
type=AssetTypeEnum.EXPORT,
683+
export_type=ProjectAssetExportTypeEnum.HISTORY,
684+
).first()
685+
686+
if not history_project_asset:
687+
raise AssertionError("History project asset not found")
688+
689+
compare_csv_files(
690+
history_project_asset,
691+
test_data["expected_project_exports_data"]["history"],
692+
is_gzip_file=True,
693+
message="Difference found for history export file.",
694+
)
695+
696+
# NOTE: Test GROUPS
697+
groups_project_asset = ProjectAsset.objects.filter(
698+
project=project,
699+
type=AssetTypeEnum.EXPORT,
700+
export_type=ProjectAssetExportTypeEnum.GROUPS,
701+
).first()
702+
703+
if not groups_project_asset:
704+
raise AssertionError("Groups project asset not found")
705+
706+
compare_csv_files(
707+
groups_project_asset,
708+
test_data["expected_project_exports_data"]["groups"],
709+
is_gzip_file=True,
710+
message="Difference found for groups export file.",
711+
)
712+
713+
# NOTE: Test TASKS
714+
tasks_project_asset = ProjectAsset.objects.filter(
715+
project=project,
716+
type=AssetTypeEnum.EXPORT,
717+
export_type=ProjectAssetExportTypeEnum.TASKS,
718+
).first()
719+
720+
if not tasks_project_asset:
721+
raise AssertionError("Tasks project asset not found")
722+
723+
compare_csv_files(
724+
tasks_project_asset,
725+
test_data["expected_project_exports_data"]["tasks"],
726+
is_gzip_file=True,
727+
message="Difference found for tasks export file.",
728+
)
729+
730+
# NOTE: Test USERS
731+
users_project_asset = ProjectAsset.objects.filter(
732+
project=project,
733+
type=AssetTypeEnum.EXPORT,
734+
export_type=ProjectAssetExportTypeEnum.USERS,
735+
).first()
736+
737+
if not users_project_asset:
738+
raise AssertionError("Users project asset not found")
739+
740+
compare_csv_files(
741+
users_project_asset,
742+
test_data["expected_project_exports_data"]["users"],
743+
is_gzip_file=True,
744+
message="Difference found for users export file.",
745+
)
746+
747+
# NOTE: Test AREA OF INTEREST FILE
748+
aoi_project_asset = ProjectAsset.objects.filter(
749+
project=project,
750+
type=AssetTypeEnum.EXPORT,
751+
export_type=ProjectAssetExportTypeEnum.AREA_OF_INTEREST,
752+
).first()
753+
754+
if not aoi_project_asset:
755+
raise AssertionError("AOI Geometry project asset not found")
756+
757+
compare_geojson_files(
758+
aoi_project_asset,
759+
aoi_geometry_filename,
760+
message="Difference found for AOI Geometry export file.",
761+
)
762+
763+
# NOTE: TEST HOT TASKING MANAGER GEOMETRY
764+
if "hot_tasking_manager_aoi" in test_data["assets"]:
765+
htm_aoi_filename = Path(settings.BASE_DIR) / test_data["assets"]["hot_tasking_manager_aoi"]
766+
htm_aoi_project_asset = ProjectAsset.objects.filter(
767+
project=project,
768+
type=AssetTypeEnum.EXPORT,
769+
export_type=ProjectAssetExportTypeEnum.HOT_TASKING_MANAGER_GEOMETRIES,
770+
).first()
771+
772+
if not htm_aoi_project_asset:
773+
raise AssertionError("HTM AOI Geometry project asset not found")
774+
775+
compare_geojson_files(
776+
htm_aoi_project_asset,
777+
htm_aoi_filename,
778+
message="Difference found for HTM AOI Geometry export file.",
779+
)
780+
781+
# NOTE: TEST MODERATE TO HIGH AGREEMENT
782+
if "moderate_to_high_agreement" in test_data["expected_project_exports_data"]:
783+
moderate_to_high_agreement_filename = Path(
784+
settings.BASE_DIR,
785+
test_data["expected_project_exports_data"]["moderate_to_high_agreement"],
786+
)
787+
788+
moderate_to_high_agreement_project_asset = ProjectAsset.objects.filter(
789+
project=project,
790+
type=AssetTypeEnum.EXPORT,
791+
export_type=ProjectAssetExportTypeEnum.MODERATE_TO_HIGH_AGREEMENT_YES_MAYBE_GEOMETRIES,
792+
).first()
793+
794+
if not moderate_to_high_agreement_project_asset:
795+
raise AssertionError("Moderate to high agreement project asset not found")
796+
797+
compare_geojson_files(
798+
moderate_to_high_agreement_project_asset,
799+
moderate_to_high_agreement_filename,
800+
message="Difference found for moderate to high agreement export file.",
801+
)

‎assets‎

0 commit comments

Comments
 (0)