diff --git a/apps/project/graphql/queries.py b/apps/project/graphql/queries.py index 2c7dc611..a5584c57 100644 --- a/apps/project/graphql/queries.py +++ b/apps/project/graphql/queries.py @@ -1,3 +1,5 @@ +import logging + import strawberry import strawberry_django from django.db.models import QuerySet @@ -7,7 +9,14 @@ from apps.project.custom_options import get_custom_options from apps.project.graphql.inputs.inputs import ProjectNameInput +from apps.project.graphql.types.project_types.validate import ( + TestValidateAoiObjectsResponse, + TestValidateTaskingManagerProjectResponse, +) from apps.project.models import Organization, Project, ProjectAsset, ProjectTypeEnum +from project_types.base.project import ValidationException +from project_types.validate.project import ValidateProject +from utils import fields from utils.geo.raster_tile_server.config import RasterConfig, RasterTileServerNameEnum, RasterTileServerNameEnumWithoutCustom from utils.geo.vector_tile_server.config import VectorConfig, VectorTileServerNameEnum, VectorTileServerNameEnumWithoutCustom @@ -25,6 +34,8 @@ ProjectType, ) +logger = logging.getLogger(__name__) + def get_tile_servers() -> RasterTileServersType: def _get_raster_tile_server_type(enum: RasterTileServerNameEnumWithoutCustom): @@ -79,6 +90,73 @@ def default_custom_options(self, project_type: ProjectTypeEnum) -> list[CustomOp for item in custom_options ] + @strawberry.field(extensions=[IsAuthenticated()]) + def test_aoi_objects( + self, + project_id: strawberry.ID | None, + asset_id: strawberry.ID | None, + ohsome_filter: str | None, + ) -> TestValidateAoiObjectsResponse: + response = TestValidateAoiObjectsResponse( + project_id=project_id, + asset_id=asset_id, + ohsome_filter=ohsome_filter, + ) + + if project_id is None: + return response.generate_error("project_id is required to test aoi elements") + + if asset_id is None: + return response.generate_error("asset_id is required to test aoi elements") + + if ohsome_filter is None: + return response.generate_error("ohsome_filter is required to test aoi elements") + + try: + object_count = ValidateProject.test_ohsome_objects_from_aoi_asset( + project_id, + asset_id, + ohsome_filter, + ) + + response.object_count = object_count + return response + except ValidationException as e: + return response.generate_error(str(e)) + except Exception as e: + raise GraphQLError(str(e)) from e + + @strawberry.field(extensions=[IsAuthenticated()]) + def test_tasking_manager_project( + self, + hot_tm_id: fields.PydanticId | None, + ohsome_filter: str | None, + ) -> TestValidateTaskingManagerProjectResponse: + response = TestValidateTaskingManagerProjectResponse( + hot_tm_id=hot_tm_id, + ohsome_filter=ohsome_filter, + ) + + if hot_tm_id is None: + return response.generate_error("hot_tm_id is required to test HOT project aoi elements") + + if ohsome_filter is None: + return response.generate_error("ohsome_filter is required to test HOT project aoi elements") + + try: + object_count = ValidateProject.test_tasking_manager_project( + hot_tm_id, + ohsome_filter, + ) + + response.object_count = object_count + + return response + except ValidationException as e: + return response.generate_error(str(e)) + except Exception as e: + raise GraphQLError(str(e)) from e + tile_servers: RasterTileServersType = strawberry.field(resolver=get_tile_servers, extensions=[IsAuthenticated()]) # Private -------------------- diff --git a/apps/project/graphql/types/project_types/validate.py b/apps/project/graphql/types/project_types/validate.py index 7836440b..bd3e9a68 100644 --- a/apps/project/graphql/types/project_types/validate.py +++ b/apps/project/graphql/types/project_types/validate.py @@ -1,6 +1,7 @@ import strawberry from project_types.validate import project as validate_project +from utils import fields @strawberry.experimental.pydantic.type(model=validate_project.ValidateObjectSourceConfig, all_fields=True) @@ -9,3 +10,30 @@ class ValidateObjectSourceConfig: ... @strawberry.experimental.pydantic.type(model=validate_project.ValidateProjectProperty, all_fields=True) class ValidateProjectPropertyType: ... + + +DEFAULT_TEST_RESPONSE_ERROR_MESSAGE: str = "Something unexpected has occurred. Please contact an admin to fix this issue." + + +@strawberry.type +class ValidateTestAoiResponse: + ok: bool = True + error: str | None = None + object_count: int | None = None + ohsome_filter: str | None = None + + def generate_error(self, message: str = DEFAULT_TEST_RESPONSE_ERROR_MESSAGE): + self.ok = False + self.error = message + return self + + +@strawberry.type +class TestValidateAoiObjectsResponse(ValidateTestAoiResponse): + project_id: strawberry.ID | None = None + asset_id: strawberry.ID | None = None + + +@strawberry.type +class TestValidateTaskingManagerProjectResponse(ValidateTestAoiResponse): + hot_tm_id: fields.PydanticId | None = None diff --git a/apps/project/tests/e2e_create_validate_project_test.py b/apps/project/tests/e2e_create_validate_project_test.py index 904fc433..356a289b 100644 --- a/apps/project/tests/e2e_create_validate_project_test.py +++ b/apps/project/tests/e2e_create_validate_project_test.py @@ -294,6 +294,32 @@ class Mutation: } """ + class Query: + TEST_AOI_OBJECTS = """ + query TestAoiObjects($assetId: ID, $projectId: ID, $ohsomeFilter: String) { + testAoiObjects(assetId: $assetId, projectId: $projectId, ohsomeFilter: $ohsomeFilter) { + ok + error + objectCount + assetId + projectId + ohsomeFilter + } + } + """ + + TEST_TASKING_MANAGER_PROJECT = """ + query TestTaskingManagerProject($hotTmId: String, $ohsomeFilter: String) { + testTaskingManagerProject(hotTmId: $hotTmId, ohsomeFilter: $ohsomeFilter) { + ok + error + objectCount + hotTmId + ohsomeFilter + } + } + """ + @pytest.mark.vcr("assets/tests/projects/validate/cassette") def test_validate_project_e2e(self): # TODO(susilnem): Add more test with filters @@ -398,6 +424,15 @@ def _test_project(self, filename: str): assert aoi_response["ok"] aoi_id = aoi_response["result"]["id"] + # Test AOI objects + ohsomeFilter = "building=* and geometry:polygon" + test_aoi_objects_content = self.query_check( + self.Query.TEST_AOI_OBJECTS, + variables={"assetId": aoi_id, "projectId": project_id, "ohsomeFilter": ohsomeFilter}, + ) + test_aoi_objects_response = test_aoi_objects_content["data"]["testAoiObjects"] + assert test_aoi_objects_response["ok"] + # Update project update_project_data = test_data["update_project"] update_project_data["image"] = image_id diff --git a/assets b/assets index e0507bd4..41037a51 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit e0507bd48b875ece261750bbb32f126d2b43d9d2 +Subproject commit 41037a51ebbb0ba162d60c97adaa818f80598fe4 diff --git a/project_types/validate/api_calls.py b/project_types/validate/api_calls.py index 0b3f2d28..0c5bebee 100644 --- a/project_types/validate/api_calls.py +++ b/project_types/validate/api_calls.py @@ -8,6 +8,7 @@ from main.config import Config from main.logging import log_extra_response +from utils.fields import PydanticLongText logger = logging.getLogger(__name__) @@ -175,6 +176,40 @@ def remove_noise_and_add_user_info(json: dict[str, Any]) -> dict[str, Any]: return json +# fixme(frozenhelium): merge this function with `ohsome` and also add appropriate messages to raised exceptions +def get_object_count_from_ohsome(area: str, ohsome_filter: PydanticLongText) -> int | None: + url = Config.OHSOME_API_LINK + "elements/count" + data = {"bpolys": area, "filter": ohsome_filter} + + logger.info("Target: %s", url) + logger.info("Filter: %s", ohsome_filter) + + # fixme(frozenhelium): use httpx for proper timeout + response = requests.post(url, data=data, timeout=100) + if response.status_code != 200: + logger.warning( + "ohsome element count request failed: check for errors in filter or geometries", + extra=log_extra_response(response=response), + ) + raise ValidateApiCallError + logger.info("Query successful.") + + response_json = response.json() + results = response_json.get("result", None) + if results is None: + return None + + first_result = results[0] + if first_result is None: + return None + + value = first_result.get("value", None) + if value is None: + return None + + return int(value) + + def ohsome(request: dict[str, Any], area: str, properties: str | None = None) -> dict[str, Any]: """Request data from Ohsome API.""" url = Config.OHSOME_API_LINK + request["endpoint"] diff --git a/project_types/validate/project.py b/project_types/validate/project.py index 08cf2dff..71f0d27b 100644 --- a/project_types/validate/project.py +++ b/project_types/validate/project.py @@ -4,6 +4,7 @@ from typing import Any import requests +import strawberry from django.contrib.gis.geos import GEOSGeometry from django.core.files.base import ContentFile from django.db import models @@ -28,7 +29,7 @@ from main.config import Config from project_types.base import project as base_project from project_types.tile_map_service.base.project import create_json_dump -from project_types.validate.api_calls import ValidateApiCallError, ohsome +from project_types.validate.api_calls import ValidateApiCallError, get_object_count_from_ohsome, ohsome from utils import fields as custom_fields from utils.asset_types.models import AoiGeometryAssetProperty from utils.common import Grouping, clean_up_none_keys, to_groups @@ -129,6 +130,119 @@ class ValidateProject( def __init__(self, project: Project): super().__init__(project) + @staticmethod + def validate_geojson_aoi(geo_json: dict): # type: ignore[reportMissingTypeArgument] + try: + _, geometry_collection = convert_json_dict_to_geometry_collection(geo_json) + except Exception as e: + raise base_project.ValidationException( + "GeoJSON does not contain a valid feature collection of polygon or multi-polygon", + ) from e + + area_km2 = get_area_of_geometry(geometry_collection) + allowed_area = 20 + + if area_km2 > allowed_area: + raise base_project.ValidationException(f"Area for AOI Geometry must be less than {allowed_area} sq. km") + + return PydanticFeatureCollection.model_validate(geo_json) + + @staticmethod + def validate_object_count(count: int | None) -> int: + if count is None or count <= 0: + raise base_project.ValidationException( + "AOI does not contain objects from selected filter.", + ) + + allowed_count = 100000 + + if count > allowed_count: + raise base_project.ValidationException( + f"AOI contains more than 100,000 objects. -> {count}", + ) + + return count + + @staticmethod + def test_ohsome_objects_from_aoi_asset(project_id: strawberry.ID, asset_id: strawberry.ID, ohsome_filter: str) -> int: + aoi_asset = ( + ProjectAsset.usable_objects() + .filter( + id=asset_id, + type=ProjectAsset.Type.INPUT, + input_type=ProjectAssetInputTypeEnum.AOI_GEOMETRY, + project_id=project_id, + ) + .first() + ) + + if not aoi_asset: + raise Exception("Could not find AOI geometry asset") + + with aoi_asset.file.open() as aoi_file: + aoi_geojson = json.loads(aoi_file.read()) + + feature_collection = PydanticFeatureCollection.model_validate(aoi_geojson) + + try: + object_count = get_object_count_from_ohsome( + feature_collection.model_dump_json(), + ohsome_filter, + ) + except Exception as e: + raise base_project.ValidationException("Failed to get object_count from ohsome") from e + + return ValidateProject.validate_object_count(object_count) + + @staticmethod + def test_tasking_manager_project( + hot_tm_id: custom_fields.PydanticId, + ohsome_filter: str, + ) -> int: + hot_tm_url = f"{Config.HOT_TASKING_MANAGER_PROJECT_API_LINK}projects/{hot_tm_id}/queries/aoi/?as_file=false" + logger.info("Fetching AOI geojson on HOT from %s", hot_tm_url) + + # FIXME(frozenhelium): duplicated logic from _validate_tasking_manager + aoi_result = requests.get(hot_tm_url, timeout=500) + if aoi_result.status_code != 200: + raise base_project.ValidationException( + f"Failed to fetch AOI GeoJSON from HOT Tasking Manager for tm_id {hot_tm_id}", + ) + + logger.info("Successfully fetched AOI geojson from HOT for tm_id %s", hot_tm_id) + + try: + geometry_dict = aoi_result.json() + except Exception as e: + raise base_project.ValidationException("HOT Tasking Manager did not respond with a valid JSON") from e + + aoi_geojson = { + "type": "FeatureCollection", + "metadata": { + "hot_tm_project_id": hot_tm_id, + }, + "features": [ + { + "type": "Feature", + "geometry": geometry_dict, + "properties": { + "hot_tm_project_id": hot_tm_id, + }, + }, + ], + } + + feature_collection = ValidateProject.validate_geojson_aoi(aoi_geojson) + try: + object_count = get_object_count_from_ohsome( + feature_collection.model_dump_json(), + ohsome_filter, + ) + except Exception as e: + raise base_project.ValidationException("Failed to get object_count from ohsome") from e + + return ValidateProject.validate_object_count(object_count) + @typing.override def get_aoi_geometry_asset(self) -> ProjectAsset | None: if self.project_type_specifics.object_source.source_type != ValidateObjectSourceTypeEnum.AOI_GEOJSON_FILE: @@ -173,7 +287,13 @@ def _get_object_geometry_from_ohsome(self, geojson: dict): # type: ignore[repor ) from e try: - return convert_json_dict_to_features(geojson_result) + features = convert_json_dict_to_features(geojson_result) + + # FIXME(frozenhelium): verify if object count validation is required + object_count = len(features) + ValidateProject.validate_object_count(object_count) + + return features except Exception as e: raise base_project.ValidationException( "OHSOME did not respond with a valid feature collection of polygon or multi-polygon", @@ -191,6 +311,8 @@ def _validate_aoi_geojson_file(self): raise Exception("Could not find AOI geometry asset") asset_specific_data = AoiGeometryAssetProperty.model_validate(aoi_asset.asset_type_specifics) + + # FIXME(frozenhelium): reuse validate_aoi_geojson allowed_area = 20 if asset_specific_data.area > allowed_area: raise base_project.ValidationException(f"Area for AOI Geometry must be less than {allowed_area} sq. km") @@ -259,6 +381,8 @@ def _validate_object_geojson_url(self): self.project.centroid = hull_center self.project.save(update_fields=["aoi_geometry", "total_area", "bbox", "centroid"]) + # FIXME(frozenhelium): add validation for object count? + return features def _validate_tasking_manager(self): diff --git a/project_types/validate/tests/api_calls_test.py b/project_types/validate/tests/api_calls_test.py index 682f6823..9144497e 100644 --- a/project_types/validate/tests/api_calls_test.py +++ b/project_types/validate/tests/api_calls_test.py @@ -7,6 +7,7 @@ from main.tests import TestCase from project_types.validate.api_calls import ( ValidateApiCallError, + get_object_count_from_ohsome, ohsome, query_osm, query_osmcha, @@ -141,6 +142,37 @@ def test_query_osm(self, mock_retry_get): # type: ignore[reportMissingParameter with pytest.raises(ValidateApiCallError): query_osm([12345], {}) + @patch("requests.post") + def test_get_object_count_from_ohsome(self, mock_post): # type: ignore[reportMissingParameterType] + sample_filter = "building=* and geometry:polygon" + sample_area = "POLYGON((8.67 49.39,8.68 49.39,8.68 49.40,8.67 49.40,8.67 49.39))" + + sample_object_count = 500 + + mock_response_data = { + "attribution": { + "url": "https://ohsome.org/copyrights", + "text": "© OpenStreetMap contributors", + }, + "apiVersion": "1.10.4", + "result": [ + { + "timestamp": "2025-10-01T12:00:00Z", + "value": sample_object_count, + }, + ], + } + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_response_data + mock_post.return_value = mock_response + + object_count = get_object_count_from_ohsome(sample_area, sample_filter) + + mock_post.assert_called_once() + assert object_count == sample_object_count + @patch("requests.post") @patch("project_types.validate.api_calls.remove_noise_and_add_user_info") def test_ohsome(self, mock_remove_noise, mock_post): # type: ignore[reportMissingParameterType] diff --git a/schema.graphql b/schema.graphql index fc80f6f7..3a800743 100644 --- a/schema.graphql +++ b/schema.graphql @@ -2053,6 +2053,8 @@ type Query { publicOrganizations(filters: OrganizationFilter, order: OrganizationOrder, pagination: OffsetPaginationInput): OrganizationTypeOffsetPaginated! publicProject(id: ID!): ProjectType! publicProjects(filters: ProjectFilter, order: ProjectOrder, pagination: OffsetPaginationInput): ProjectTypeOffsetPaginated! + testAoiObjects(projectId: ID, assetId: ID, ohsomeFilter: String): TestValidateAoiObjectsResponse! @isAuthenticated + testTaskingManagerProject(hotTmId: String, ohsomeFilter: String): TestValidateTaskingManagerProjectResponse! @isAuthenticated tileServers: RasterTileServersType! @isAuthenticated tutorial(id: ID!): TutorialType! @isAuthenticated tutorialAsset(id: ID!): TutorialAssetType! @isAuthenticated @@ -2188,6 +2190,23 @@ type StreetTutorialTaskPropertyType { mapillaryImageId: String! } +type TestValidateAoiObjectsResponse { + assetId: ID + error: String + objectCount: Int + ohsomeFilter: String + ok: Boolean! + projectId: ID +} + +type TestValidateTaskingManagerProjectResponse { + error: String + hotTmId: String + objectCount: Int + ohsomeFilter: String + ok: Boolean! +} + """Model representing assets for a tutorial.""" input TutorialAssetCreateInput { clientId: String!