From 7e00957cb8935ca69b303a5cea456042cc8ec5cc Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Sun, 12 May 2024 15:20:24 +0200 Subject: [PATCH 01/10] feat(building-comparison): add metadata table --- .../building_comparison/indicator.py | 57 +++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/ohsome_quality_api/indicators/building_comparison/indicator.py b/ohsome_quality_api/indicators/building_comparison/indicator.py index 044d3ab18..981e47ef2 100644 --- a/ohsome_quality_api/indicators/building_comparison/indicator.py +++ b/ohsome_quality_api/indicators/building_comparison/indicator.py @@ -1,3 +1,4 @@ +import asyncio import logging import os from functools import cache @@ -6,7 +7,6 @@ import geojson import plotly.graph_objects as pgo import psycopg -import yaml from async_lru import alru_cache from dateutil import parser from geojson import Feature @@ -44,7 +44,11 @@ def __init__( self.ratio: dict[str, float | None] = {} self.warnings: dict[str, str | None] = {} # self.data_ref: list = load_reference_datasets() # reference datasets - for key, val in load_datasets_metadata().items(): + asyncio.run(self.init_async_data()) + + async def init_async_data(self): + datasets_metadata = await load_datasets_metadata() + for key, val in datasets_metadata.items(): self.data_ref[key] = val self.area_osm[key] = None # osm building area self.area_ref[key] = None # reference building area [sqkm] @@ -55,7 +59,7 @@ def __init__( async def coverage(cls, inverse=False) -> list[Feature]: # TODO: could also return a Feature Collection features = [] - datasets = load_datasets_metadata() + datasets = await load_datasets_metadata() for val in datasets.values(): if inverse: table = val["coverage"]["inversed"] @@ -279,11 +283,11 @@ def check_minor_edge_cases(self, dataset: str) -> str: def format_sources(self): sources = [] - for dataset in self.data_ref.values(): - if dataset["link"] is not None: - sources.append(f"" f"{dataset['name']}") + for dataset_key, dataset_value in self.data_ref.items(): + if dataset_value["link"] is not None: + sources.append(f"{dataset_key}") else: - sources.append(f"{dataset}") + sources.append(dataset_key) result = ", ".join(sources) return result @@ -313,7 +317,38 @@ async def get_reference_building_area(feature_str: str, table_name: str) -> floa @cache -def load_datasets_metadata() -> dict: - file_path = os.path.join(os.path.dirname(__file__), "datasets.yaml") - with open(file_path, "r") as f: - return yaml.safe_load(f) +async def load_datasets_metadata() -> dict: + """Load dataset metadata from the database.""" + dns = "postgres://{user}:{password}@{host}:{port}/{database}".format( + host=get_config_value("postgres_host"), + port=get_config_value("postgres_port"), + database=get_config_value("postgres_db"), + user=get_config_value("postgres_user"), + password=get_config_value("postgres_password"), + ) + + dataset_metadata = {} + + async with await psycopg.AsyncConnection.connect(dns) as con: + async with con.cursor() as cur: + await cur.execute("SELECT * FROM building_comparison_metadata") + async for row in cur: + dataset_name = row[0] + link = row[1] + date = row[2].strftime("%Y-%m-%d") # Convert date object to string + description = row[3] + color = row[4] + table_name = row[5] + coverage_simple = row[6] + coverage_inversed = row[7] + dataset_metadata[dataset_name] = { + "link": link, + "date": date, + "description": description, + "color": color, + "table_name": table_name, + "coverage_simple": coverage_simple, + "coverage_inversed": coverage_inversed, + } + + return dataset_metadata From 6d1fdd3a9af79dd95c2915e0b41facea2899e9e0 Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Mon, 13 May 2024 13:24:09 +0200 Subject: [PATCH 02/10] feat: add init function to base indicator class --- ohsome_quality_api/indicators/base.py | 8 ++++++++ ohsome_quality_api/oqt.py | 2 ++ 2 files changed, 10 insertions(+) diff --git a/ohsome_quality_api/indicators/base.py b/ohsome_quality_api/indicators/base.py index ee0b02f82..6703bae69 100644 --- a/ohsome_quality_api/indicators/base.py +++ b/ohsome_quality_api/indicators/base.py @@ -123,6 +123,14 @@ async def coverage(cls, inverse=False) -> list[Feature]: else: return [Feature(Polygon(coordinates=[]))] + @abstractmethod + async def init(self) -> None: + """Initialize the indicator. + + This method should be used to initialize data. + """ + pass + @abstractmethod async def preprocess(self) -> None: """Get fetch and preprocess data. diff --git a/ohsome_quality_api/oqt.py b/ohsome_quality_api/oqt.py index b93043da9..1c9c6e8f2 100644 --- a/ohsome_quality_api/oqt.py +++ b/ohsome_quality_api/oqt.py @@ -77,6 +77,8 @@ async def _create_indicator( indicator_class = get_class_from_key(class_type="indicator", key=key) indicator = indicator_class(topic, feature) + logging.info("Run initialization") + indicator.init() logging.info("Run preprocessing") await indicator.preprocess() logging.info("Run calculation") From faa8ad69c9dee0dc84247c647c1172681080df88 Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Mon, 13 May 2024 13:28:51 +0200 Subject: [PATCH 03/10] feat(building-comparison): add init function to building-comparison indicator --- .../building_comparison/indicator.py | 52 +++++++++---------- .../indicators/test_building_comparison.py | 14 +++++ 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/ohsome_quality_api/indicators/building_comparison/indicator.py b/ohsome_quality_api/indicators/building_comparison/indicator.py index 981e47ef2..7e0e63a9b 100644 --- a/ohsome_quality_api/indicators/building_comparison/indicator.py +++ b/ohsome_quality_api/indicators/building_comparison/indicator.py @@ -1,7 +1,5 @@ -import asyncio import logging import os -from functools import cache from string import Template import geojson @@ -11,6 +9,7 @@ from dateutil import parser from geojson import Feature from numpy import mean +from shapely import wkb from ohsome_quality_api.config import get_config_value from ohsome_quality_api.definitions import Color, get_attribution @@ -44,16 +43,6 @@ def __init__( self.ratio: dict[str, float | None] = {} self.warnings: dict[str, str | None] = {} # self.data_ref: list = load_reference_datasets() # reference datasets - asyncio.run(self.init_async_data()) - - async def init_async_data(self): - datasets_metadata = await load_datasets_metadata() - for key, val in datasets_metadata.items(): - self.data_ref[key] = val - self.area_osm[key] = None # osm building area - self.area_ref[key] = None # reference building area [sqkm] - self.area_cov[key] = None # covered area [%] - self.ratio[key] = None @classmethod async def coverage(cls, inverse=False) -> list[Feature]: @@ -62,11 +51,11 @@ async def coverage(cls, inverse=False) -> list[Feature]: datasets = await load_datasets_metadata() for val in datasets.values(): if inverse: - table = val["coverage"]["inversed"] + table = val["coverage_inversed"] else: - table = val["coverage"]["simple"] + table = val["coverage_simple"] feature = await db_client.get_reference_coverage(table) - feature.properties.update({"refernce_dataset": val["name"]}) + feature.properties.update({"refernce_dataset": val["table_name"]}) features.append(feature) return features @@ -74,12 +63,21 @@ async def coverage(cls, inverse=False) -> list[Feature]: def attribution(cls) -> str: return get_attribution(["OSM", "EUBUCCO", "Microsoft Buildings"]) + async def init(self) -> None: + datasets_metadata = await load_datasets_metadata() + for key, val in datasets_metadata.items(): + self.data_ref[key] = val + self.area_osm[key] = None # osm building area + self.area_ref[key] = None # reference building area [sqkm] + self.area_cov[key] = None # covered area [%] + self.ratio[key] = None + async def preprocess(self) -> None: for key, val in self.data_ref.items(): # get coverage [%] self.area_cov[key] = await db_client.get_intersection_area( self.feature, - val["coverage"]["simple"], + val["coverage_simple"], ) self.warnings[key] = self.check_major_edge_cases(key) if self.warnings[key] != "": @@ -88,7 +86,7 @@ async def preprocess(self) -> None: # clip input geom with coverage of reference dataset feature = await db_client.get_intersection_geom( self.feature, - val["coverage"]["simple"], + val["coverage_simple"], ) # get reference building area @@ -129,14 +127,15 @@ def calculate(self) -> None: self.warnings[key] += template.substitute( ratio=round(self.ratio[key] * 100, 2), coverage=round(self.area_cov[key] * 100, 2), - dataset=self.data_ref[key]["name"], + dataset=self.data_ref[key]["table_name"], ) except ZeroDivisionError: self.ratio[key] = None self.warnings[key] += ( - f"Warning: Reference dataset {self.data_ref[key]['name']} covers " - f"AoI with {round(self.area_cov[key] * 100, 2)}%, but has no " - "building area. No quality estimation with reference is possible. " + f"Warning: Reference dataset {self.data_ref[key]['table_name']}" + f" covers AoI with {round(self.area_cov[key] * 100, 2)}%, but has" + f" no building area. No quality estimation with reference " + f"is possible. " ) self.result.description += self.warnings[key] + "\n" @@ -184,12 +183,12 @@ def create_figure(self) -> None: for key, dataset in self.data_ref.items(): if None in (self.area_ref[key], self.area_osm[key]): continue - ref_x.append(dataset["name"]) + ref_x.append(dataset["table_name"]) ref_y.append(round(self.area_ref[key], 2)) ref_data.append(dataset) - osm_x.append(dataset["name"]) + osm_x.append(dataset["table_name"]) osm_y.append(round(self.area_osm[key], 2)) - ref_hover.append(f"{dataset['name']} ({dataset['date']})") + ref_hover.append(f"{dataset['table_name']} ({dataset['date']})") osm_hover.append(f"OSM ({self.result.timestamp_osm:%b %d, %Y})") ref_color.append(Color[dataset["color"]].value) osm_area.append(round(self.area_osm[key], 2)) @@ -316,7 +315,6 @@ async def get_reference_building_area(feature_str: str, table_name: str) -> floa return res[0] or 0.0 -@cache async def load_datasets_metadata() -> dict: """Load dataset metadata from the database.""" dns = "postgres://{user}:{password}@{host}:{port}/{database}".format( @@ -339,8 +337,8 @@ async def load_datasets_metadata() -> dict: description = row[3] color = row[4] table_name = row[5] - coverage_simple = row[6] - coverage_inversed = row[7] + coverage_simple = wkb.loads(bytes.fromhex(row[6])) + coverage_inversed = wkb.loads(bytes.fromhex(row[7])) dataset_metadata[dataset_name] = { "link": link, "date": date, diff --git a/tests/integrationtests/indicators/test_building_comparison.py b/tests/integrationtests/indicators/test_building_comparison.py index 83fd92c2d..3fc21b133 100644 --- a/tests/integrationtests/indicators/test_building_comparison.py +++ b/tests/integrationtests/indicators/test_building_comparison.py @@ -112,6 +112,7 @@ def test_init(self, topic_building_area, feature_germany_berlin): def test_get_sources(self, topic_building_area, feature_germany_berlin): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) source = indicator.format_sources() assert "EUBUCCO" in source @@ -130,6 +131,7 @@ class TestPreprocess: ) def test_preprocess(self, topic_building_area, feature_germany_berlin): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) for area in indicator.area_osm.values(): @@ -147,6 +149,7 @@ def test_preprocess_no_intersection( mock_get_intersection_area_none, ): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) for area in indicator.area_cov.values(): @@ -166,6 +169,7 @@ def test_preprocess_some_intersection( feature_germany_berlin, ): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) assert 1.0 in list(indicator.area_cov.values()) @@ -187,6 +191,7 @@ def test_calculate( feature_germany_berlin, ): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is not None @@ -206,6 +211,7 @@ def test_calculate_reference_area_0( feature_germany_heidelberg, ): indicator = BuildingComparison(topic_building_area, feature_germany_heidelberg) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is None @@ -222,6 +228,7 @@ def test_calculate_above_one_th( feature_germany_heidelberg, ): indicator = BuildingComparison(topic_building_area, feature_germany_heidelberg) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is None @@ -246,6 +253,7 @@ def test_calculate_no_intersection( feature_germany_heidelberg, ): indicator = BuildingComparison(topic_building_area, feature_germany_heidelberg) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is None @@ -269,6 +277,7 @@ def test_calculate_some_intersection( feature_germany_heidelberg, ): indicator = BuildingComparison(topic_building_area, feature_germany_heidelberg) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is not None @@ -299,6 +308,7 @@ def test_calculate_above_one_th_and_expected( feature_germany_heidelberg, ): indicator = BuildingComparison(topic_building_area, feature_germany_heidelberg) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is not None @@ -328,6 +338,7 @@ class TestFigure: ) def test_create_figure(self, topic_building_area, feature_germany_berlin): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() indicator.create_figure() @@ -343,6 +354,7 @@ def test_create_figure(self, topic_building_area, feature_germany_berlin): ) def test_create_figure_manual(self, topic_building_area, feature_germany_berlin): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() indicator.create_figure() @@ -360,6 +372,7 @@ def test_create_figure_above_one_th( feature_germany_berlin, ): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() indicator.create_figure() @@ -379,6 +392,7 @@ def test_create_figure_building_area_zero( feature_germany_berlin, ): indicator = BuildingComparison(topic_building_area, feature_germany_berlin) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() indicator.create_figure() From 710fed5c812a308e9e276cdb7623d13d8f0ea676 Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Mon, 13 May 2024 13:46:18 +0200 Subject: [PATCH 04/10] fix(integrationtests): fix mocking for tests --- .../indicators/test_building_comparison.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integrationtests/indicators/test_building_comparison.py b/tests/integrationtests/indicators/test_building_comparison.py index 3fc21b133..82c474c08 100644 --- a/tests/integrationtests/indicators/test_building_comparison.py +++ b/tests/integrationtests/indicators/test_building_comparison.py @@ -15,7 +15,8 @@ @pytest.fixture def mock_get_building_area(class_mocker): async def side_effect_function(*args, **kwargs): - if args[1] == "EUBUCCO": + if not hasattr(side_effect_function, "called"): + side_effect_function.called = True return 6000000.791645115 else: return 5000000.791645115 @@ -39,7 +40,8 @@ def mock_get_building_area_low(class_mocker): @pytest.fixture def mock_get_building_area_low_some(class_mocker): async def side_effect_function(*args, **kwargs): - if args[1] == "EUBUCCO": + if not hasattr(side_effect_function, "called"): + side_effect_function.called = True return 1 else: return 6000000.791645115 @@ -90,7 +92,8 @@ def mock_get_intersection_area_none(class_mocker): @pytest.fixture def mock_get_intersection_area_some(class_mocker): async def side_effect(*args, **kwargs): - if "eubucco" in args[1]: + if not hasattr(side_effect, "called"): + side_effect.called = True return 0.0 # 0 % else: return 1.0 # 100 % From 3c9c833f8cb48d5d978bb72cc2164854f9f8055c Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Mon, 13 May 2024 17:46:43 +0200 Subject: [PATCH 05/10] fix: coverage functions now work with metadata table --- ohsome_quality_api/geodatabase/client.py | 8 ++++---- .../geodatabase/select_coverage.sql | 8 ++++++-- .../geodatabase/select_intersection.sql | 16 +++++++++++----- .../indicators/building_comparison/indicator.py | 12 +++++++----- ohsome_quality_api/oqt.py | 2 +- 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/ohsome_quality_api/geodatabase/client.py b/ohsome_quality_api/geodatabase/client.py index ac08362a0..7f7b26551 100644 --- a/ohsome_quality_api/geodatabase/client.py +++ b/ohsome_quality_api/geodatabase/client.py @@ -70,13 +70,13 @@ async def get_shdi(bpoly: Feature | FeatureCollection) -> list[Record]: # TODO: Check calls of the function -async def get_reference_coverage(table_name: str) -> Feature: +async def get_reference_coverage(table_name: str, coverage_type: str) -> Feature: """Get reference coverage for a bounding polygon.""" file_path = os.path.join(WORKING_DIR, "select_coverage.sql") with open(file_path, "r") as file: query = file.read() async with get_connection() as conn: - result = await conn.fetch(query.format(table_name=table_name)) + result = await conn.fetchrow(query, coverage_type, table_name) return Feature(geometry=geojson.loads(result[0]["geom"])) @@ -91,7 +91,7 @@ async def get_intersection_area(bpoly: Feature, table_name: str) -> float: query = file.read() geom = str(bpoly.geometry) async with get_connection() as conn: - result = await conn.fetch(query.format(table_name=table_name), geom) + result = await conn.fetch(query, geom, table_name) if result: return result[0]["area_ratio"] else: @@ -105,7 +105,7 @@ async def get_intersection_geom(bpoly: Feature, table_name: str) -> Feature: query = file.read() geom = str(bpoly.geometry) async with get_connection() as conn: - result = await conn.fetch(query.format(table_name=table_name), geom) + result = await conn.fetch(query, geom, table_name) if result: return Feature(geometry=geojson.loads(result[0]["geom"])) else: diff --git a/ohsome_quality_api/geodatabase/select_coverage.sql b/ohsome_quality_api/geodatabase/select_coverage.sql index adb0454ce..33186cb79 100644 --- a/ohsome_quality_api/geodatabase/select_coverage.sql +++ b/ohsome_quality_api/geodatabase/select_coverage.sql @@ -1,4 +1,8 @@ SELECT - ST_AsGeoJSON (ST_Transform (geom, 4326)) as geom + ST_AsGeoJSON (ST_Transform ($1, 4326)) as geom FROM - {table_name}; + building_comparison_metadata +WHERE + name like $2; + + diff --git a/ohsome_quality_api/geodatabase/select_intersection.sql b/ohsome_quality_api/geodatabase/select_intersection.sql index cca64f558..c3388347b 100644 --- a/ohsome_quality_api/geodatabase/select_intersection.sql +++ b/ohsome_quality_api/geodatabase/select_intersection.sql @@ -1,13 +1,19 @@ WITH bpoly AS ( SELECT - ST_Setsrid (ST_GeomFromGeoJSON ($1), 4326) AS geom + ST_SetSRID(ST_GeomFromGeoJSON($1), 4326) AS geom +), +selected_coverage AS ( + SELECT coverage_simple AS coverage + FROM building_comparison_metadata bcm + WHERE name LIKE $2 ) SELECT -- ratio of area within coverage (empty if outside, between 0-1 if intersection) - ST_Area (ST_Intersection (bpoly.geom, coverage.geom)) / ST_Area (bpoly.geom) as area_ratio, - ST_AsGeoJSON (ST_Intersection (bpoly.geom, coverage.geom)) AS geom + ST_Area(ST_Intersection(bpoly.geom, selected_coverage.coverage)) / ST_Area(bpoly.geom) AS area_ratio, + ST_AsGeoJSON(ST_Intersection(bpoly.geom, selected_coverage.coverage)) AS geom FROM bpoly, - {table_name} coverage + selected_coverage WHERE - ST_Intersects (bpoly.geom, coverage.geom) + ST_Intersects(bpoly.geom, selected_coverage.coverage); + diff --git a/ohsome_quality_api/indicators/building_comparison/indicator.py b/ohsome_quality_api/indicators/building_comparison/indicator.py index 7e0e63a9b..5f6300b44 100644 --- a/ohsome_quality_api/indicators/building_comparison/indicator.py +++ b/ohsome_quality_api/indicators/building_comparison/indicator.py @@ -51,10 +51,12 @@ async def coverage(cls, inverse=False) -> list[Feature]: datasets = await load_datasets_metadata() for val in datasets.values(): if inverse: - table = val["coverage_inversed"] + coverage_type = "coverage_inversed" else: - table = val["coverage_simple"] - feature = await db_client.get_reference_coverage(table) + coverage_type = "coverage_simple" + feature = await db_client.get_reference_coverage( + val["table_name"], coverage_type + ) feature.properties.update({"refernce_dataset": val["table_name"]}) features.append(feature) return features @@ -77,7 +79,7 @@ async def preprocess(self) -> None: # get coverage [%] self.area_cov[key] = await db_client.get_intersection_area( self.feature, - val["coverage_simple"], + val["table_name"], ) self.warnings[key] = self.check_major_edge_cases(key) if self.warnings[key] != "": @@ -86,7 +88,7 @@ async def preprocess(self) -> None: # clip input geom with coverage of reference dataset feature = await db_client.get_intersection_geom( self.feature, - val["coverage_simple"], + val["table_name"], ) # get reference building area diff --git a/ohsome_quality_api/oqt.py b/ohsome_quality_api/oqt.py index 1c9c6e8f2..0bd85190c 100644 --- a/ohsome_quality_api/oqt.py +++ b/ohsome_quality_api/oqt.py @@ -78,7 +78,7 @@ async def _create_indicator( indicator = indicator_class(topic, feature) logging.info("Run initialization") - indicator.init() + await indicator.init() logging.info("Run preprocessing") await indicator.preprocess() logging.info("Run calculation") From e801983f8ad2e19bd87b29e2b4cf983abbd1c814 Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Tue, 14 May 2024 14:18:48 +0200 Subject: [PATCH 06/10] feat(road-comparison): use metadata table --- .../building_comparison/indicator.py | 11 +-- .../indicators/road_comparison/indicator.py | 96 +++++++++++++------ .../indicators/test_road_comparison.py | 9 ++ 3 files changed, 80 insertions(+), 36 deletions(-) diff --git a/ohsome_quality_api/indicators/building_comparison/indicator.py b/ohsome_quality_api/indicators/building_comparison/indicator.py index 5f6300b44..7f649f8d6 100644 --- a/ohsome_quality_api/indicators/building_comparison/indicator.py +++ b/ohsome_quality_api/indicators/building_comparison/indicator.py @@ -9,7 +9,6 @@ from dateutil import parser from geojson import Feature from numpy import mean -from shapely import wkb from ohsome_quality_api.config import get_config_value from ohsome_quality_api.definitions import Color, get_attribution @@ -331,7 +330,11 @@ async def load_datasets_metadata() -> dict: async with await psycopg.AsyncConnection.connect(dns) as con: async with con.cursor() as cur: - await cur.execute("SELECT * FROM building_comparison_metadata") + await cur.execute( + "SELECT * " + "FROM building_comparison_metadata " + "WHERE indicator = 'building-comparison';" + ) async for row in cur: dataset_name = row[0] link = row[1] @@ -339,16 +342,12 @@ async def load_datasets_metadata() -> dict: description = row[3] color = row[4] table_name = row[5] - coverage_simple = wkb.loads(bytes.fromhex(row[6])) - coverage_inversed = wkb.loads(bytes.fromhex(row[7])) dataset_metadata[dataset_name] = { "link": link, "date": date, "description": description, "color": color, "table_name": table_name, - "coverage_simple": coverage_simple, - "coverage_inversed": coverage_inversed, } return dataset_metadata diff --git a/ohsome_quality_api/indicators/road_comparison/indicator.py b/ohsome_quality_api/indicators/road_comparison/indicator.py index 65d57d8a4..a841c2bdc 100644 --- a/ohsome_quality_api/indicators/road_comparison/indicator.py +++ b/ohsome_quality_api/indicators/road_comparison/indicator.py @@ -4,7 +4,6 @@ import geojson import plotly.graph_objects as pgo import psycopg -import yaml from async_lru import alru_cache from geojson import Feature from numpy import mean @@ -44,27 +43,21 @@ def __init__( self.ratio: dict[str, float | None] = {} self.warnings: dict[str, str | None] = {} # self.data_ref: list = load_reference_datasets() # reference datasets - for key, val in load_datasets_metadata().items(): - self.data_ref[key] = val - self.area_cov[key] = None # covered area [%] - self.length_matched[key] = None - self.length_total[key] = None - self.length_osm[key] = None - self.ratio[key] = None - self.warnings[key] = None @classmethod async def coverage(cls, inverse=False) -> list[Feature]: # TODO: could also return a Feature Collection features = [] - datasets = load_datasets_metadata() + datasets = await load_datasets_metadata() for val in datasets.values(): if inverse: - table = val["coverage"]["inversed"] + coverage_type = "coverage_inversed" else: - table = val["coverage"]["simple"] - feature = await db_client.get_reference_coverage(table) - feature.properties.update({"refernce_dataset": val["name"]}) + coverage_type = "coverage_simple" + feature = await db_client.get_reference_coverage( + val["table_name"], coverage_type + ) + feature.properties.update({"refernce_dataset": val["table_name"]}) features.append(feature) return features @@ -73,12 +66,23 @@ def attribution(cls) -> str: # TODO: add attribution return get_attribution(["OSM"]) + async def init(self) -> None: + dataset_metadata = await load_datasets_metadata() + for key, val in dataset_metadata.items(): + self.data_ref[key] = val + self.area_cov[key] = None # covered area [%] + self.length_matched[key] = None + self.length_total[key] = None + self.length_osm[key] = None + self.ratio[key] = None + self.warnings[key] = None + async def preprocess(self) -> None: for key, val in self.data_ref.items(): # get area covered by reference dataset [%] self.area_cov[key] = await db_client.get_intersection_area( self.feature, - val["coverage"]["simple"], + val["table_name"], ) self.warnings[key] = self.check_major_edge_cases(key) if self.warnings[key] != "": @@ -87,7 +91,7 @@ async def preprocess(self) -> None: # clip input geom with coverage of reference dataset feature = await db_client.get_intersection_geom( self.feature, - val["coverage"]["simple"], + val["table_name"], ) # get covered road length @@ -124,14 +128,15 @@ def calculate(self) -> None: except ZeroDivisionError: self.ratio[key] = None self.warnings[key] += ( - f"Warning: Reference dataset {self.data_ref[key]['name']} covers " - f"AOI with {round(self.area_cov[key] * 100, 2)}%, but has no " - "road length. No quality estimation with reference is possible. " + f"Warning: Reference dataset {self.data_ref[key]['table_name']}" + f" covers AOI with {round(self.area_cov[key] * 100, 2)}%, but " + f"has no road length. No quality estimation with reference is" + f" possible. " ) self.result.description += self.warnings[key] + "\n" self.result.description += ( - f"{self.data_ref[key]['name']} has a road length of " + f"{self.data_ref[key]['table_name']} has a road length of " f"{(self.length_total[key]/1000):.2f} km, of which " f"{(self.length_matched[key]/1000):.2f} km are covered by roads in " f"OSM. " @@ -175,9 +180,9 @@ def create_figure(self) -> None: for key, val in self.ratio.items(): if val is None: continue - ref_name.append(self.data_ref[key]["name"]) + ref_name.append(self.data_ref[key]["table_name"]) ref_color.append(Color[self.data_ref[key]["color"]].value) - ref_processingdate.append(self.data_ref[key]["processing_date"]) + ref_processingdate.append(self.data_ref[key]["date"]) ref_ratio.append(val) for i, (name, ratio, date) in enumerate( @@ -271,11 +276,11 @@ def check_minor_edge_cases(self, dataset: str) -> str: def format_sources(self): sources = [] - for dataset in self.data_ref.values(): - if dataset["link"] is not None: - sources.append(f"" f"{dataset['name']}") + for dataset_key, dataset_value in self.data_ref.items(): + if dataset_value["link"] is not None: + sources.append(f"{dataset_key}") else: - sources.append(f"{dataset}") + sources.append(dataset_key) result = ", ".join(sources) return result @@ -311,7 +316,38 @@ async def get_matched_roadlengths( return res[0], res[1] -def load_datasets_metadata() -> dict: - file_path = os.path.join(os.path.dirname(__file__), "datasets.yaml") - with open(file_path, "r") as f: - return yaml.safe_load(f) +async def load_datasets_metadata() -> dict: + """Load dataset metadata from the database.""" + dns = "postgres://{user}:{password}@{host}:{port}/{database}".format( + host=get_config_value("postgres_host"), + port=get_config_value("postgres_port"), + database=get_config_value("postgres_db"), + user=get_config_value("postgres_user"), + password=get_config_value("postgres_password"), + ) + + dataset_metadata = {} + + async with await psycopg.AsyncConnection.connect(dns) as con: + async with con.cursor() as cur: + await cur.execute( + "SELECT * " + "FROM building_comparison_metadata " + "WHERE indicator = 'road-comparison';" + ) + async for row in cur: + dataset_name = row[0] + link = row[1] + date = row[2].strftime("%Y-%m-%d") # Convert date object to string + description = row[3] + color = row[4] + table_name = row[5] + dataset_metadata[dataset_name] = { + "link": link, + "date": date, + "description": description, + "color": color, + "table_name": table_name, + } + + return dataset_metadata diff --git a/tests/integrationtests/indicators/test_road_comparison.py b/tests/integrationtests/indicators/test_road_comparison.py index 214902ccd..6b2a1a120 100644 --- a/tests/integrationtests/indicators/test_road_comparison.py +++ b/tests/integrationtests/indicators/test_road_comparison.py @@ -81,6 +81,7 @@ def test_init(self, topic_major_roads_length, feature_malta): def test_get_sources(self, topic_major_roads_length, feature_malta): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) source = indicator.format_sources() assert ( "" @@ -102,6 +103,7 @@ class TestPreprocess: ) def test_preprocess(self, topic_major_roads_length, feature_malta): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) for length in indicator.length_total.values(): @@ -116,6 +118,7 @@ def test_preprocess_no_intersection( mock_get_intersection_area_none, ): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) for area in indicator.area_cov.values(): @@ -137,6 +140,7 @@ def test_calculate( feature_malta, ): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is not None @@ -156,6 +160,7 @@ def test_calculate_reference_lenght_0( feature_malta, ): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is None @@ -174,6 +179,7 @@ def test_calculate_no_intersection( feature_malta, ): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() assert indicator.result.value is None @@ -195,6 +201,7 @@ class TestFigure: ) def test_create_figure(self, topic_major_roads_length, feature_malta): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() indicator.create_figure() @@ -210,6 +217,7 @@ def test_create_figure(self, topic_major_roads_length, feature_malta): ) def test_create_figure_manual(self, topic_major_roads_length, feature_malta): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() indicator.create_figure() @@ -225,6 +233,7 @@ def test_create_figure_building_area_zero( self, topic_major_roads_length, feature_malta ): indicator = RoadComparison(topic_major_roads_length, feature_malta) + asyncio.run(indicator.init()) asyncio.run(indicator.preprocess()) indicator.calculate() indicator.create_figure() From 6868afdbc1a1c5d468c1bb3f23ccf874d783bfd2 Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Thu, 16 May 2024 16:26:50 +0200 Subject: [PATCH 07/10] fix tests for new init function and comparison indicators --- .../geodatabase/select_coverage.sql | 4 +-- .../geodatabase/select_intersection.sql | 4 +-- .../attribute_completeness/indicator.py | 3 ++ .../building_comparison/indicator.py | 35 +++++++++++-------- .../building_completeness/indicator.py | 3 ++ .../indicators/currentness/indicator.py | 3 ++ .../indicators/density/indicator.py | 3 ++ .../mapping_saturation/indicator.py | 3 ++ .../indicators/minimal/indicator.py | 3 ++ .../indicators/road_comparison/indicator.py | 31 +++++++++------- ohsome_quality_api/oqt.py | 2 +- 11 files changed, 61 insertions(+), 33 deletions(-) diff --git a/ohsome_quality_api/geodatabase/select_coverage.sql b/ohsome_quality_api/geodatabase/select_coverage.sql index 33186cb79..23cd5365a 100644 --- a/ohsome_quality_api/geodatabase/select_coverage.sql +++ b/ohsome_quality_api/geodatabase/select_coverage.sql @@ -1,8 +1,8 @@ SELECT ST_AsGeoJSON (ST_Transform ($1, 4326)) as geom FROM - building_comparison_metadata + comparison_indicators_metadata WHERE - name like $2; + dataset_name_snake_case like $2; diff --git a/ohsome_quality_api/geodatabase/select_intersection.sql b/ohsome_quality_api/geodatabase/select_intersection.sql index c3388347b..377932c71 100644 --- a/ohsome_quality_api/geodatabase/select_intersection.sql +++ b/ohsome_quality_api/geodatabase/select_intersection.sql @@ -4,8 +4,8 @@ WITH bpoly AS ( ), selected_coverage AS ( SELECT coverage_simple AS coverage - FROM building_comparison_metadata bcm - WHERE name LIKE $2 + FROM comparison_indicators_metadata + WHERE dataset_name_snake_case LIKE $2 ) SELECT -- ratio of area within coverage (empty if outside, between 0-1 if intersection) diff --git a/ohsome_quality_api/indicators/attribute_completeness/indicator.py b/ohsome_quality_api/indicators/attribute_completeness/indicator.py index cc615e199..137bc6ee7 100644 --- a/ohsome_quality_api/indicators/attribute_completeness/indicator.py +++ b/ohsome_quality_api/indicators/attribute_completeness/indicator.py @@ -41,6 +41,9 @@ def __init__(self, topic: Topic, feature: Feature, attribute_key: str) -> None: self.absolute_value_1 = None self.absolute_value_2 = None + async def init(self) -> None: + pass + async def preprocess(self) -> None: attribute = get_attribute(self.topic.key, self.attribute_key) # Get attribute filter diff --git a/ohsome_quality_api/indicators/building_comparison/indicator.py b/ohsome_quality_api/indicators/building_comparison/indicator.py index f18be42e2..62860969c 100644 --- a/ohsome_quality_api/indicators/building_comparison/indicator.py +++ b/ohsome_quality_api/indicators/building_comparison/indicator.py @@ -53,9 +53,11 @@ async def coverage(cls, inverse=False) -> list[Feature]: else: coverage_type = "coverage_simple" feature = await db_client.get_reference_coverage( - val["table_name"], coverage_type + val["dataset_name_snake_case"], coverage_type + ) + feature.properties.update( + {"refernce_dataset": val["dataset_name_snake_case"]} ) - feature.properties.update({"refernce_dataset": val["table_name"]}) features.append(feature) return features @@ -77,7 +79,7 @@ async def preprocess(self) -> None: # get coverage [%] self.area_cov[key] = await db_client.get_intersection_area( self.feature, - val["table_name"], + val["dataset_name_snake_case"], ) if self.check_major_edge_cases(key) != "": continue @@ -85,12 +87,12 @@ async def preprocess(self) -> None: # clip input geom with coverage of reference dataset feature = await db_client.get_intersection_geom( self.feature, - val["table_name"], + val["dataset_name_snake_case"], ) # get reference building area result = await get_reference_building_area( - geojson.dumps(feature), val["table_name"] + geojson.dumps(feature), val["dataset_name_snake_case"] ) self.area_ref[key] = result / (1000 * 1000) @@ -119,7 +121,7 @@ def calculate(self) -> None: description = template.substitute( ratio=round(self.ratio[key] * 100, 2), coverage=round(self.area_cov[key] * 100, 2), - dataset=self.data_ref[key]["name"], + dataset=self.data_ref[key]["dataset_name_snake_case"], ) result_description = " ".join((result_description, edge_case, description)) @@ -173,10 +175,10 @@ def create_figure(self) -> None: for key, dataset in self.data_ref.items(): if None in (self.area_ref[key], self.area_osm[key]): continue - ref_x.append(dataset["table_name"]) + ref_x.append(dataset["dataset_name_snake_case"]) ref_y.append(round(self.area_ref[key], 2)) ref_data.append(dataset) - osm_x.append(dataset["table_name"]) + osm_x.append(dataset["dataset_name_snake_case"]) osm_y.append(round(self.area_osm[key], 2)) ref_hover.append(f"{dataset['table_name']} ({dataset['date']})") osm_hover.append(f"OSM ({self.result.timestamp_osm:%b %d, %Y})") @@ -320,17 +322,20 @@ async def load_datasets_metadata() -> dict: async with con.cursor() as cur: await cur.execute( "SELECT * " - "FROM building_comparison_metadata " - "WHERE indicator = 'building-comparison';" + "FROM comparison_indicators_metadata " + "WHERE indicator = 'building_comparison';" ) async for row in cur: dataset_name = row[0] - link = row[1] - date = row[2].strftime("%Y-%m-%d") # Convert date object to string - description = row[3] - color = row[4] - table_name = row[5] + dataset_name_snake_case = row[1] + link = row[2] + date = row[3].strftime("%Y-%m-%d") # Convert date object to string + description = row[4] + color = row[5] + table_name = row[6] dataset_metadata[dataset_name] = { + "dataset_name": dataset_name, + "dataset_name_snake_case": dataset_name_snake_case, "link": link, "date": date, "description": description, diff --git a/ohsome_quality_api/indicators/building_completeness/indicator.py b/ohsome_quality_api/indicators/building_completeness/indicator.py index 5ae5b5346..b28742f38 100644 --- a/ohsome_quality_api/indicators/building_completeness/indicator.py +++ b/ohsome_quality_api/indicators/building_completeness/indicator.py @@ -78,6 +78,9 @@ def threshhold_yellow(cls): """Above or equal to this value label should be yellow.""" return 0.2 + async def init(self) -> None: + pass + async def preprocess(self) -> None: # Get hex-cells hex_cells: FeatureCollection = await get_hex_cells(self.feature) diff --git a/ohsome_quality_api/indicators/currentness/indicator.py b/ohsome_quality_api/indicators/currentness/indicator.py index 05dd9b358..8eea92480 100644 --- a/ohsome_quality_api/indicators/currentness/indicator.py +++ b/ohsome_quality_api/indicators/currentness/indicator.py @@ -60,6 +60,9 @@ def __init__( self.bin_in_between: Bin self.bin_out_of_date: Bin + async def init(self) -> None: + pass + async def preprocess(self): """Fetch all latest contributions in monthly buckets since 2008 diff --git a/ohsome_quality_api/indicators/density/indicator.py b/ohsome_quality_api/indicators/density/indicator.py index 116d0ef69..fe9c62dc2 100644 --- a/ohsome_quality_api/indicators/density/indicator.py +++ b/ohsome_quality_api/indicators/density/indicator.py @@ -29,6 +29,9 @@ def green_threshold_function(self, area): def yellow_threshold_function(self, area): return self.threshold_red * area + async def init(self) -> None: + pass + async def preprocess(self) -> None: query_results_count = await ohsome_client.query(self.topic, self.feature) self.area_sqkm = calculate_area(self.feature) / (1000 * 1000) diff --git a/ohsome_quality_api/indicators/mapping_saturation/indicator.py b/ohsome_quality_api/indicators/mapping_saturation/indicator.py index 877710402..f88973f27 100644 --- a/ohsome_quality_api/indicators/mapping_saturation/indicator.py +++ b/ohsome_quality_api/indicators/mapping_saturation/indicator.py @@ -63,6 +63,9 @@ def __init__( self.best_fit: models.BaseStatModel | None = None self.fitted_models: list[models.BaseStatModel] = [] + async def init(self) -> None: + pass + async def preprocess(self) -> None: query_results = await ohsome_client.query( self.topic, diff --git a/ohsome_quality_api/indicators/minimal/indicator.py b/ohsome_quality_api/indicators/minimal/indicator.py index 00bb02022..d7af68ff5 100644 --- a/ohsome_quality_api/indicators/minimal/indicator.py +++ b/ohsome_quality_api/indicators/minimal/indicator.py @@ -14,6 +14,9 @@ def __init__(self, topic: Topic, feature: Feature) -> None: super().__init__(topic=topic, feature=feature) self.count = 0 + async def init(self) -> None: + pass + async def preprocess(self) -> None: query_results = await ohsome_client.query(self.topic, self.feature) self.count = query_results["result"][0]["value"] diff --git a/ohsome_quality_api/indicators/road_comparison/indicator.py b/ohsome_quality_api/indicators/road_comparison/indicator.py index bbb9cea9a..7d41d4051 100644 --- a/ohsome_quality_api/indicators/road_comparison/indicator.py +++ b/ohsome_quality_api/indicators/road_comparison/indicator.py @@ -55,9 +55,11 @@ async def coverage(cls, inverse=False) -> list[Feature]: else: coverage_type = "coverage_simple" feature = await db_client.get_reference_coverage( - val["table_name"], coverage_type + val["dataset_name_snake_case"], coverage_type + ) + feature.properties.update( + {"refernce_dataset": val["dataset_name_snake_case"]} ) - feature.properties.update({"refernce_dataset": val["table_name"]}) features.append(feature) return features @@ -82,7 +84,7 @@ async def preprocess(self) -> None: # get area covered by reference dataset [%] self.area_cov[key] = await db_client.get_intersection_area( self.feature, - val["table_name"], + val["dataset_name_snake_case"], ) self.warnings[key] = self.check_major_edge_cases(key) if self.warnings[key] != "": @@ -91,7 +93,7 @@ async def preprocess(self) -> None: # clip input geom with coverage of reference dataset feature = await db_client.get_intersection_geom( self.feature, - val["table_name"], + val["dataset_name_snake_case"], ) # get covered road length @@ -124,7 +126,7 @@ def calculate(self) -> None: self.result.description += self.warnings[key] + "\n" self.result.description += ( - f"{self.data_ref[key]['table_name']} has a road length of " + f"{self.data_ref[key]['dataset_name']} has a road length of " f"{(self.length_total[key]/1000):.2f} km, of which " f"{(self.length_matched[key]/1000):.2f} km are covered by roads in " f"OSM. " @@ -165,7 +167,7 @@ def create_figure(self) -> None: for key, val in self.ratio.items(): if val is None: continue - ref_name.append(self.data_ref[key]["table_name"]) + ref_name.append(self.data_ref[key]["dataset_name"]) ref_color.append(Color[self.data_ref[key]["color"]].value) ref_processingdate.append(self.data_ref[key]["date"]) ref_ratio.append(val) @@ -313,17 +315,20 @@ async def load_datasets_metadata() -> dict: async with con.cursor() as cur: await cur.execute( "SELECT * " - "FROM building_comparison_metadata " - "WHERE indicator = 'road-comparison';" + "FROM comparison_indicators_metadata " + "WHERE indicator = 'road_comparison';" ) async for row in cur: dataset_name = row[0] - link = row[1] - date = row[2].strftime("%Y-%m-%d") # Convert date object to string - description = row[3] - color = row[4] - table_name = row[5] + dataset_name_snake_case = row[1] + link = row[2] + date = row[3] # Convert date object to string + description = row[4] + color = row[5] + table_name = row[6] dataset_metadata[dataset_name] = { + "dataset_name": dataset_name, + "dataset_name_snake_case": dataset_name_snake_case, "link": link, "date": date, "description": description, diff --git a/ohsome_quality_api/oqt.py b/ohsome_quality_api/oqt.py index 0bd85190c..c2f62de7a 100644 --- a/ohsome_quality_api/oqt.py +++ b/ohsome_quality_api/oqt.py @@ -77,7 +77,7 @@ async def _create_indicator( indicator_class = get_class_from_key(class_type="indicator", key=key) indicator = indicator_class(topic, feature) - logging.info("Run initialization") + logging.info("Run initialisation") await indicator.init() logging.info("Run preprocessing") await indicator.preprocess() From fe00371d6b05699e2a51576e072553dbc9d956b7 Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Thu, 16 May 2024 16:30:57 +0200 Subject: [PATCH 08/10] chore: add CHANGELOG --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 266375283..387cde6b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,20 @@ # Changelog +## Current Main + +### Other Changes + +- road-comparison: covered Microsoft Roads are now calculated with tag `highway=* and type:way` ([#791]) +- comparison-indicators: now request metadata from database ([#791]) + + +[#791]: https://github.com/GIScience/ohsome-quality-api/pull/791 + ## Release 1.3.0 ### Breaking Changes -- major-roads-length: rename to `roads` and chnage filter to `highway=* and type:way` ([#786]) +- major-roads-length: rename to `roads` and change filter to `highway=* and type:way` ([#786]) ### New Features From 06dc4887ce3e9f889de4b067fe82d377a9065a3c Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Thu, 16 May 2024 18:11:31 +0200 Subject: [PATCH 09/10] fix: get_reference_coverage no works with metadata table --- ohsome_quality_api/geodatabase/client.py | 6 +++--- ohsome_quality_api/geodatabase/select_coverage.sql | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ohsome_quality_api/geodatabase/client.py b/ohsome_quality_api/geodatabase/client.py index 7f7b26551..647a87de9 100644 --- a/ohsome_quality_api/geodatabase/client.py +++ b/ohsome_quality_api/geodatabase/client.py @@ -74,10 +74,10 @@ async def get_reference_coverage(table_name: str, coverage_type: str) -> Feature """Get reference coverage for a bounding polygon.""" file_path = os.path.join(WORKING_DIR, "select_coverage.sql") with open(file_path, "r") as file: - query = file.read() + query = file.read().replace("{coverage_type}", coverage_type) async with get_connection() as conn: - result = await conn.fetchrow(query, coverage_type, table_name) - return Feature(geometry=geojson.loads(result[0]["geom"])) + result = await conn.fetchrow(query, table_name) + return Feature(geometry=geojson.loads(result["geom"])) async def get_intersection_area(bpoly: Feature, table_name: str) -> float: diff --git a/ohsome_quality_api/geodatabase/select_coverage.sql b/ohsome_quality_api/geodatabase/select_coverage.sql index 23cd5365a..6e03bd523 100644 --- a/ohsome_quality_api/geodatabase/select_coverage.sql +++ b/ohsome_quality_api/geodatabase/select_coverage.sql @@ -1,8 +1,8 @@ SELECT - ST_AsGeoJSON (ST_Transform ($1, 4326)) as geom + ST_AsGeoJSON (ST_Transform ({coverage_type}, 4326)) as geom FROM comparison_indicators_metadata WHERE - dataset_name_snake_case like $2; + dataset_name_snake_case like $1; From cd9d495e7143d0c52604aaa9720bccd8dfba1cc2 Mon Sep 17 00:00:00 2001 From: Gigaszi Date: Thu, 16 May 2024 18:13:05 +0200 Subject: [PATCH 10/10] refactor: delete dataset.yml files --- .../building_comparison/datasets.yaml | 25 ------------------- .../indicators/road_comparison/datasets.yaml | 13 ---------- 2 files changed, 38 deletions(-) delete mode 100644 ohsome_quality_api/indicators/building_comparison/datasets.yaml delete mode 100644 ohsome_quality_api/indicators/road_comparison/datasets.yaml diff --git a/ohsome_quality_api/indicators/building_comparison/datasets.yaml b/ohsome_quality_api/indicators/building_comparison/datasets.yaml deleted file mode 100644 index da92d159f..000000000 --- a/ohsome_quality_api/indicators/building_comparison/datasets.yaml +++ /dev/null @@ -1,25 +0,0 @@ -EUBUCCO: - name: EUBUCCO - link: https://docs.eubucco.com/ - date: Nov 3, 2022 - description: >- - EUBUCCO is a dataset of building footprints for Europe. - It is derived from administrative datasets. - color: PURPLE - coverage: - simple: eubucco_coverage_simple - inversed: eubucco_coverage_inversed - table_name: eubucco - -Microsoft Buildings: - name: Microsoft Building Footprints - link: https://planetarycomputer.microsoft.com/dataset/ms-buildings - date: July 5, 2022 - description: >- - Microsoft Building Footprints is a dataset of building footprints for the world. - It is derived from satellite imagery. - color: ORANGE - coverage: - simple: microsoft_buildings_coverage_simple - inversed: microsoft_buildings_coverage_inversed - table_name: microsoft_buildings diff --git a/ohsome_quality_api/indicators/road_comparison/datasets.yaml b/ohsome_quality_api/indicators/road_comparison/datasets.yaml deleted file mode 100644 index d10ddf08e..000000000 --- a/ohsome_quality_api/indicators/road_comparison/datasets.yaml +++ /dev/null @@ -1,13 +0,0 @@ -Microsoft Roads: - name: Microsoft Roads - link: https://github.com/microsoft/RoadDetections - date: February 27, 2023 - description: >- - Microsoft Road Detections is a dataset of road world-wide. - It is derived from satellite imagery. - color: ORANGE - coverage: - simple: microsoft_roads_coverage_simple - inversed: microsoft_roads_coverage_inversed - table_name: microsoft_roads_midpoint - processing_date: 2024-04-05