Skip to content

Commit 095f094

Browse files
jonhealy1jonhealy1
authored
Merge branch 'main' into update-datetime-filter
2 parents 272d108 + a569587 commit 095f094

File tree

11 files changed

+182
-25
lines changed

11 files changed

+182
-25
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
88

99
## [Unreleased]
1010

11+
### Added
12+
13+
- Added support for enum queryables [#390](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/390)
14+
1115
### Changed
1216

1317
- Improved datetime query handling to only check start and end datetime values when datetime is None [#396](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/396)

‎Makefile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,11 @@ APP_HOST ?= 0.0.0.0
33
EXTERNAL_APP_PORT ?= 8080
44

55
ES_APP_PORT ?= 8080
6+
OS_APP_PORT ?= 8082
7+
68
ES_HOST ?= docker.for.mac.localhost
79
ES_PORT ?= 9200
810

9-
OS_APP_PORT ?= 8082
10-
OS_HOST ?= docker.for.mac.localhost
11-
OS_PORT ?= 9202
12-
1311
run_es = docker compose \
1412
run \
1513
-p ${EXTERNAL_APP_PORT}:${ES_APP_PORT} \

‎stac_fastapi/core/stac_fastapi/core/base_database_logic.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"""Base database logic."""
22

33
import abc
4-
from typing import Any, Dict, Iterable, Optional
4+
from typing import Any, Dict, Iterable, List, Optional
55

66

77
class BaseDatabaseLogic(abc.ABC):
@@ -36,6 +36,18 @@ async def delete_item(
3636
"""Delete an item from the database."""
3737
pass
3838

39+
@abc.abstractmethod
40+
async def get_items_mapping(self, collection_id: str) -> Dict[str, Dict[str, Any]]:
41+
"""Get the mapping for the items in the collection."""
42+
pass
43+
44+
@abc.abstractmethod
45+
async def get_items_unique_values(
46+
self, collection_id: str, field_names: Iterable[str], *, limit: int = ...
47+
) -> Dict[str, List[str]]:
48+
"""Get the unique values for the given fields in the collection."""
49+
pass
50+
3951
@abc.abstractmethod
4052
async def create_collection(self, collection: Dict, refresh: bool = False) -> None:
4153
"""Create a collection in the database."""

‎stac_fastapi/core/stac_fastapi/core/extensions/filter.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,17 @@
6060
"maximum": 100,
6161
},
6262
}
63+
"""Queryables that are present in all collections."""
64+
65+
OPTIONAL_QUERYABLES: Dict[str, Dict[str, Any]] = {
66+
"platform": {
67+
"$enum": True,
68+
"description": "Satellite platform identifier",
69+
},
70+
}
71+
"""Queryables that are present in some collections."""
72+
73+
ALL_QUERYABLES: Dict[str, Dict[str, Any]] = DEFAULT_QUERYABLES | OPTIONAL_QUERYABLES
6374

6475

6576
class LogicalOp(str, Enum):

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
TokenPaginationExtension,
3838
TransactionExtension,
3939
)
40+
from stac_fastapi.extensions.core.filter import FilterConformanceClasses
4041
from stac_fastapi.extensions.third_party import BulkTransactionExtension
4142
from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient
4243
from stac_fastapi.sfeos_helpers.filter import EsAsyncBaseFiltersClient
@@ -56,7 +57,7 @@
5657
client=EsAsyncBaseFiltersClient(database=database_logic)
5758
)
5859
filter_extension.conformance_classes.append(
59-
"http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators"
60+
FilterConformanceClasses.ADVANCED_COMPARISON_OPERATORS
6061
)
6162

6263
aggregation_extension = AggregationExtension(

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -871,6 +871,37 @@ async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]:
871871
except ESNotFoundError:
872872
raise NotFoundError(f"Mapping for index {index_name} not found")
873873

874+
async def get_items_unique_values(
875+
self, collection_id: str, field_names: Iterable[str], *, limit: int = 100
876+
) -> Dict[str, List[str]]:
877+
"""Get the unique values for the given fields in the collection."""
878+
limit_plus_one = limit + 1
879+
index_name = index_alias_by_collection_id(collection_id)
880+
881+
query = await self.client.search(
882+
index=index_name,
883+
body={
884+
"size": 0,
885+
"aggs": {
886+
field: {"terms": {"field": field, "size": limit_plus_one}}
887+
for field in field_names
888+
},
889+
},
890+
)
891+
892+
result: Dict[str, List[str]] = {}
893+
for field, agg in query["aggregations"].items():
894+
if len(agg["buckets"]) > limit:
895+
logger.warning(
896+
"Skipping enum field %s: exceeds limit of %d unique values. "
897+
"Consider excluding this field from enumeration or increase the limit.",
898+
field,
899+
limit,
900+
)
901+
continue
902+
result[field] = [bucket["key"] for bucket in agg["buckets"]]
903+
return result
904+
874905
async def create_collection(self, collection: Collection, **kwargs: Any):
875906
"""Create a single collection in the database.
876907

‎stac_fastapi/opensearch/stac_fastapi/opensearch/app.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
TokenPaginationExtension,
3232
TransactionExtension,
3333
)
34+
from stac_fastapi.extensions.core.filter import FilterConformanceClasses
3435
from stac_fastapi.extensions.third_party import BulkTransactionExtension
3536
from stac_fastapi.opensearch.config import OpensearchSettings
3637
from stac_fastapi.opensearch.database_logic import (
@@ -56,7 +57,7 @@
5657
client=EsAsyncBaseFiltersClient(database=database_logic)
5758
)
5859
filter_extension.conformance_classes.append(
59-
"http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators"
60+
FilterConformanceClasses.ADVANCED_COMPARISON_OPERATORS
6061
)
6162

6263
aggregation_extension = AggregationExtension(

‎stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,37 @@ async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]:
880880
except exceptions.NotFoundError:
881881
raise NotFoundError(f"Mapping for index {index_name} not found")
882882

883+
async def get_items_unique_values(
884+
self, collection_id: str, field_names: Iterable[str], *, limit: int = 100
885+
) -> Dict[str, List[str]]:
886+
"""Get the unique values for the given fields in the collection."""
887+
limit_plus_one = limit + 1
888+
index_name = index_alias_by_collection_id(collection_id)
889+
890+
query = await self.client.search(
891+
index=index_name,
892+
body={
893+
"size": 0,
894+
"aggs": {
895+
field: {"terms": {"field": field, "size": limit_plus_one}}
896+
for field in field_names
897+
},
898+
},
899+
)
900+
901+
result: Dict[str, List[str]] = {}
902+
for field, agg in query["aggregations"].items():
903+
if len(agg["buckets"]) > limit:
904+
logger.warning(
905+
"Skipping enum field %s: exceeds limit of %d unique values. "
906+
"Consider excluding this field from enumeration or increase the limit.",
907+
field,
908+
limit,
909+
)
910+
continue
911+
result[field] = [bucket["key"] for bucket in agg["buckets"]]
912+
return result
913+
883914
async def create_collection(self, collection: Collection, **kwargs: Any):
884915
"""Create a single collection in the database.
885916

‎stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
"""Filter client implementation for Elasticsearch/OpenSearch."""
22

33
from collections import deque
4-
from typing import Any, Dict, Optional
4+
from typing import Any, Dict, Optional, Tuple
55

66
import attr
77

88
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
9-
from stac_fastapi.core.extensions.filter import DEFAULT_QUERYABLES
9+
from stac_fastapi.core.extensions.filter import ALL_QUERYABLES, DEFAULT_QUERYABLES
1010
from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
1111
from stac_fastapi.sfeos_helpers.mappings import ES_MAPPING_TYPE_TO_JSON
1212

@@ -59,31 +59,31 @@ async def get_queryables(
5959

6060
mapping_data = await self.database.get_items_mapping(collection_id)
6161
mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
62-
stack = deque(mapping_properties.items())
62+
stack: deque[Tuple[str, Dict[str, Any]]] = deque(mapping_properties.items())
63+
enum_fields: Dict[str, Dict[str, Any]] = {}
6364

6465
while stack:
65-
field_name, field_def = stack.popleft()
66+
field_fqn, field_def = stack.popleft()
6667

6768
# Iterate over nested fields
6869
field_properties = field_def.get("properties")
6970
if field_properties:
70-
# Fields in Item Properties should be exposed with their un-prefixed names,
71-
# and not require expressions to prefix them with properties,
72-
# e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
73-
if field_name == "properties":
74-
stack.extend(field_properties.items())
75-
else:
76-
stack.extend(
77-
(f"{field_name}.{k}", v) for k, v in field_properties.items()
78-
)
71+
stack.extend(
72+
(f"{field_fqn}.{k}", v) for k, v in field_properties.items()
73+
)
7974

8075
# Skip non-indexed or disabled fields
8176
field_type = field_def.get("type")
8277
if not field_type or not field_def.get("enabled", True):
8378
continue
8479

80+
# Fields in Item Properties should be exposed with their un-prefixed names,
81+
# and not require expressions to prefix them with properties,
82+
# e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
83+
field_name = field_fqn.removeprefix("properties.")
84+
8585
# Generate field properties
86-
field_result = DEFAULT_QUERYABLES.get(field_name, {})
86+
field_result = ALL_QUERYABLES.get(field_name, {})
8787
properties[field_name] = field_result
8888

8989
field_name_human = field_name.replace("_", " ").title()
@@ -95,4 +95,13 @@ async def get_queryables(
9595
if field_type in {"date", "date_nanos"}:
9696
field_result.setdefault("format", "date-time")
9797

98+
if field_result.pop("$enum", False):
99+
enum_fields[field_fqn] = field_result
100+
101+
if enum_fields:
102+
for field_fqn, unique_values in (
103+
await self.database.get_items_unique_values(collection_id, enum_fields)
104+
).items():
105+
enum_fields[field_fqn]["enum"] = unique_values
106+
98107
return queryables

‎stac_fastapi/tests/conftest.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727
from stac_fastapi.core.rate_limit import setup_rate_limit
2828
from stac_fastapi.core.route_dependencies import get_route_dependencies
2929
from stac_fastapi.core.utilities import get_bool_env
30+
from stac_fastapi.extensions.core.filter import FilterConformanceClasses
3031
from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient
32+
from stac_fastapi.sfeos_helpers.filter import EsAsyncBaseFiltersClient
3133

3234
if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch":
3335
from stac_fastapi.opensearch.config import AsyncOpensearchSettings as AsyncSettings
@@ -39,9 +41,11 @@
3941
)
4042
else:
4143
from stac_fastapi.elasticsearch.config import (
42-
ElasticsearchSettings as SearchSettings,
4344
AsyncElasticsearchSettings as AsyncSettings,
4445
)
46+
from stac_fastapi.elasticsearch.config import (
47+
ElasticsearchSettings as SearchSettings,
48+
)
4549
from stac_fastapi.elasticsearch.database_logic import (
4650
DatabaseLogic,
4751
create_collection_index,
@@ -198,6 +202,13 @@ def bulk_txn_client():
198202
async def app():
199203
settings = AsyncSettings()
200204

205+
filter_extension = FilterExtension(
206+
client=EsAsyncBaseFiltersClient(database=database)
207+
)
208+
filter_extension.conformance_classes.append(
209+
FilterConformanceClasses.ADVANCED_COMPARISON_OPERATORS
210+
)
211+
201212
aggregation_extension = AggregationExtension(
202213
client=EsAsyncBaseAggregationClient(
203214
database=database, session=None, settings=settings
@@ -217,7 +228,7 @@ async def app():
217228
FieldsExtension(),
218229
QueryExtension(),
219230
TokenPaginationExtension(),
220-
FilterExtension(),
231+
filter_extension,
221232
FreeTextExtension(),
222233
]
223234

@@ -313,7 +324,6 @@ async def app_client_rate_limit(app_rate_limit):
313324

314325
@pytest_asyncio.fixture(scope="session")
315326
async def app_basic_auth():
316-
317327
stac_fastapi_route_dependencies = """[
318328
{
319329
"routes":[{"method":"*","path":"*"}],

0 commit comments

Comments
 (0)