Skip to content

Commit c84db16

Browse files
author
thomas loubrieu
committed
fix forgotten nested properties management, extend changes to elasticsearch
1 parent 310a25d commit c84db16

File tree

2 files changed

+168
-47
lines changed

2 files changed

+168
-47
lines changed

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py‎

Lines changed: 107 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,20 @@ def __attrs_post_init__(self):
154154

155155
aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING
156156

157+
# constants for field names
158+
# they are used in multiple methods
159+
# and could be overwritten in subclasses used with alternate opensearch mappings.
160+
PROPERTIES_DATETIME_FIELD = "properties.datetime"
161+
PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime"
162+
PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime"
163+
COLLECTION_FIELD = "collection"
164+
GEOMETRY_FIELD = "geometry"
165+
166+
@staticmethod
167+
def __nested_field__(field: str):
168+
"""Convert opensearch field to nested field format."""
169+
return field.replace(".", "__")
170+
157171
"""CORE LOGIC"""
158172

159173
async def get_all_collections(
@@ -436,7 +450,10 @@ def apply_ids_filter(search: Search, item_ids: List[str]):
436450
@staticmethod
437451
def apply_collections_filter(search: Search, collection_ids: List[str]):
438452
"""Database logic to search a list of STAC collection ids."""
439-
return search.filter("terms", collection=collection_ids)
453+
collection_nested_field = DatabaseLogic.__nested_field__(
454+
DatabaseLogic.COLLECTION_FIELD
455+
)
456+
return search.filter("terms", **{collection_nested_field: collection_ids})
440457

441458
@staticmethod
442459
def apply_datetime_filter(
@@ -461,6 +478,16 @@ def apply_datetime_filter(
461478
if not datetime_search:
462479
return search, datetime_search
463480

481+
nested_datetime_field = DatabaseLogic.__nested_field__(
482+
DatabaseLogic.PROPERTIES_DATETIME_FIELD
483+
)
484+
nested_start_datetime_field = DatabaseLogic.__nested_field__(
485+
DatabaseLogic.PROPERTIES_START_DATETIME_FIELD
486+
)
487+
nested_end_datetime_field = DatabaseLogic.__nested_field__(
488+
DatabaseLogic.PROPERTIES_END_DATETIME_FIELD
489+
)
490+
464491
if USE_DATETIME:
465492
if "eq" in datetime_search:
466493
# For exact matches, include:
@@ -470,28 +497,42 @@ def apply_datetime_filter(
470497
Q(
471498
"bool",
472499
filter=[
473-
Q("exists", field="properties.datetime"),
500+
Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD),
474501
Q(
475502
"term",
476-
**{"properties__datetime": datetime_search["eq"]},
503+
**{nested_datetime_field: datetime_search["eq"]},
477504
),
478505
],
479506
),
480507
Q(
481508
"bool",
482-
must_not=[Q("exists", field="properties.datetime")],
509+
must_not=[
510+
Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)
511+
],
483512
filter=[
484-
Q("exists", field="properties.start_datetime"),
485-
Q("exists", field="properties.end_datetime"),
513+
Q(
514+
"exists",
515+
field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD,
516+
),
517+
Q(
518+
"exists",
519+
field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD,
520+
),
486521
Q(
487522
"range",
488-
properties__start_datetime={
489-
"lte": datetime_search["eq"]
523+
**{
524+
nested_start_datetime_field: {
525+
"lte": datetime_search["eq"]
526+
}
490527
},
491528
),
492529
Q(
493530
"range",
494-
properties__end_datetime={"gte": datetime_search["eq"]},
531+
**{
532+
nested_end_datetime_field: {
533+
"gte": datetime_search["eq"]
534+
}
535+
},
495536
),
496537
],
497538
),
@@ -504,32 +545,46 @@ def apply_datetime_filter(
504545
Q(
505546
"bool",
506547
filter=[
507-
Q("exists", field="properties.datetime"),
548+
Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD),
508549
Q(
509550
"range",
510-
properties__datetime={
511-
"gte": datetime_search["gte"],
512-
"lte": datetime_search["lte"],
551+
**{
552+
nested_datetime_field: {
553+
"gte": datetime_search["gte"],
554+
"lte": datetime_search["lte"],
555+
}
513556
},
514557
),
515558
],
516559
),
517560
Q(
518561
"bool",
519-
must_not=[Q("exists", field="properties.datetime")],
562+
must_not=[
563+
Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)
564+
],
520565
filter=[
521-
Q("exists", field="properties.start_datetime"),
522-
Q("exists", field="properties.end_datetime"),
566+
Q(
567+
"exists",
568+
field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD,
569+
),
570+
Q(
571+
"exists",
572+
field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD,
573+
),
523574
Q(
524575
"range",
525-
properties__start_datetime={
526-
"lte": datetime_search["lte"]
576+
**{
577+
nested_start_datetime_field: {
578+
"lte": datetime_search["lte"]
579+
}
527580
},
528581
),
529582
Q(
530583
"range",
531-
properties__end_datetime={
532-
"gte": datetime_search["gte"]
584+
**{
585+
nested_end_datetime_field: {
586+
"gte": datetime_search["gte"]
587+
}
533588
},
534589
),
535590
],
@@ -545,31 +600,53 @@ def apply_datetime_filter(
545600
filter_query = Q(
546601
"bool",
547602
filter=[
548-
Q("exists", field="properties.start_datetime"),
549-
Q("exists", field="properties.end_datetime"),
603+
Q(
604+
"exists",
605+
field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD,
606+
),
607+
Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD),
550608
Q(
551609
"range",
552-
properties__start_datetime={"lte": datetime_search["eq"]},
610+
**{
611+
nested_start_datetime_field: {
612+
"lte": datetime_search["eq"]
613+
}
614+
},
553615
),
554616
Q(
555617
"range",
556-
properties__end_datetime={"gte": datetime_search["eq"]},
618+
**{
619+
nested_end_datetime_field: {
620+
"gte": datetime_search["eq"]
621+
}
622+
},
557623
),
558624
],
559625
)
560626
else:
561627
filter_query = Q(
562628
"bool",
563629
filter=[
564-
Q("exists", field="properties.start_datetime"),
565-
Q("exists", field="properties.end_datetime"),
630+
Q(
631+
"exists",
632+
field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD,
633+
),
634+
Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD),
566635
Q(
567636
"range",
568-
properties__start_datetime={"lte": datetime_search["lte"]},
637+
**{
638+
nested_start_datetime_field: {
639+
"lte": datetime_search["lte"]
640+
}
641+
},
569642
),
570643
Q(
571644
"range",
572-
properties__end_datetime={"gte": datetime_search["gte"]},
645+
**{
646+
nested_end_datetime_field: {
647+
"gte": datetime_search["gte"]
648+
}
649+
},
573650
),
574651
],
575652
)
@@ -594,7 +671,7 @@ def apply_bbox_filter(search: Search, bbox: List):
594671
Q(
595672
{
596673
"geo_shape": {
597-
"geometry": {
674+
DatabaseLogic.GEOMETRY_FIELD: {
598675
"shape": {
599676
"type": "polygon",
600677
"coordinates": bbox2polygon(*bbox),
@@ -1708,7 +1785,7 @@ def bulk_sync(
17081785
kwargs = kwargs or {}
17091786

17101787
# Resolve the `refresh` parameter
1711-
refresh = kwargs.get("refresh", self.async_settings.database_refresh)
1788+
refresh = kwargs.get("refresh", self.sync_settings.database_refresh)
17121789
refresh = validate_refresh(refresh)
17131790

17141791
# Log the bulk insert attempt

‎stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py‎

Lines changed: 61 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,11 @@ def __attrs_post_init__(self):
163163
COLLECTION_FIELD = "collection"
164164
GEOMETRY_FIELD = "geometry"
165165

166+
@staticmethod
167+
def __nested_field__(field: str):
168+
"""Convert opensearch field to nested field format."""
169+
return field.replace(".", "__")
170+
166171
"""CORE LOGIC"""
167172

168173
async def get_all_collections(
@@ -445,9 +450,10 @@ def apply_ids_filter(search: Search, item_ids: List[str]):
445450
@staticmethod
446451
def apply_collections_filter(search: Search, collection_ids: List[str]):
447452
"""Database logic to search a list of STAC collection ids."""
448-
return search.filter(
449-
"terms", **{DatabaseLogic.COLLECTION_FIELD: collection_ids}
453+
collection_nested_field = DatabaseLogic.__nested_field__(
454+
DatabaseLogic.COLLECTION_FIELD
450455
)
456+
return search.filter("terms", **{collection_nested_field: collection_ids})
451457

452458
@staticmethod
453459
def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]):
@@ -490,6 +496,16 @@ def apply_datetime_filter(
490496
# False: Always search only by start/end datetime
491497
USE_DATETIME = get_bool_env("USE_DATETIME", default=True)
492498

499+
nested_datetime_field = DatabaseLogic.__nested_field__(
500+
DatabaseLogic.PROPERTIES_DATETIME_FIELD
501+
)
502+
nested_start_datetime_field = DatabaseLogic.__nested_field__(
503+
DatabaseLogic.PROPERTIES_START_DATETIME_FIELD
504+
)
505+
nested_end_datetime_field = DatabaseLogic.__nested_field__(
506+
DatabaseLogic.PROPERTIES_END_DATETIME_FIELD
507+
)
508+
493509
if USE_DATETIME:
494510
if "eq" in datetime_search:
495511
# For exact matches, include:
@@ -502,7 +518,7 @@ def apply_datetime_filter(
502518
Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD),
503519
Q(
504520
"term",
505-
**{"properties__datetime": datetime_search["eq"]},
521+
**{nested_datetime_field: datetime_search["eq"]},
506522
),
507523
],
508524
),
@@ -522,13 +538,19 @@ def apply_datetime_filter(
522538
),
523539
Q(
524540
"range",
525-
properties__start_datetime={
526-
"lte": datetime_search["eq"]
541+
**{
542+
nested_start_datetime_field: {
543+
"lte": datetime_search["eq"]
544+
}
527545
},
528546
),
529547
Q(
530548
"range",
531-
properties__end_datetime={"gte": datetime_search["eq"]},
549+
**{
550+
nested_end_datetime_field: {
551+
"gte": datetime_search["eq"]
552+
}
553+
},
532554
),
533555
],
534556
),
@@ -544,9 +566,11 @@ def apply_datetime_filter(
544566
Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD),
545567
Q(
546568
"range",
547-
properties__datetime={
548-
"gte": datetime_search["gte"],
549-
"lte": datetime_search["lte"],
569+
**{
570+
nested_datetime_field: {
571+
"gte": datetime_search["gte"],
572+
"lte": datetime_search["lte"],
573+
}
550574
},
551575
),
552576
],
@@ -567,14 +591,18 @@ def apply_datetime_filter(
567591
),
568592
Q(
569593
"range",
570-
properties__start_datetime={
571-
"lte": datetime_search["lte"]
594+
**{
595+
nested_start_datetime_field: {
596+
"lte": datetime_search["lte"]
597+
}
572598
},
573599
),
574600
Q(
575601
"range",
576-
properties__end_datetime={
577-
"gte": datetime_search["gte"]
602+
**{
603+
nested_end_datetime_field: {
604+
"gte": datetime_search["gte"]
605+
}
578606
},
579607
),
580608
],
@@ -597,11 +625,19 @@ def apply_datetime_filter(
597625
Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD),
598626
Q(
599627
"range",
600-
properties__start_datetime={"lte": datetime_search["eq"]},
628+
**{
629+
nested_start_datetime_field: {
630+
"lte": datetime_search["eq"]
631+
}
632+
},
601633
),
602634
Q(
603635
"range",
604-
properties__end_datetime={"gte": datetime_search["eq"]},
636+
**{
637+
nested_end_datetime_field: {
638+
"gte": datetime_search["eq"]
639+
}
640+
},
605641
),
606642
],
607643
)
@@ -616,11 +652,19 @@ def apply_datetime_filter(
616652
Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD),
617653
Q(
618654
"range",
619-
properties__start_datetime={"lte": datetime_search["lte"]},
655+
**{
656+
nested_start_datetime_field: {
657+
"lte": datetime_search["lte"]
658+
}
659+
},
620660
),
621661
Q(
622662
"range",
623-
properties__end_datetime={"gte": datetime_search["gte"]},
663+
**{
664+
nested_end_datetime_field: {
665+
"gte": datetime_search["gte"]
666+
}
667+
},
624668
),
625669
],
626670
)

0 commit comments

Comments
 (0)