Skip to content

Commit 5b7dd4e

Browse files
jonhealy1jonhealy1
committed
es_update
1 parent 56f4f7b commit 5b7dd4e

File tree

2 files changed

+175
-107
lines changed

2 files changed

+175
-107
lines changed

‎stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py‎

Lines changed: 59 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -245,121 +245,76 @@ def apply_collections_filter(search: Search, collection_ids: List[str]):
245245
@staticmethod
246246
def apply_datetime_filter(
247247
search: Search, interval: Optional[Union[DateTimeType, str]]
248-
):
248+
) -> Search:
249249
"""Apply a filter to search on datetime, start_datetime, and end_datetime fields.
250250
251251
Args:
252-
search (Search): The search object to filter.
253-
interval: Optional[Union[DateTimeType, str]]
252+
search: The search object to filter.
253+
interval: Optional datetime interval to filter by. Can be:
254+
- A single datetime string
255+
- A datetime range string (e.g., "2020-01-01/2020-12-31")
256+
- A datetime object
257+
- A tuple of (start_datetime, end_datetime)
254258
255259
Returns:
256-
Search: The filtered search object.
260+
The filtered search object.
257261
"""
262+
if not interval:
263+
return search
264+
258265
should = []
259266
datetime_search = return_date(interval)
260267

261-
# If the request is a single datetime return
262-
# items with datetimes equal to the requested datetime OR
263-
# the requested datetime is between their start and end datetimes
264268
if "eq" in datetime_search:
265-
should.extend(
266-
[
267-
Q(
268-
"bool",
269-
filter=[
270-
Q(
271-
"term",
272-
properties__datetime=datetime_search["eq"],
273-
),
274-
],
275-
),
276-
Q(
277-
"bool",
278-
filter=[
279-
Q(
280-
"range",
281-
properties__start_datetime={
282-
"lte": datetime_search["eq"],
283-
},
284-
),
285-
Q(
286-
"range",
287-
properties__end_datetime={
288-
"gte": datetime_search["eq"],
289-
},
290-
),
291-
],
292-
),
293-
]
294-
)
295-
296-
# If the request is a date range return
297-
# items with datetimes within the requested date range OR
298-
# their startdatetime ithin the requested date range OR
299-
# their enddatetime ithin the requested date range OR
300-
# the requested daterange within their start and end datetimes
269+
# For exact matches, include:
270+
# 1. Items with matching exact datetime
271+
# 2. Items with datetime:null where the time falls within their range
272+
should = [
273+
Q("term", **{"properties.datetime": datetime_search["eq"]}),
274+
Q(
275+
"bool",
276+
must_not=[Q("exists", field="properties.datetime")],
277+
filter=[
278+
Q(
279+
"range",
280+
properties__start_datetime={"lte": datetime_search["eq"]},
281+
),
282+
Q(
283+
"range",
284+
properties__end_datetime={"gte": datetime_search["eq"]},
285+
),
286+
],
287+
),
288+
]
289+
return search.query(Q("bool", should=should, minimum_should_match=1))
301290
else:
302-
should.extend(
303-
[
304-
Q(
305-
"bool",
306-
filter=[
307-
Q(
308-
"range",
309-
properties__datetime={
310-
"gte": datetime_search["gte"],
311-
"lte": datetime_search["lte"],
312-
},
313-
),
314-
],
315-
),
316-
Q(
317-
"bool",
318-
filter=[
319-
Q(
320-
"range",
321-
properties__start_datetime={
322-
"gte": datetime_search["gte"],
323-
"lte": datetime_search["lte"],
324-
},
325-
),
326-
],
327-
),
328-
Q(
329-
"bool",
330-
filter=[
331-
Q(
332-
"range",
333-
properties__end_datetime={
334-
"gte": datetime_search["gte"],
335-
"lte": datetime_search["lte"],
336-
},
337-
),
338-
],
339-
),
340-
Q(
341-
"bool",
342-
filter=[
343-
Q(
344-
"range",
345-
properties__start_datetime={
346-
"lte": datetime_search["gte"]
347-
},
348-
),
349-
Q(
350-
"range",
351-
properties__end_datetime={
352-
"gte": datetime_search["lte"]
353-
},
354-
),
355-
],
356-
),
357-
]
358-
)
359-
360-
search = search.query(Q("bool", filter=[Q("bool", should=should)]))
361-
362-
return search
291+
# For date ranges, include both:
292+
# 1. Items with datetime in the range
293+
should = [
294+
Q(
295+
"range",
296+
properties__datetime={
297+
"gte": datetime_search["gte"],
298+
"lte": datetime_search["lte"],
299+
},
300+
),
301+
# 2. Items with datetime:null that overlap the search range
302+
Q(
303+
"bool",
304+
must_not=[Q("exists", field="properties.datetime")],
305+
filter=[
306+
Q(
307+
"range",
308+
properties__start_datetime={"lte": datetime_search["lte"]},
309+
),
310+
Q(
311+
"range",
312+
properties__end_datetime={"gte": datetime_search["gte"]},
313+
),
314+
],
315+
),
316+
]
317+
return search.query(Q("bool", should=should, minimum_should_match=1))
363318

364319
@staticmethod
365320
def apply_bbox_filter(search: Search, bbox: List):

‎stac_fastapi/tests/resources/test_item.py‎

Lines changed: 116 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from stac_fastapi.core.datetime_utils import datetime_to_str, now_to_rfc3339_str
1616
from stac_fastapi.types.core import LandingPageMixin
1717

18-
from ..conftest import create_item, refresh_indices
18+
from ..conftest import create_collection, create_item, refresh_indices
1919

2020
if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch":
2121
from stac_fastapi.opensearch.database_logic import DatabaseLogic
@@ -398,8 +398,8 @@ async def test_item_search_temporal_intersecting_window_post(app_client, ctx):
398398
test_item = ctx.item
399399

400400
item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"])
401-
item_date_before = item_date - timedelta(days=10)
402-
item_date_after = item_date - timedelta(days=2)
401+
item_date_before = item_date - timedelta(days=2) # Changed from 10 to 2
402+
item_date_after = item_date + timedelta(days=2) # Changed from -2 to +2
403403

404404
params = {
405405
"collections": [test_item["collection"]],
@@ -973,3 +973,116 @@ async def test_search_datetime_validation_errors(app_client):
973973
# assert link["wms:transparent"]
974974
# return True
975975
# assert False, resp_json
976+
977+
978+
@pytest.mark.asyncio
979+
async def test_search_datetime_with_null_datetime(
980+
app_client, txn_client, load_test_data
981+
):
982+
"""Test datetime filtering when properties.datetime is null"""
983+
# First, create the test collection
984+
test_collection = load_test_data("test_collection.json")
985+
await create_collection(txn_client, collection=test_collection)
986+
987+
# Create test items with different datetime scenarios
988+
base_item = load_test_data("test_item.json")
989+
990+
# Item with null datetime but valid start/end datetimes
991+
null_dt_item = deepcopy(base_item)
992+
null_dt_item["id"] = "null-datetime-item"
993+
null_dt_item["properties"]["datetime"] = None
994+
null_dt_item["properties"]["start_datetime"] = "2020-01-01T00:00:00Z"
995+
null_dt_item["properties"]["end_datetime"] = "2020-01-02T00:00:00Z"
996+
await create_item(txn_client, null_dt_item)
997+
998+
# Item with valid datetime
999+
valid_dt_item = deepcopy(base_item)
1000+
valid_dt_item["id"] = "valid-datetime-item"
1001+
valid_dt_item["properties"]["datetime"] = "2020-01-01T12:00:00Z"
1002+
await create_item(txn_client, valid_dt_item)
1003+
1004+
await refresh_indices(txn_client)
1005+
1006+
# Test 1: Search for exact datetime that should match both items
1007+
# since the time falls within the null-datetime-item's range
1008+
resp = await app_client.get(
1009+
"/search",
1010+
params={
1011+
"datetime": "2020-01-01T12:00:00Z",
1012+
"collections": [base_item["collection"]],
1013+
},
1014+
)
1015+
assert resp.status_code == 200
1016+
feature_ids = {f["id"] for f in resp.json()["features"]}
1017+
assert "valid-datetime-item" in feature_ids
1018+
assert "null-datetime-item" in feature_ids
1019+
1020+
# Test 2: Search for a range that includes both items
1021+
resp = await app_client.get(
1022+
"/search",
1023+
params={
1024+
"datetime": "2020-01-01T00:00:00Z/2020-01-03T00:00:00Z",
1025+
"collections": [base_item["collection"]],
1026+
},
1027+
)
1028+
assert resp.status_code == 200
1029+
feature_ids = {f["id"] for f in resp.json()["features"]}
1030+
assert "valid-datetime-item" in feature_ids
1031+
assert "null-datetime-item" in feature_ids
1032+
1033+
# Test 3: Search with POST request to test JSON body
1034+
resp = await app_client.post(
1035+
"/search",
1036+
json={
1037+
"datetime": "2020-01-01T00:00:00Z/2020-01-02T00:00:00Z",
1038+
"collections": [base_item["collection"]],
1039+
},
1040+
)
1041+
assert resp.status_code == 200
1042+
feature_ids = {f["id"] for f in resp.json()["features"]}
1043+
assert "null-datetime-item" in feature_ids
1044+
1045+
# Add this after creating the other test items
1046+
# Item with datetime outside its start/end range
1047+
range_item = deepcopy(base_item)
1048+
range_item["id"] = "range-item"
1049+
range_item["properties"]["datetime"] = "2020-01-03T00:00:00Z" # Outside the range
1050+
range_item["properties"]["start_datetime"] = "2020-01-01T00:00:00Z"
1051+
range_item["properties"]["end_datetime"] = "2020-01-02T00:00:00Z"
1052+
await create_item(txn_client, range_item)
1053+
1054+
# Refresh indices after creating all items
1055+
await refresh_indices(txn_client)
1056+
1057+
# Add these test cases after the existing ones
1058+
# Test 4: Search for exact datetime that matches range_item's datetime (should match)
1059+
resp = await app_client.get(
1060+
"/search",
1061+
params={
1062+
"datetime": "2020-01-03T00:00:00Z", # Matches range_item's datetime
1063+
"collections": [base_item["collection"]],
1064+
},
1065+
)
1066+
assert resp.status_code == 200
1067+
feature_ids = {f["id"] for f in resp.json()["features"]}
1068+
assert "range-item" in feature_ids # Should match on exact datetime
1069+
assert "null-datetime-item" not in feature_ids # Should not match
1070+
assert "valid-datetime-item" not in feature_ids # Should not match
1071+
1072+
# Test 5: Search for range that includes range_item's range but not its datetime
1073+
resp = await app_client.get(
1074+
"/search",
1075+
params={
1076+
"datetime": "2020-01-01T12:00:00Z/2020-01-02T12:00:00Z", # Within range_item's range
1077+
"collections": [base_item["collection"]],
1078+
},
1079+
)
1080+
assert resp.status_code == 200
1081+
feature_ids = {f["id"] for f in resp.json()["features"]}
1082+
assert (
1083+
"range-item" not in feature_ids
1084+
) # Should not match as datetime is outside this range
1085+
assert "null-datetime-item" in feature_ids # Should match if in range
1086+
1087+
# Clean up
1088+
await txn_client.delete_collection(test_collection["id"])

0 commit comments

Comments
 (0)