Merge branch 'main' into extend_datetime_search

Author: jonhealy1

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- Advanced comparison (LIKE, IN, BETWEEN) operators to the Filter extension [#178](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/178)
+
 ### Changed
 
 - Extended Datetime Search to search on start_datetime and end_datetime as well as datetime fields. [#182](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/182)
@@ -19,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166)
 - Upgrade stac-fastapi to v2.4.9 [#172](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/172)
+- Set correct default filter-lang for GET /search requests [#179](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/179)
 
 ## [v1.0.0]
 

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py

@@ -24,6 +24,11 @@
 settings = ElasticsearchSettings()
 session = Session.create_from_settings(settings)
 
+filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient())
+filter_extension.conformance_classes.append(
+    "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators"
+)
+
 extensions = [
     TransactionExtension(client=TransactionsClient(session=session), settings=settings),
     BulkTransactionExtension(client=BulkTransactionsClient(session=session)),
@@ -32,7 +37,7 @@
     SortExtension(),
     TokenPaginationExtension(),
     ContextExtension(),
-    FilterExtension(client=EsAsyncBaseFiltersClient()),
+    filter_extension,
 ]
 
 post_request_model = create_post_request_model(extensions)

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py

@@ -368,12 +368,12 @@ async def get_search(
             base_args["sortby"] = sort_param
 
         if filter:
-            if filter_lang == "cql2-text":
+            if filter_lang == "cql2-json":
                 base_args["filter-lang"] = "cql2-json"
-                base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter)))
+                base_args["filter"] = orjson.loads(unquote_plus(filter))
             else:
                 base_args["filter-lang"] = "cql2-json"
-                base_args["filter"] = orjson.loads(unquote_plus(filter))
+                base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter)))
 
         if fields:
             includes = set()

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py

@@ -4,12 +4,16 @@
 Basic CQL2 (AND, OR, NOT), comparison operators (=, <>, <, <=, >, >=), and IS NULL.
 The comparison operators are allowed against string, numeric, boolean, date, and datetime types.
 
+Advanced comparison operators (http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators)
+defines the LIKE, IN, and BETWEEN operators.
+
 Basic Spatial Operators (http://www.opengis.net/spec/cql2/1.0/conf/basic-spatial-operators)
 defines the intersects operator (S_INTERSECTS).
 """
 from __future__ import annotations
 
 import datetime
+import re
 from enum import Enum
 from typing import List, Union
 
@@ -78,6 +82,17 @@ def to_es(self):
             )
 
 
+class AdvancedComparisonOp(str, Enum):
+    """Advanced Comparison operator.
+
+    CQL2 advanced comparison operators like (~), between, and in.
+    """
+
+    like = "like"
+    between = "between"
+    _in = "in"
+
+
 class SpatialIntersectsOp(str, Enum):
     """Spatial intersections operator s_intersects."""
 
@@ -152,8 +167,8 @@ def validate(cls, v):
 class Clause(BaseModel):
     """Filter extension clause."""
 
-    op: Union[LogicalOp, ComparisonOp, SpatialIntersectsOp]
-    args: List[Arg]
+    op: Union[LogicalOp, ComparisonOp, AdvancedComparisonOp, SpatialIntersectsOp]
+    args: List[Union[Arg, List[Arg]]]
 
     def to_es(self):
         """Generate an Elasticsearch expression for this Clause."""
@@ -171,6 +186,30 @@ def to_es(self):
                     "must_not": [{"term": {to_es(self.args[0]): to_es(self.args[1])}}]
                 }
             }
+        elif self.op == AdvancedComparisonOp.like:
+            return {
+                "wildcard": {
+                    to_es(self.args[0]): {
+                        "value": cql2_like_to_es(str(to_es(self.args[1]))),
+                        "case_insensitive": "false",
+                    }
+                }
+            }
+        elif self.op == AdvancedComparisonOp.between:
+            return {
+                "range": {
+                    to_es(self.args[0]): {
+                        "gte": to_es(self.args[1]),
+                        "lte": to_es(self.args[2]),
+                    }
+                }
+            }
+        elif self.op == AdvancedComparisonOp._in:
+            if not isinstance(self.args[1], List):
+                raise RuntimeError(f"Arg {self.args[1]} is not a list")
+            return {
+                "terms": {to_es(self.args[0]): [to_es(arg) for arg in self.args[1]]}
+            }
         elif (
             self.op == ComparisonOp.lt
             or self.op == ComparisonOp.lte
@@ -210,3 +249,19 @@ def to_es(arg: Arg):
         return arg
     else:
         raise RuntimeError(f"unknown arg {repr(arg)}")
+
+
+def cql2_like_to_es(string):
+    """Convert wildcard characters in CQL2 ('_' and '%') to Elasticsearch wildcard characters ('?' and '*', respectively). Handle escape characters and pass through Elasticsearch wildcards."""
+    percent_pattern = r"(?<!\\)%"
+    underscore_pattern = r"(?<!\\)_"
+    escape_pattern = r"\\(?=[_%])"
+
+    for pattern in [
+        (percent_pattern, "*"),
+        (underscore_pattern, "?"),
+        (escape_pattern, ""),
+    ]:
+        string = re.sub(pattern[0], pattern[1], string)
+
+    return string

stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json

@@ -0,0 +1,9 @@
+{
+  "op": "like",
+  "args": [
+    {
+      "property": "scene_id"
+    },
+    "LC82030282019133%"
+  ]
+}

stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json

@@ -0,0 +1,10 @@
+{
+    "op": "like",
+    "args": [
+      {
+        "property": "scene_id"
+      },
+      "LC82030282019133LGN0_"
+    ]
+  }
+  

stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json

@@ -0,0 +1,33 @@
+{
+    "op": "and",
+    "args": [
+        {
+            "op": "between",
+            "args": [
+                {
+                    "property": "cloud_cover"
+                },
+                0.1,
+                0.2
+            ]
+        },
+        {
+            "op": "=",
+            "args": [
+                {
+                    "property": "landsat:wrs_row"
+                },
+                28
+            ]
+        },
+        {
+            "op": "=",
+            "args": [
+                {
+                    "property": "landsat:wrs_path"
+                },
+                203
+            ]
+        }
+    ]
+}

stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json

@@ -0,0 +1,13 @@
+{
+    "op": "and",
+    "args": [
+        {
+            "op": "in",
+            "args": [
+                {"property": "id"},
+                ["LC08_L1TP_060247_20180905_20180912_01_T1_L1TP"]
+            ]
+        },
+        {"op": "=", "args": [{"property": "collection"}, "landsat8_l1tp"]}
+    ]
+}