replace cql2 package with pygeofilter (#22)

* use pytest_lazy_fixtures for lazyfixtures; remove the cql2 and replace it with pygeofilter

* remove pytest<8 restriction and see what it do

* try using pytest-lazy-fixtures instead of pytest-lazy-fixture

* refractor utm_search and test it more, add more test_search as well as test_array for color.py

* utilize parser of cql from the deps remove custom code

* update depndabot

* gitignore sorted

* move all package settings into pyproject.toml

Author: Scartography

.coveragerc

@@ -15,11 +15,15 @@ updates:
       - dependency-name: "croniter"
       - dependency-name: "lxml"
       - dependency-name: "mapchete[complete]"
-      - dependency-name: "opencv-python"
+      - dependency-name: "opencv-python-headless"
       - dependency-name: "Pillow"
       - dependency-name: "pydantic"
+      - dependency-name: "pygeofilter"
       - dependency-name: "pystac[urllib3]"
       - dependency-name: "pystac-client"
+      - dependency-name: "pytest"
+      - dependency-name: "pytest-coverage"
+      - dependency-name: "pytest-lazy-fixtures"
       - dependency-name: "retry"
       - dependency-name: "rtree"
       - dependency-name: "scipy"

.github/dependabot.yml

@@ -1,14 +1,16 @@
-.eggs
-*.egg-info
-*.pyc
+build/
 .cache
-htmlcov
 .coverage
 .coverage.*
-build/
 dist/
-.pytest*
+*.egg-info
+.eggs
+examples/sentinel-2*/
 *.gfs
-.vscode/
+htmlcov
+.mypy_cache
+*.pyc
 __pycache__
-examples/sentinel-2*/
+.pytest*
+.ruff_cache
+.vscode/

.gitignore

@@ -2,9 +2,10 @@
 import json
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Generator, List, Optional, Type, Union
+from typing import Any, Callable, Dict, Generator, List, Optional, Set, Type, Union
 
-from cql2 import Expr
+from pygeofilter.parsers.ecql import parse as parse_ecql
+from pygeofilter.backends.native.evaluate import NativeEvaluator
 from pydantic import BaseModel
 from mapchete.path import MPath, MPathLike
 from mapchete.types import Bounds
@@ -17,6 +18,8 @@
 from shapely.geometry.base import BaseGeometry
 
 from mapchete_eo.io.assets import get_assets, get_metadata_assets
+from mapchete_eo.product import blacklist_products
+from mapchete_eo.settings import mapchete_eo_settings
 from mapchete_eo.types import TimeRange
 
 logger = logging.getLogger(__name__)
@@ -53,6 +56,11 @@ class CollectionSearcher(ABC):
     config_cls: Type[BaseModel]
     collection: str
     stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None
+    blacklist: Set[str] = (
+        blacklist_products(mapchete_eo_settings.blacklist)
+        if mapchete_eo_settings.blacklist
+        else set()
+    )
 
     def __init__(
         self,
@@ -70,17 +78,21 @@ def client(self) -> CollectionClient: ...
     @cached_property
     def eo_bands(self) -> List[str]: ...
 
-    @abstractmethod
+    @property
+    def config(self) -> BaseModel:
+        return self.config_cls()
+
     @cached_property
-    def id(self) -> str: ...
+    def id(self) -> str:
+        return self.client.id
 
-    @abstractmethod
     @cached_property
-    def description(self) -> str: ...
+    def description(self) -> str:
+        return self.client.description
 
-    @abstractmethod
     @cached_property
-    def stac_extensions(self) -> List[str]: ...
+    def stac_extensions(self) -> List[str]:
+        return self.client.stac_extensions
 
     @abstractmethod
     def search(
@@ -240,9 +252,12 @@ def filter_items(
     and passed down to the individual search approaches via said config and this Function.
     """
     if query:
-        expr = Expr(query)
+        ast = parse_ecql(query)
+        evaluator = NativeEvaluator(use_getattr=False)
+        filter_func = evaluator.evaluate(ast)
         for item in items:
-            if expr.matches(item.properties):
+            # pystac items store metadata in 'properties'
+            if filter_func(item.properties):
                 yield item
     else:
         yield from items

mapchete_eo/search/base.py

@@ -50,18 +50,21 @@ def deprecate_max_cloud_cover(cls, values: Dict[str, Any]) -> Dict[str, Any]:
             path=MPath(
                 "https://sentinel-s2-l2a-stac.s3.amazonaws.com/sentinel-s2-l2a.json"
             ),
+            endpoint="s3://sentinel-s2-l2a-stac",
         ),
         S2_L1C=dict(
             id="sentinel-s2-l1c",
             path=MPath(
                 "https://sentinel-s2-l1c-stac.s3.amazonaws.com/sentinel-s2-l1c.json"
             ),
+            endpoint="s3://sentinel-s2-l1c-stac",
         ),
         S1_GRD=dict(
             id="sentinel-s1-l1c",
             path=MPath(
                 "https://sentinel-s1-l1c-stac.s3.amazonaws.com/sentinel-s1-l1c.json"
             ),
+            endpoint="s3://sentinel-s1-l1c-stac",
         ),
     )
     search_index: Optional[MPathLike] = None

mapchete_eo/search/config.py

@@ -3,7 +3,7 @@
 import logging
 from datetime import datetime
 from functools import cached_property
-from typing import Any, Dict, Generator, Iterator, List, Optional, Set, Union
+from typing import Any, Dict, Generator, Iterator, List, Optional, Union
 
 from mapchete import Timer
 from mapchete.tile import BufferedTilePyramid
@@ -13,22 +13,15 @@
 from shapely.geometry import shape, box
 from shapely.geometry.base import BaseGeometry
 
-from mapchete_eo.product import blacklist_products
 from mapchete_eo.search.base import CollectionSearcher, StaticCollectionWriterMixin
 from mapchete_eo.search.config import StacSearchConfig, patch_invalid_assets
-from mapchete_eo.settings import mapchete_eo_settings
 from mapchete_eo.types import TimeRange
 
 logger = logging.getLogger(__name__)
 
 
 class STACSearchCollection(StaticCollectionWriterMixin, CollectionSearcher):
     collection: str
-    blacklist: Set[str] = (
-        blacklist_products(mapchete_eo_settings.blacklist)
-        if mapchete_eo_settings.blacklist
-        else set()
-    )
     config_cls = StacSearchConfig
 
     @cached_property
@@ -45,18 +38,6 @@ def eo_bands(self) -> List[str]:
             logger.debug("cannot find eo:bands definition from collections")
             return []
 
-    @cached_property
-    def id(self) -> str:
-        return self.client.id
-
-    @cached_property
-    def description(self) -> str:
-        return self.client.description
-
-    @cached_property
-    def stac_extensions(self) -> List[str]:
-        return self.client.stac_extensions
-
     def search(
         self,
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,

mapchete_eo/search/stac_search.py

@@ -65,18 +65,6 @@ def eo_bands(self) -> List[str]:
             logger.debug("cannot find eo:bands definition")
             return []
 
-    @cached_property
-    def id(self) -> str:
-        return self.client.id
-
-    @cached_property
-    def description(self) -> str:
-        return self.client.description
-
-    @cached_property
-    def stac_extensions(self) -> List[str]:
-        return self.client.stac_extensions
-
     def search(
         self,
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,

mapchete_eo/search/stac_static.py

@@ -1,47 +1,51 @@
 import datetime
 from functools import cached_property
 import logging
-from typing import Any, Dict, Generator, List, Optional, Set, Union
+from typing import Any, Dict, Generator, List, Optional, Union
 
 from mapchete.io.vector import fiona_open
 from mapchete.path import MPath, MPathLike
 from mapchete.types import Bounds, BoundsLike
 from pystac.collection import Collection
 from pystac.item import Item
+from pystac_client import CollectionClient
 from shapely.errors import GEOSException
 from shapely.geometry import shape
 from shapely.geometry.base import BaseGeometry
 
 from mapchete_eo.exceptions import ItemGeometryError
-from mapchete_eo.product import blacklist_products
 from mapchete_eo.search.base import (
     CollectionSearcher,
     StaticCollectionWriterMixin,
     filter_items,
 )
 from mapchete_eo.search.config import UTMSearchConfig
 from mapchete_eo.search.s2_mgrs import S2Tile, s2_tiles_from_bounds
-from mapchete_eo.settings import mapchete_eo_settings
 from mapchete_eo.time import day_range, to_datetime
 from mapchete_eo.types import TimeRange
 
 logger = logging.getLogger(__name__)
 
 
 class UTMSearchCatalog(StaticCollectionWriterMixin, CollectionSearcher):
-    endpoint: str
-    id: str
-    day_subdir_schema: str
-    stac_json_endswith: str
-    description: str
-    stac_extensions: List[str]
-    blacklist: Set[str] = (
-        blacklist_products(mapchete_eo_settings.blacklist)
-        if mapchete_eo_settings.blacklist
-        else set()
-    )
     config_cls = UTMSearchConfig
 
+    @cached_property
+    def endpoint(self) -> Optional[str]:
+        for collection_properties in self.config.sinergise_aws_collections.values():
+            if collection_properties["id"] == self.collection.split("/")[-1].replace(
+                ".json", ""
+            ):
+                return collection_properties.get("endpoint")
+        return None
+
+    day_subdir_schema: str = "{year}/{month:02d}/{day:02d}"
+    stac_json_endswith: str = "T{tile_id}.json"
+
+    @cached_property
+    def client(self) -> CollectionClient:
+        return next(self.get_collections())
+
     @cached_property
     def eo_bands(self) -> List[str]:  # pragma: no cover
         for (
@@ -85,8 +89,9 @@ def _raw_search(
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[Bounds] = None,
         area: Optional[BaseGeometry] = None,
-        config: UTMSearchConfig = UTMSearchConfig(),
+        config: Optional[UTMSearchConfig] = None,
     ) -> Generator[Item, None, None]:
+        config = config or UTMSearchConfig()
         if time is None:
             raise ValueError("time must be given")
         if area is not None and area.is_empty:

mapchete_eo/search/utm_search.py

@@ -24,7 +24,7 @@ classifiers = [
 ]
 dependencies = [
     "click",
-    "cql2",
+    "pygeofilter",
     "croniter",
     "lxml",
     "mapchete[complete]>=2025.10.0",
@@ -36,7 +36,6 @@ dependencies = [
     "retry",
     "rtree",
     "scipy",
-    "retry",
     "tqdm",
     "xarray",
 ]
@@ -47,9 +46,9 @@ docs = [
   "sphinx-rtd-theme"
 ]
 test = [
-    "pytest<8",
+    "pytest",
     "pytest-coverage",
-    "pytest-lazy-fixture"
+    "pytest-lazy-fixtures"
 ]
 
 [project.entry-points."mapchete.cli.commands"]
@@ -76,4 +75,16 @@ include = [
 ]
 
 [tool.mypy]
-ignore_missing_imports = true
+ignore_missing_imports = true
+
+[tool.coverage.run]
+branch = true
+source = ["mapchete_eo"]
+
+[tool.pytest.ini_options]
+addopts = "--durations 20 --verbose --nf --cov=mapchete_eo --cov-report=term-missing:skip-covered"
+testpaths = ["tests"]
+markers = [
+    "remote: marks tests which require acces to remote resources (deselect with '-m \"not remote\"')",
+    "use_cdse_test_env: enables CDSE S3 environment access",
+]