Merge branch 'issue298-stac-bands'

soxofaan · soxofaan · commit 1168bc55052e · 2025-05-15T18:15:56.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,6 +28,8 @@ and start a new "In Progress" section above it.
 - Preserve original non-spatial dimensions in `resample_cube_spatial` dry run ([#397](https://github.com/Open-EO/openeo-python-driver/issues/397))
 - Fix compatibility with Shapely2 ([#158](https://github.com/Open-EO/openeo-python-driver/issues/158))
 - Allow `overlap` in `apply_neighborhood` to be not specified ([#401](https://github.com/Open-EO/openeo-python-driver/issues/401))
+- Start including STAC-1.1.0-style "bands" metadata in assets in batch job results ([#298](https://github.com/Open-EO/openeo-python-driver/issues/298))
+- Start including STAC-1.1.0-style "bands" summaries in collection metadata ([#298](https://github.com/Open-EO/openeo-python-driver/issues/298))
 
 
 ## 0.133.0
diff --git a/openeo_driver/constants.py b/openeo_driver/constants.py
@@ -1,12 +1,16 @@
 class STAC_EXTENSION:
     PROCESSING = "https://stac-extensions.github.io/processing/v1.1.0/schema.json"
     EO = "https://stac-extensions.github.io/eo/v1.1.0/schema.json"
+    EO_V110 = "https://stac-extensions.github.io/eo/v1.1.0/schema.json"
+    EO_V200 = "https://stac-extensions.github.io/eo/v2.0.0/schema.json"
     FILEINFO = "https://stac-extensions.github.io/file/v2.1.0/schema.json"
     PROJECTION = "https://stac-extensions.github.io/projection/v1.1.0/schema.json"
     DATACUBE = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json"
     MLMODEL = "https://stac-extensions.github.io/ml-model/v1.0.0/schema.json"
     CARD4LOPTICAL = "https://stac-extensions.github.io/card4l/v0.1.0/optical/schema.json"
     CARD4LSAR = "https://stac-extensions.github.io/card4l/v0.1.0/sar/schema.json"
+    RASTER_V110 = "https://stac-extensions.github.io/raster/v1.1.0/schema.json"
+    RASTER_V200 = "https://stac-extensions.github.io/raster/v2.0.0/schema.json"
 
 
 class JOB_STATUS:
diff --git a/openeo_driver/save_result.py b/openeo_driver/save_result.py
@@ -306,6 +306,8 @@ def __init__(
             warnings.warn("AggregatePolygonResult: GeometryCollection or DriverVectorCube expected but got {t}".format(t=type(regions)))
         self._regions = regions
         self._metadata = metadata
+        # TODO #298 this "raster:bands" helper is old-style
+        #      and just used for "statistics" which moved to the common metadata in v2
         self.raster_bands = None
 
     def get_data(self):
@@ -662,7 +664,6 @@ class AggregatePolygonResultCSV(AggregatePolygonResult):
     def __init__(self, csv_dir, regions: Union[GeometryCollection, DriverVectorCube, DelayedVector, BaseGeometry], metadata: CollectionMetadata = None):
         super().__init__(timeseries=None, regions=regions, metadata=metadata)
         self._csv_dir = csv_dir
-        self.raster_bands = None
 
     def get_data(self):
         if self.data is None:
@@ -717,6 +718,8 @@ def stats(band):
                 stats["stddev"] = series.std()
                 stats["valid_percent"] = ((100.0 * len(series.dropna()) / len(series)) if len(series) else None)
                 return {"statistics": stats}
+
+            # TODO #298 `raster:bands>statistics` has moved to common STAC in raster extension 2.0.0
             self.raster_bands = [stats(b) for b in bands]
 
         if self.is_format('covjson', 'coveragejson'):
diff --git a/openeo_driver/util/stac.py b/openeo_driver/util/stac.py
@@ -0,0 +1,25 @@
+"""
+Generic helpers to handle/consume/produce STAC items, collections, metadata constructs.
+"""
+from typing import Any
+
+import collections.abc
+
+
+def sniff_stac_extension_prefix(data: Any, prefix: str) -> bool:
+    """
+    Recursively walk through a data structure to
+    find a particular STAC extension prefix
+    in object keys (e.g. "eo:" in a "eo:bands" field of an asset).
+
+    :param data: data structure to scan
+    :param prefix: STAC extension prefix to look for,
+        e.g. "eo:", "raster:", "proj:", ...
+    """
+    if isinstance(data, dict):
+        if any(isinstance(k, str) and k.startswith(prefix) for k in data.keys()):
+            return True
+        return sniff_stac_extension_prefix(data=list(data.values()), prefix=prefix)
+    elif isinstance(data, collections.abc.Iterable) and not isinstance(data, (str, bytes)):
+        return any(sniff_stac_extension_prefix(data=x, prefix=prefix) for x in data)
+    return False
diff --git a/openeo_driver/views.py b/openeo_driver/views.py
@@ -72,6 +72,7 @@
 from openeo_driver.users.auth import HttpAuthHandler
 from openeo_driver.util.geometry import BoundingBox, reproject_geometry
 from openeo_driver.util.logging import ExtraLoggingFilter, FlaskRequestCorrelationIdLogging
+from openeo_driver.util.stac import sniff_stac_extension_prefix
 from openeo_driver.utils import EvalEnv, filter_supported_kwargs, smart_bool
 
 _log = logging.getLogger(__name__)
@@ -1166,7 +1167,9 @@ def job_results_canonical_url() -> str:
             )
 
         assets = {
-            filename: _asset_object(job_id, user_id, filename, asset_metadata, job_info)
+            filename: _asset_object(
+                job_id=job_id, user_id=user_id, filename=filename, asset_metadata=asset_metadata, job_info=job_info
+            )
             for filename, asset_metadata in result_assets.items()
             if asset_metadata.get("asset", True)
         }
@@ -1211,7 +1214,7 @@ def job_result_item_url(item_id) -> str:
                     "type": "Collection",
                     "stac_version": "1.0.0",
                     "stac_extensions": [
-                        STAC_EXTENSION.EO,
+                        STAC_EXTENSION.EO_V110,
                         STAC_EXTENSION.FILEINFO,
                         STAC_EXTENSION.PROCESSING,
                         STAC_EXTENSION.PROJECTION,
@@ -1274,8 +1277,8 @@ def job_result_item_url(item_id) -> str:
                 STAC_EXTENSION.FILEINFO,
             ]
 
-            if any("eo:bands" in asset_object for asset_object in result["assets"].values()):
-                result["stac_extensions"].append(STAC_EXTENSION.EO)
+            if sniff_stac_extension_prefix(result["assets"].values(), prefix="eo:"):
+                result["stac_extensions"].append(STAC_EXTENSION.EO_V110)
 
             if any(key.startswith("proj:") for key in result["properties"]) or any(
                 key.startswith("proj:") for key in result["assets"]
@@ -1439,9 +1442,9 @@ def _get_job_result_item(job_id, item_id, user_id):
             "type": "Feature",
             "stac_version": "1.0.0",
             "stac_extensions": [
-                STAC_EXTENSION.EO,
+                STAC_EXTENSION.EO_V110,
                 STAC_EXTENSION.FILEINFO,
-                STAC_EXTENSION.PROJECTION
+                STAC_EXTENSION.PROJECTION,
             ],
             "id": item_id,
             "geometry": geometry,
@@ -1514,6 +1517,7 @@ def _asset_object(job_id, user_id, filename: str, asset_metadata: dict, job_info
                 ),
                 "type": asset_metadata.get("type", asset_metadata.get("media_type", "application/octet-stream")),
                 "roles": asset_metadata.get("roles", ["data"]),
+                # TODO: eliminate this legacy "raster:bands" construct at some point?
                 "raster:bands": asset_metadata.get("raster:bands"),
                 "file:size": asset_metadata.get("file:size"),
                 "alternate": asset_metadata.get("alternate"),
@@ -1524,20 +1528,31 @@ def _asset_object(job_id, user_id, filename: str, asset_metadata: dict, job_info
             return result_dict
         bands = asset_metadata.get("bands")
 
+        if bands:
+            # TODO: eliminate this legacy "eo:bands" construct at some point?
+            result_dict["eo:bands"] = [
+                dict_no_none(
+                    {
+                        "name": band.name,
+                        "center_wavelength": band.wavelength_um,
+                    }
+                )
+                for band in bands
+            ]
+            # TODO: "bands" is a STAC>=1.1 feature, but here we don't know what version we are in.
+            result_dict["bands"] = [
+                dict_no_none(
+                    {
+                        "name": band.name,
+                        "eo:center_wavelength": band.wavelength_um,
+                    }
+                )
+                for band in bands
+            ]
+
         result_dict.update(
             dict_no_none(
                 **{
-                    "eo:bands": [
-                        dict_no_none(
-                            **{
-                                "name": band.name,
-                                "center_wavelength": band.wavelength_um,
-                            }
-                        )
-                        for band in bands
-                    ]
-                    if bands
-                    else None,
                     "proj:bbox": asset_metadata.get("proj:bbox", job_info.proj_bbox),
                     "proj:epsg": asset_metadata.get("proj:epsg", job_info.epsg),
                     "proj:shape": asset_metadata.get("proj:shape", job_info.proj_shape),
@@ -1835,7 +1850,9 @@ def _normalize_collection_metadata(metadata: dict, api_version: ComparableVersio
     # Version dependent metadata conversions
     cube_dims_100 = deep_get(metadata, "cube:dimensions", default=None)
     cube_dims_040 = deep_get(metadata, "properties", "cube:dimensions", default=None)
+    bands_110 = deep_get(metadata, "summaries", "bands", default=None)
     eo_bands_100 = deep_get(metadata, "summaries", "eo:bands", default=None)
+    # TODO do we still need normalization of openEO 0.4 style eo:bands?
     eo_bands_040 = deep_get(metadata, "properties", "eo:bands", default=None)
     extent_spatial_100 = deep_get(metadata, "extent", "spatial", "bbox", default=None)
     extent_spatial_040 = deep_get(metadata, "extent", "spatial", default=None)
@@ -1845,6 +1862,19 @@ def _normalize_collection_metadata(metadata: dict, api_version: ComparableVersio
     if full and not cube_dims_100 and cube_dims_040:
         _log.warning("Collection metadata 'cube:dimensions' in API 0.4 style instead of 1.0 style")
         metadata["cube:dimensions"] = cube_dims_040
+    if full and not bands_110 and eo_bands_100:
+        _log.warning("_normalize_collection_metadata: converting eo:bands to bands metadata")
+        # TODO #298/#363: "bands" is a STAC>=1.1 feature, but here we don't know what version we are in.
+        metadata["summaries"]["bands"] = [
+            dict_no_none(
+                {
+                    "name": b.get("name"),
+                    "eo:common_name": b.get("common_name"),
+                    "eo:center_wavelength": b.get("center_wavelength"),
+                }
+            )
+            for b in eo_bands_100
+        ]
     if full and not eo_bands_100 and eo_bands_040:
         _log.warning("Collection metadata 'eo:bands' in API 0.4 style instead of 1.0 style")
         metadata.setdefault("summaries", {})
@@ -1872,13 +1902,14 @@ def _normalize_collection_metadata(metadata: dict, api_version: ComparableVersio
                     dim["extent"] = interval
 
     # Make sure some required fields are set.
+    # TODO #363 bump stac_version default to 1.0.0 or even 1.1.0?
     metadata.setdefault("stac_version", "0.9.0")
     metadata.setdefault(
         "stac_extensions",
         [
             # TODO: enable these extensions only when necessary?
             STAC_EXTENSION.DATACUBE,
-            STAC_EXTENSION.EO,
+            STAC_EXTENSION.EO_V110,
         ],
     )
 
diff --git a/tests/test_save_result.py b/tests/test_save_result.py
@@ -7,6 +7,7 @@
 import numpy as np
 import pytest
 from shapely.geometry import GeometryCollection, Polygon
+import dirty_equals
 
 from openeo.metadata import CollectionMetadata
 from openeo_driver.datacube import DriverVectorCube
@@ -181,8 +182,6 @@ def test_aggregate_polygon_result_inconsistent_bands():
 
 
 def test_aggregate_polygon_result_CSV(tmp_path):
-
-
     metadata = CollectionMetadata({
         "cube:dimensions": {
             "x": {"type": "spatial"},
@@ -196,28 +195,66 @@ def test_aggregate_polygon_result_CSV(tmp_path):
         Polygon([(6, 1), (1, 7), (9, 9)])
     ])
 
-
     result = AggregatePolygonResultCSV(csv_dir=Path(__file__).parent / "data" /"aggregate_spatial_spacetime_cube", regions=regions_with_nonexistant, metadata=metadata)
     result.set_format("json")
 
     assets = result.write_assets(tmp_path / "ignored")
-    theAsset = assets.popitem()[1]
-    filename = theAsset['href']
+    [(_, asset_metadata)] = assets.items()
 
-    assert 'application/json' == theAsset['type']
-    assert ["red", "green", "blue"] == [b['name'] for b in theAsset['bands']]
-    assert 'raster:bands' in theAsset
-    assert 'file:size' in theAsset
+    assert "application/json" == asset_metadata["type"]
+    assert asset_metadata["bands"] == [
+        dirty_equals.IsPartialDict(name="red"),
+        dirty_equals.IsPartialDict(name="green"),
+        dirty_equals.IsPartialDict(name="blue"),
+    ]
+    assert asset_metadata["raster:bands"] == [
+        {
+            "statistics": {
+                "minimum": pytest.approx(4646, rel=0.1),
+                "mean": pytest.approx(4646, rel=0.1),
+                "maximum": pytest.approx(4646, rel=0.1),
+                "stddev": pytest.approx(0.31, rel=0.1),
+                "valid_percent": 100.0,
+            }
+        },
+        {
+            "statistics": {
+                "minimum": pytest.approx(4865, rel=0.1),
+                "mean": pytest.approx(4865, rel=0.1),
+                "maximum": pytest.approx(4865, rel=0.1),
+                "stddev": pytest.approx(0.265, rel=0.1),
+                "valid_percent": 100.0,
+            }
+        },
+        {
+            "statistics": {
+                "minimum": pytest.approx(5178, rel=0.1),
+                "mean": pytest.approx(5178, rel=0.1),
+                "maximum": pytest.approx(5178, rel=0.1),
+                "stddev": pytest.approx(0.41, rel=0.1),
+                "valid_percent": 100.0,
+            }
+        },
+    ]
 
-    assert 'mean' in theAsset['raster:bands'][0]["statistics"]
-    assert 'minimum' in theAsset['raster:bands'][0]["statistics"]
-    assert 100.0 == theAsset['raster:bands'][0]["statistics"]['valid_percent']
+    assert "file:size" in asset_metadata
 
-    expected = {'2017-09-05T00:00:00Z': [[4646.262612301313, 4865.926572218383, 5178.517363510712], [None, None, None], [4645.719597475695, 4865.467252259935, 5177.803342998465]], '2017-09-06T00:00:00Z': [[None, None, None], [None, None, None], [4645.719597475695, 4865.467252259935, 5177.803342998465]]}
+    filename = asset_metadata["href"]
+    expected = {
+        "2017-09-05T00:00:00Z": [
+            [4646.262612301313, 4865.926572218383, 5178.517363510712],
+            [None, None, None],
+            [4645.719597475695, 4865.467252259935, 5177.803342998465],
+        ],
+        "2017-09-06T00:00:00Z": [
+            [None, None, None],
+            [None, None, None],
+            [4645.719597475695, 4865.467252259935, 5177.803342998465],
+        ],
+    }
     with open(filename) as f:
-
         timeseries_ds = json.load(f)
-        assert expected == timeseries_ds
+    assert timeseries_ds == expected
 
 class TestAggregatePolygonSpatialResult:
 
diff --git a/tests/test_views.py b/tests/test_views.py
diff --git a/tests/util/test_stac.py b/tests/util/test_stac.py
