feat/use new multiscales (#75)

* add option for using new multiscales convention in optimized conversion

* remove OOP converter and use plain functions

* use pydantic models for fingerprinting sentinel2 product

* fix multiscales to use da.coarsen and propagate encoding

* ensure that dtype is preserved after resampling

* add new multiscales JSON example

* add mypy pydantic plugin

* lint

* add s1 and s2 demo data to tests, and don't test against remote urls

* fix e2e tests

* remove network test workflow from CI

* remove extra type definition and update tests

* remove explicit zarr groups in favor of dynamic test fixtures

* docstrings

* Enhance CRS initialization and update S2 optimization commands

- Added `initialize_crs_from_dataset` function to extract CRS from dataset metadata.
- Updated S2 optimization commands to include new CRS handling.
- Removed unused arguments related to geometry and meteorology groups.
- Added comprehensive tests for CRS initialization from various sources.

* Refactor code formatting for clarity in S2 optimization functions

* fix failing / warning tests

* add strict JSON schema equality check to e2e tests

* support both flavors of multiscale metadata

* dont manage return codes in cli functions

* add s2 optimized test

* add optimized geozarr exmaple hierarchies

* format JSON documents

* mid-debug of e2e tests

* WIP e2e fixes

* make cf standard name validator become a pass-through when no internet connection

* update example schemas

* narrow type to just tuples in types.py

* refactor consolidation

* use consolidated=False in conversion

* update tests

* lint

* add both multiscales types to output

* update comments in tests

---------

Co-authored-by: Emmanuel Mathot <[email protected]>

Author: d-v-b

.github/workflows/ci.yml

@@ -70,44 +70,6 @@ jobs:
           .venv
         key: uv-main-${{ hashFiles('uv.lock') }}
 
-  test-network:
-    runs-on: ubuntu-latest
-    # Temporarily enabled for PR to verify test fix
-    if: github.event_name == 'push' || github.event_name == 'pull_request'
-    steps:
-    - uses: actions/checkout@v5
-
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.11'
-
-    - name: Restore global uv cache
-      id: cache-restore
-      uses: actions/cache/restore@v4
-      with:
-        path: |
-          ~/.cache/uv
-          ~/.local/share/uv
-          .venv
-        key: uv-main-${{ hashFiles('uv.lock') }}
-        restore-keys: |
-          uv-main-
-
-    - name: Install uv
-      uses: astral-sh/setup-uv@v7
-      with:
-        version: "0.8.4"
-        python-version: "3.13"
-        enable-cache: false
-
-    - name: Install dependencies
-      run: uv sync --group dev --group test
-
-    - name: Run network tests
-      run: |
-        uv run pytest tests/ -v --tb=short -m "network"
-
   security:
     runs-on: ubuntu-latest
     steps:

.vscode/launch.json

@@ -168,10 +168,12 @@
             "module": "eopf_geozarr",
             "args": [
                 "convert-s2-optimized",
-                "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202509-s02msil2a/08/products/cpm_v256/S2A_MSIL2A_20250908T100041_N0511_R122_T32TQM_20250908T115116.zarr",
+                // "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202509-s02msil2a/08/products/cpm_v256/S2A_MSIL2A_20250908T100041_N0511_R122_T32TQM_20250908T115116.zarr",
+                "https://objects.eodc.eu/e05ab01a9d56408d82ac32d69a5aae2a:202511-s02msil2a-eu/15/products/cpm_v262/S2B_MSIL2A_20251115T091139_N0511_R050_T35SLU_20251115T111807.zarr",
                 // "s3://esa-zarr-sentinel-explorer-fra/tests-output/sentinel-2-l2a-opt/S2A_MSIL2A_20250908T100041_N0511_R122_T32TQM_20250908T115116.zarr",
-                "./tests-output/eopf_geozarr/s2l2_optimized.zarr",
-                "--spatial-chunk", "1024",
+                "s3://esa-zarr-sentinel-explorer-fra/tests-output/sentinel-2-l2a-pr75/S2B_MSIL2A_20251115T091139_N0511_R050_T35SLU_20251115T111807.zarr",
+                // "./tests-output/eopf_geozarr/s2l2_optimized.zarr",
+                "--spatial-chunk", "512",
                 "--compression-level", "5",
                 "--enable-sharding",
                 "--dask-cluster",

pyproject.toml

@@ -129,6 +129,13 @@ warn_unused_ignores = true
 warn_no_return = true
 warn_unreachable = true
 strict_equality = true
+plugins = ["pydantic.mypy"]
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true
+warn_untyped_fields = true
 
 [[tool.mypy.overrides]]
 module = ["zarr.*", "xarray.*", "rioxarray.*", "cf_xarray.*", "dask.*"]

src/eopf_geozarr/cli.py

@@ -1182,14 +1182,6 @@ def add_s2_optimization_commands(subparsers: Any) -> None:
         choices=range(1, 10),
         help="Compression level 1-9 (default: 3)",
     )
-    s2_parser.add_argument(
-        "--skip-geometry", action="store_true", help="Skip creating geometry group"
-    )
-    s2_parser.add_argument(
-        "--skip-meteorology",
-        action="store_true",
-        help="Skip creating meteorology group",
-    )
     s2_parser.add_argument(
         "--skip-validation", action="store_true", help="Skip output validation"
     )
@@ -1204,7 +1196,7 @@ def add_s2_optimization_commands(subparsers: Any) -> None:
     s2_parser.set_defaults(func=convert_s2_optimized_command)
 
 
-def convert_s2_optimized_command(args: Any) -> int:
+def convert_s2_optimized_command(args: Any) -> None:
     """Execute S2 optimized conversion command."""
     # Set up dask cluster if requested
     dask_client = setup_dask_cluster(
@@ -1229,22 +1221,10 @@ def convert_s2_optimized_command(args: Any) -> int:
             enable_sharding=args.enable_sharding,
             spatial_chunk=args.spatial_chunk,
             compression_level=args.compression_level,
-            create_geometry_group=not args.skip_geometry,
-            create_meteorology_group=not args.skip_meteorology,
             validate_output=not args.skip_validation,
-            verbose=args.verbose,
         )
 
         log.info("✅ S2 optimization completed", output_path=args.output_path)
-        return 0
-
-    except Exception as e:
-        log.info("❌ Error during S2 optimization", error=str(e))
-        if args.verbose:
-            import traceback
-
-            traceback.print_exc()
-        return 1
     finally:
         # Clean up dask client if it was created
         if dask_client is not None:

src/eopf_geozarr/conversion/geozarr.py

@@ -128,6 +128,8 @@ def create_geozarr_dataset(
         dt, groups, gcp_group
     )
 
+    log.info("GeoZarr groups prepared", groups_prepared=list(geozarr_groups.keys()))
+
     # Create the GeoZarr compliant store through iterative processing
     dt_geozarr = iterative_copy(
         dt,
@@ -180,11 +182,19 @@ def setup_datatree_metadata_geozarr_spec_compliant(
         epsg_CPM_260 = epsg_CPM_260.split(":")[-1]
 
     for key in groups:
-        if not dt[key].data_vars:
+        # Check if key exists in DataTree by attempting to access it
+        try:
+            node = dt[key]
+        except KeyError:
+            log.info(f"Skipping group {key} - not found in DataTree")
+            continue
+
+        if not node.data_vars:
+            log.info(f"Skipping group {key} - no data variables")
             continue
 
         log.info(f"Processing group {key} for GeoZarr compliance")
-        ds = dt[key].to_dataset().copy()
+        ds = node.to_dataset().copy()
 
         if gcp_group is not None:
             ds_gcp = dt[gcp_group].to_dataset()
@@ -233,7 +243,11 @@ def setup_datatree_metadata_geozarr_spec_compliant(
         _setup_grid_mapping(ds, grid_mapping_var_name)
 
         geozarr_groups[key] = ds
+        log.info(f"Added {key} to geozarr_groups")
 
+    log.info(
+        f"Returning geozarr_groups with {len(geozarr_groups)} groups: {list(geozarr_groups.keys())}"
+    )
     return geozarr_groups
 
 
@@ -287,7 +301,7 @@ def iterative_copy(
     dt_result.to_zarr(
         output_path,
         mode="a",
-        consolidated=True,
+        consolidated=False,
         compute=True,
         storage_options=storage_options,
     )
@@ -638,7 +652,7 @@ def create_geozarr_compliant_multiscales(
             level=ol["level"],
             width=ol["width"],
             height=ol["height"],
-            scale_factor=ol["scale_factor"],
+            scale_factor=ol["scale_relative"],
         )
 
     # Create native CRS tile matrix set
@@ -679,7 +693,7 @@ def create_geozarr_compliant_multiscales(
 
         width = overview["width"]
         height = overview["height"]
-        scale_factor = overview["scale_factor"]
+        scale_factor = overview["scale_relative"]
 
         log.info(
             f"Creating overview level (scale) {level} with scale factor {scale_factor}"
@@ -733,7 +747,7 @@ def create_geozarr_compliant_multiscales(
             output_path,
             group=overview_group,
             mode="w",
-            consolidated=True,
+            consolidated=False,
             zarr_format=3,
             encoding=encoding,
             align_chunks=align_chunks_flag,
@@ -824,7 +838,9 @@ def calculate_overview_levels(
             "zoom": zoom,
             "width": current_width,
             "height": current_height,
-            "scale_factor": 2**level,
+            "translation_relative": 0.0,
+            "scale_absolute": 1.0,
+            "scale_relative": 2**level,
         }
         overview_levels.append(overview_level)  # type: ignore[arg-type]
 
@@ -877,8 +893,8 @@ def create_native_crs_tile_matrix_set(
         scale_denominator = cell_size * 3779.5275
 
         # Calculate matrix dimensions
-        tile_width = overview["chunks"][1][0] if "chunks" in overview else 256
-        tile_height = overview["chunks"][0][0] if "chunks" in overview else 256
+        tile_width = overview["chunks"][1][0] if "chunks" in overview else 256  # type: ignore[index]
+        tile_height = overview["chunks"][0][0] if "chunks" in overview else 256  # type: ignore[index]
         matrix_width = int(np.ceil(width / tile_width))
         matrix_height = int(np.ceil(height / tile_height))
 
@@ -889,7 +905,7 @@ def create_native_crs_tile_matrix_set(
                 "id": matrix_id,
                 "scaleDenominator": scale_denominator,
                 "cellSize": cell_size,
-                "pointOfOrigin": [left, top],
+                "pointOfOrigin": (left, top),
                 "tileWidth": tile_width,
                 "tileHeight": tile_height,
                 "matrixWidth": matrix_width,
@@ -910,8 +926,8 @@ def create_native_crs_tile_matrix_set(
         "title": f"Native CRS Tile Matrix Set ({native_crs})",
         "crs": crs_uri,
         "supportedCRS": crs_uri,
-        "orderedAxes": ["X", "Y"],
-        "tileMatrices": tile_matrices,
+        "orderedAxes": ("X", "Y"),
+        "tileMatrices": tuple(tile_matrices),
     }
 
 
@@ -1802,3 +1818,7 @@ def _is_sentinel1(dt: xr.DataTree) -> bool:
         return True
     else:
         return False
+
+
+def get_zarr_group(data: xr.DataTree) -> zarr.Group:
+    return data._close.__self__.zarr_group

src/eopf_geozarr/data_api/geozarr/common.py

@@ -4,7 +4,8 @@
 import urllib
 import urllib.request
 from dataclasses import dataclass
-from typing import Annotated, Any, Mapping, Self, TypeVar
+from typing import Annotated, Any, Mapping, Self, TypeGuard, TypeVar
+from urllib.error import URLError
 
 from cf_xarray.utils import parse_cf_standard_name_table
 from pydantic import AfterValidator, BaseModel, Field, model_validator
@@ -98,12 +99,9 @@ def get_cf_standard_names(url: str) -> tuple[str, ...]:
 
     req = urllib.request.Request(url, headers=headers)
 
-    try:
-        with urllib.request.urlopen(req) as response:
-            content = response.read()  # Read the entire response body into memory
-            content_fobj = io.BytesIO(content)
-    except urllib.error.URLError as e:
-        raise e
+    with urllib.request.urlopen(req) as response:
+        content = response.read()  # Read the entire response body into memory
+        content_fobj = io.BytesIO(content)
 
     _info, table, _aliases = parse_cf_standard_name_table(source=content_fobj)
     return tuple(table.keys())
@@ -117,7 +115,13 @@ def get_cf_standard_names(url: str) -> tuple[str, ...]:
 
 # this does IO against github. consider locally storing this data instead if fetching every time
 # is problematic.
-CF_STANDARD_NAMES = get_cf_standard_names(url=CF_STANDARD_NAME_URL)
+
+try:
+    CF_STANDARD_NAMES = get_cf_standard_names(url=CF_STANDARD_NAME_URL)
+    DO_CF_NAME_VALIDATION = True
+except URLError:
+    CF_STANDARD_NAMES = ()
+    DO_CF_NAME_VALIDATION = False
 
 
 def check_standard_name(name: str) -> str:
@@ -139,12 +143,13 @@ def check_standard_name(name: str) -> str:
     ValueError
         If the standard name is not valid.
     """
-
-    if name in CF_STANDARD_NAMES:
-        return name
-    raise ValueError(
-        f"Invalid standard name: {name}. This name was not found in the list of CF standard names."
-    )
+    if DO_CF_NAME_VALIDATION:
+        if name in CF_STANDARD_NAMES:
+            return name
+        raise ValueError(
+            f"Invalid standard name: {name}. This name was not found in the list of CF standard names."
+        )
+    return name
 
 
 CFStandardName = Annotated[str, AfterValidator(check_standard_name)]
@@ -245,9 +250,9 @@ class TileMatrixSet(BaseModel):
     tileMatrices: tuple[TileMatrix, ...]
 
 
-class Multiscales(BaseModel, extra="allow"):
+class TMSMultiscales(BaseModel, extra="allow"):
     """
-    Multiscale metadata for a GeoZarr dataset.
+    Multiscale metadata for a GeoZarr dataset based on the OGC TileMatrixSet standard
 
     Attributes
     ----------
@@ -307,4 +312,8 @@ class MultiscaleGroupAttrs(BaseModel, extra="allow"):
     multiscales: MultiscaleAttrs
     """
 
-    multiscales: Multiscales
+    multiscales: TMSMultiscales
+
+
+def is_none(data: object) -> TypeGuard[None]:
+    return data is None

src/eopf_geozarr/data_api/geozarr/geoproj.py

@@ -4,18 +4,14 @@
 
 from __future__ import annotations
 
-from typing import Literal, Self, TypeGuard
+from typing import Literal, Self
 
 from pydantic import BaseModel, Field, model_validator
 from typing_extensions import TypedDict
 
+from eopf_geozarr.data_api.geozarr.common import is_none
 from eopf_geozarr.data_api.geozarr.projjson import ProjJSON
 
-
-def is_none(data: object) -> TypeGuard[None]:
-    return data is None
-
-
 GEO_PROJ_UUID: Literal["f17cb550-5864-4468-aeb7-f3180cfb622f"] = (
     "f17cb550-5864-4468-aeb7-f3180cfb622f"
 )