Merge branch 'main' into forman-47-example_nb

Author: forman

CHANGES.md

@@ -2,6 +2,10 @@
 
 ### Fixes
 
+* When `slice_source` was given as class or function and passed  
+  to the `zappend()` function either as configuration entry or as keyword 
+  argument, a `ValidationError` was accidentally raised. [#49]
+
 * Fixed an issue where an absolute lock file path was computed if the target 
   Zarr path was relative in the local filesystem, and had no parent directory.
   [#45]

tests/test_api.py

@@ -6,8 +6,10 @@
 import unittest
 
 import xarray as xr
+
+from zappend.api import FileObj
+from zappend.api import SliceSource
 from zappend.api import zappend
-from zappend.fsutil.fileobj import FileObj
 from .helpers import clear_memory_fs
 from .helpers import make_test_dataset
 
@@ -50,6 +52,27 @@ def test_some_slices_local(self):
             for slice_dir in slices:
                 shutil.rmtree(slice_dir, ignore_errors=True)
 
+    def test_some_slices_with_class_slice_source(self):
+        target_dir = "memory://target.zarr"
+        slices = [make_test_dataset(), make_test_dataset(), make_test_dataset()]
+        zappend(slices, target_dir=target_dir, slice_source=MySliceSource)
+        ds = xr.open_zarr(target_dir)
+        self.assertEqual({"time": 9, "y": 50, "x": 100}, ds.sizes)
+        self.assertEqual({"chl"}, set(ds.data_vars))
+        self.assertEqual({"time", "y", "x"}, set(ds.coords))
+
+    def test_some_slices_with_func_slice_source(self):
+        def process_slice(ctx, slice_ds: xr.Dataset) -> SliceSource:
+            return MySliceSource(ctx, slice_ds)
+
+        target_dir = "memory://target.zarr"
+        slices = [make_test_dataset(), make_test_dataset(), make_test_dataset()]
+        zappend(slices, target_dir=target_dir, slice_source=process_slice)
+        ds = xr.open_zarr(target_dir)
+        self.assertEqual({"time": 9, "y": 50, "x": 100}, ds.sizes)
+        self.assertEqual({"chl"}, set(ds.data_vars))
+        self.assertEqual({"time", "y", "x"}, set(ds.coords))
+
     def test_some_slices_with_profiling(self):
         target_dir = "memory://target.zarr"
         slices = [
@@ -74,3 +97,12 @@ def test_some_slices_with_profiling(self):
         finally:
             if os.path.exists("prof.out"):
                 os.remove("prof.out")
+
+
+class MySliceSource(SliceSource):
+    def __init__(self, ctx, slice_ds):
+        super().__init__(ctx)
+        self.slice_ds = slice_ds
+
+    def get_dataset(self) -> xr.Dataset:
+        return self.slice_ds.drop_vars(["tsm"])

tests/test_config.py

@@ -9,6 +9,7 @@
 import pytest
 import yaml
 
+from zappend.config import exclude_from_config
 from zappend.config import get_config_schema
 from zappend.config import merge_configs
 from zappend.config import normalize_config
@@ -220,6 +221,18 @@ def test_merge_config(self):
             merge_configs({"a": {"b": 2, "c": 4}}, {"a": {"b": 3}}),
         )
 
+    def test_exclude_from_config(self):
+        with exclude_from_config({}) as config:
+            self.assertEqual({}, config)
+        with exclude_from_config({"a": 1, "b": 2}) as config:
+            self.assertEqual({"a": 1, "b": 2}, config)
+        with exclude_from_config({}, "a") as config:
+            self.assertEqual({}, config)
+        with exclude_from_config({"a": 1, "b": 2}, "a") as config:
+            self.assertEqual({"b": 2}, config)
+        with exclude_from_config({"a": 1, "b": 2}, "b", "a") as config:
+            self.assertEqual({}, config)
+
     def test_get_config_schema(self):
         schema = get_config_schema()
         self.assertIn("properties", schema)

zappend/config/__init__.py

@@ -5,6 +5,7 @@
 from .config import ConfigItem
 from .config import ConfigList
 from .config import ConfigLike
+from .config import exclude_from_config
 from .config import merge_configs
 from .config import normalize_config
 from .config import validate_config

zappend/config/config.py

@@ -2,6 +2,7 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
+from contextlib import contextmanager
 import json
 import os.path
 from typing import Any
@@ -135,3 +136,8 @@ def _merge_values(value_1: Any, value_2: Any) -> Any:
     if isinstance(value_1, (list, tuple)) and isinstance(value_2, (list, tuple)):
         return _merge_lists(value_1, value_2)
     return value_2
+
+
+@contextmanager
+def exclude_from_config(config: dict[str, Any], *keys: str) -> dict[str, Any]:
+    yield {k: v for k, v in config.items() if k not in keys}

zappend/processor.py

@@ -7,6 +7,7 @@
 import xarray as xr
 
 from .config import ConfigLike
+from .config import exclude_from_config
 from .config import normalize_config
 from .config import validate_config
 from .context import Context
@@ -31,20 +32,24 @@ class Processor:
             the aforementioned. If a sequence is used, subsequent configurations
             are incremental to the previous ones.
         kwargs: Additional configuration parameters.
-            Can be used to pass or override configuration values in *config*.
+            Can be used to pass or override configuration values in `config`.
     """
 
     def __init__(self, config: ConfigLike = None, **kwargs):
         config = normalize_config(config)
         config.update({k: v for k, v in kwargs.items() if v is not None})
-        validate_config(config)
+        # Validate value of slice_source later,
+        # it could be a callable instead of a str
+        # See https://github.com/bcdev/zappend/issues/49
+        with exclude_from_config(config, "slice_source") as _config:
+            validate_config(_config)
         configure_logging(config.get("logging"))
         self._profiler = Profiler(config.get("profiling"))
         self._config = config
 
     def process_slices(self, slices: Iterable[SliceObj]):
-        """Process the given *slices*.
-        Passes each slice in *slices* to the `process_slice()` method.
+        """Process the given `slices`.
+        Passes each slice in `slices` to the `process_slice()` method.
 
         Args:
             slices: Slice objects.