Started addressing #20 and #21

Author: forman

tests/test_api.py

@@ -5,6 +5,7 @@
 import shutil
 import unittest
 
+import numpy as np
 import xarray as xr
 
 from zappend.api import FileObj
@@ -14,6 +15,7 @@
 from .helpers import make_test_dataset
 
 
+# noinspection PyMethodMayBeStatic
 class ApiTest(unittest.TestCase):
     def setUp(self):
         clear_memory_fs()
@@ -73,6 +75,79 @@ def process_slice(ctx, slice_ds: xr.Dataset) -> SliceSource:
         self.assertEqual({"chl"}, set(ds.data_vars))
         self.assertEqual({"time", "y", "x"}, set(ds.coords))
 
+    def test_some_slices_with_inc_append_step(self):
+        target_dir = "memory://target.zarr"
+        slices = [
+            make_test_dataset(index=0, shape=(1, 50, 100)),
+            make_test_dataset(index=1, shape=(1, 50, 100)),
+            make_test_dataset(index=2, shape=(1, 50, 100)),
+        ]
+        zappend(slices, target_dir=target_dir, append_step="1d")
+        ds = xr.open_zarr(target_dir)
+        np.testing.assert_array_equal(
+            ds.time.values,
+            np.array(["2024-01-01", "2024-01-02", "2024-01-03"], dtype=np.datetime64),
+        )
+
+    def test_some_slices_with_dec_append_step(self):
+        target_dir = "memory://target.zarr"
+        slices = [
+            make_test_dataset(index=2, shape=(1, 50, 100)),
+            make_test_dataset(index=1, shape=(1, 50, 100)),
+            make_test_dataset(index=0, shape=(1, 50, 100)),
+        ]
+        zappend(slices, target_dir=target_dir, append_step="-1d")
+        ds = xr.open_zarr(target_dir)
+        np.testing.assert_array_equal(
+            ds.time.values,
+            np.array(["2024-01-03", "2024-01-02", "2024-01-01"], dtype=np.datetime64),
+        )
+
+    def test_some_slices_with_one_missing_append_step(self):
+        target_dir = "memory://target.zarr"
+        slices = [
+            make_test_dataset(index=0, shape=(1, 50, 100)),
+            make_test_dataset(index=2, shape=(1, 50, 100)),
+        ]
+        zappend(slices, target_dir=target_dir, append_step="1d")
+        ds = xr.open_zarr(target_dir)
+        np.testing.assert_array_equal(
+            ds.time.values,
+            np.array(["2024-01-01", "2024-01-02", "2024-01-03"], dtype=np.datetime64),
+        )
+
+    def test_some_slices_with_three_missing_append_steps(self):
+        target_dir = "memory://target.zarr"
+        slices = [
+            make_test_dataset(index=0, shape=(1, 50, 100)),
+            make_test_dataset(index=4, shape=(1, 50, 100)),
+        ]
+        zappend(slices, target_dir=target_dir, append_step="1d")
+        ds = xr.open_zarr(target_dir)
+        np.testing.assert_array_equal(
+            ds.time.values,
+            np.array(
+                ["2024-01-01", "2024-01-02", "2024-01-03", "2024-01-04", "2024-01-05"],
+                dtype=np.datetime64,
+            ),
+        )
+
+    def test_it_raises_for_wrong_append_step(self):
+        # TODO: implement me
+        pass
+
+    def test_some_slices_with_inc_append_labels(self):
+        # TODO: implement me
+        pass
+
+    def test_some_slices_with_dec_append_labels(self):
+        # TODO: implement me
+        pass
+
+    def test_it_raises_for_none_inc_append_labels(self):
+        # TODO: implement me
+        pass
+
     def test_some_slices_with_profiling(self):
         target_dir = "memory://target.zarr"
         slices = [

tests/test_config.py

@@ -233,13 +233,16 @@ def test_exclude_from_config(self):
         with exclude_from_config({"a": 1, "b": 2}, "b", "a") as config:
             self.assertEqual({}, config)
 
+
+class ConfigSchemaTest(unittest.TestCase):
     def test_get_config_schema(self):
         schema = get_config_schema()
         self.assertIn("properties", schema)
         self.assertIsInstance(schema["properties"], dict)
         self.assertEqual(
             {
                 "append_dim",
+                "append_step",
                 "disable_rollback",
                 "dry_run",
                 "excluded_variables",

tests/test_context.py

@@ -5,8 +5,9 @@
 import unittest
 
 import pytest
+import numpy as np
 import xarray as xr
-from zappend.api import zappend
+
 from zappend.context import Context
 from zappend.fsutil.fileobj import FileObj
 from zappend.metadata import DatasetMetadata
@@ -37,20 +38,32 @@ def test_with_existing_target(self):
     def test_append_dim(self):
         ctx = Context({"target_dir": "memory://target.zarr"})
         self.assertEqual("time", ctx.append_dim_name)
-
         ctx = Context({"target_dir": "memory://target.zarr", "append_dim": "depth"})
         self.assertEqual("depth", ctx.append_dim_name)
 
+    def test_append_step(self):
+        make_test_dataset(uri="memory://target.zarr")
+        ctx = Context({"target_dir": "memory://target.zarr"})
+        self.assertEqual(None, ctx.append_step_size)
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "1d"})
+        self.assertEqual("1d", ctx.append_step_size)
+
+    def test_last_append_label(self):
+        make_test_dataset(uri="memory://target.zarr")
+        ctx = Context({"target_dir": "memory://target.zarr"})
+        self.assertEqual(None, ctx.last_append_label)
+        ctx = Context({"target_dir": "memory://TARGET.zarr", "append_step": "1d"})
+        self.assertEqual(None, ctx.last_append_label)
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "1d"})
+        self.assertEqual(np.datetime64("2024-01-03"), ctx.last_append_label)
+
     def test_slice_polling(self):
         ctx = Context({"target_dir": "memory://target.zarr"})
         self.assertEqual((None, None), ctx.slice_polling)
-
         ctx = Context({"target_dir": "memory://target.zarr", "slice_polling": False})
         self.assertEqual((None, None), ctx.slice_polling)
-
         ctx = Context({"target_dir": "memory://target.zarr", "slice_polling": True})
         self.assertEqual((2, 60), ctx.slice_polling)
-
         ctx = Context(
             {
                 "target_dir": "memory://target.zarr",

zappend/config/__init__.py

@@ -3,13 +3,14 @@
 # https://opensource.org/licenses/MIT.
 
 from .config import ConfigItem
-from .config import ConfigList
 from .config import ConfigLike
+from .config import ConfigList
 from .config import exclude_from_config
 from .config import merge_configs
 from .config import normalize_config
 from .config import validate_config
 from .defaults import DEFAULT_APPEND_DIM
+from .defaults import DEFAULT_APPEND_STEP
 from .defaults import DEFAULT_SLICE_POLLING_INTERVAL
 from .defaults import DEFAULT_SLICE_POLLING_TIMEOUT
 from .defaults import DEFAULT_ZARR_VERSION

zappend/config/defaults.py

@@ -4,6 +4,7 @@
 
 DEFAULT_ZARR_VERSION = 2
 DEFAULT_APPEND_DIM = "time"
+DEFAULT_APPEND_STEP = None
 
 DEFAULT_SLICE_POLLING_INTERVAL = 2
 DEFAULT_SLICE_POLLING_TIMEOUT = 60

zappend/config/schema.py

@@ -6,6 +6,7 @@
 from typing import Any, Literal
 
 from .defaults import DEFAULT_APPEND_DIM
+from .defaults import DEFAULT_APPEND_STEP
 from .defaults import DEFAULT_SLICE_POLLING_INTERVAL
 from .defaults import DEFAULT_SLICE_POLLING_TIMEOUT
 from .defaults import DEFAULT_ZARR_VERSION
@@ -473,6 +474,31 @@
             "minLength": 1,
             "default": DEFAULT_APPEND_DIM,
         },
+        append_step={
+            "description": (
+                "If set, enforces a step size in the append dimension between two"
+                " slices or just enforces a direction."
+            ),
+            "oneOf": [
+                {
+                    "description": "Arbitrary step size or not applicable.",
+                    "const": None,
+                },
+                {"description": "Monotonically increasing.", "const": "+"},
+                {"description": "Monotonically decreasing.", "const": "-"},
+                {
+                    "description": "A time delta value.",
+                    "type": "string",
+                    "not": {"const": ""},
+                },
+                {
+                    "description": "A numerical delta value.",
+                    "type": "number",
+                    "not": {"const": 0},
+                },
+            ],
+            "default": DEFAULT_APPEND_STEP,
+        },
         included_variables={
             "description": (
                 "Specifies the names of variables to be included in"

zappend/context.py

@@ -9,6 +9,7 @@
 import xarray as xr
 
 from .config import DEFAULT_APPEND_DIM
+from .config import DEFAULT_APPEND_STEP
 from .config import DEFAULT_SLICE_POLLING_INTERVAL
 from .config import DEFAULT_SLICE_POLLING_TIMEOUT
 from .config import DEFAULT_ZARR_VERSION
@@ -36,10 +37,18 @@ def __init__(self, config: Dict[str, Any]):
         target_storage_options = config.get("target_storage_options")
         self._target_dir = FileObj(target_uri, storage_options=target_storage_options)
 
+        self._append_dim_name = config.get("append_dim") or DEFAULT_APPEND_DIM
+        self._append_step_size = config.get("append_step") or DEFAULT_APPEND_STEP
+        self._last_append_label = None
+
         try:
             with xr.open_zarr(
                 target_uri, storage_options=target_storage_options
             ) as target_dataset:
+                if self.append_step_size is not None:
+                    append_var = target_dataset.get(self._append_dim_name)
+                    if append_var is not None and append_var.size > 0:
+                        self._last_append_label = append_var[-1]
                 target_metadata = DatasetMetadata.from_dataset(target_dataset, config)
         except FileNotFoundError:
             target_metadata = None
@@ -74,10 +83,24 @@ def zarr_version(self) -> int:
     @property
     def append_dim_name(self) -> str:
         """The name of the append dimension along which slice datasets will be
-        concatenated.
-        If not configured, it defaults to `"time"`.
+        concatenated. Defaults to `"time"`.
+        """
+        return self._append_dim_name
+
+    @property
+    def append_step_size(self) -> int | float | str | None:
+        """The enforced step size in the append dimension between two slices.
+        Defaults to `None`.
+        """
+        return self._append_step_size
+
+    @property
+    def last_append_label(self) -> Any | None:
+        """The last label found in the coordinate variable that corresponds to
+        the append dimension. Its value is `None` if no such variable exists or the
+        variable is empty or if [append_step_size][append_step_size] is `None`.
         """
-        return self._config.get("append_dim") or DEFAULT_APPEND_DIM
+        return self._last_append_label
 
     @property
     def target_metadata(self) -> DatasetMetadata | None: