new configuration setting append_step

forman · forman · commit 52da874e135e · 2024-02-05T18:41:45.000+01:00
diff --git a/CHANGES.md b/CHANGES.md
@@ -2,6 +2,16 @@
 
 ### Enhancements
 
+* A new configuration setting `append_step` can be used to validate
+  the step sizes between the labels of a coordinate variable associated with
+  the append dimension. Its value can be a number for numerical labels
+  or a time delta value of the form `8h` (8 hours) or `2D` (two days) for
+  date/time labels. The value can also be negative. [#21] 
+
+* The configuration setting `append_step` can take the special values
+  `"+"` and `"-"` which are used to verify that the labels are monotonically 
+  increasing or decreasing. [#20]
+
 * It is now possible to reference environment variables
   in configuration files using the syntax `${ENV_VAR}`. [#36]
 
diff --git a/docs/guide.md b/docs/guide.md
@@ -97,9 +97,27 @@ specify its name:
 }
 ```
 
-All other non-variadic dimensions can and should be specified using the
-`fixed_dims` setting which is a mapping from dimension name to the 
-fixed dimension size, e.g.:
+The configuration setting `append_step` can be used to validate the step sizes 
+between the labels of a coordinate variable associated with the append dimension. 
+Its value can be a number for numerical labels or a timedelta value of the form
+`<count><unit>` for date/time labels. In the latter case `<count>` is an integer 
+and `<units>` is one of the possible
+[numpy datetime units](https://numpy.org/doc/stable/reference/arrays.datetime.html#datetime-units), 
+for example, `8h` (8 hours) or `2D` (two days). Numerical and timedelta values
+may be negative. `append_step` can also take the two special values `"+"` and 
+`"-"`. In this case it is just verified that the append labels are monotonically 
+increasing or decreasing.
+
+```json
+{
+    "append_dim": "time",
+    "append_step": "2D"
+}
+```
+
+Other, non-variadic dimensions besides the append dimension can and should 
+be specified using the `fixed_dims` setting which is a mapping from dimension 
+name to the fixed dimension size, e.g.:
 
 ```json
 {
diff --git a/tests/test_api.py b/tests/test_api.py
@@ -6,6 +6,7 @@
 import unittest
 
 import numpy as np
+import pytest
 import xarray as xr
 
 from zappend.api import FileObj
@@ -82,7 +83,7 @@ def test_some_slices_with_inc_append_step(self):
             make_test_dataset(index=1, shape=(1, 50, 100)),
             make_test_dataset(index=2, shape=(1, 50, 100)),
         ]
-        zappend(slices, target_dir=target_dir, append_step="1d")
+        zappend(slices, target_dir=target_dir, append_step="1D")
         ds = xr.open_zarr(target_dir)
         np.testing.assert_array_equal(
             ds.time.values,
@@ -96,57 +97,118 @@ def test_some_slices_with_dec_append_step(self):
             make_test_dataset(index=1, shape=(1, 50, 100)),
             make_test_dataset(index=0, shape=(1, 50, 100)),
         ]
-        zappend(slices, target_dir=target_dir, append_step="-1d")
+        zappend(slices, target_dir=target_dir, append_step="-1D")
         ds = xr.open_zarr(target_dir)
         np.testing.assert_array_equal(
             ds.time.values,
             np.array(["2024-01-03", "2024-01-02", "2024-01-01"], dtype=np.datetime64),
         )
 
-    def test_some_slices_with_one_missing_append_step(self):
+    # # See https://github.com/bcdev/zappend/issues/21
+    #
+    # def test_some_slices_with_one_missing_append_step(self):
+    #     target_dir = "memory://target.zarr"
+    #     slices = [
+    #         make_test_dataset(index=0, shape=(1, 50, 100)),
+    #         make_test_dataset(index=2, shape=(1, 50, 100)),
+    #     ]
+    #     zappend(slices, target_dir=target_dir, append_step="1D")
+    #     ds = xr.open_zarr(target_dir)
+    #     np.testing.assert_array_equal(
+    #         ds.time.values,
+    #         np.array(
+    #             ["2024-01-01", "2024-01-02", "2024-01-03"], dtype="datetime64[ns]"
+    #         ),
+    #     )
+
+    # # See https://github.com/bcdev/zappend/issues/21
+    #
+    # def test_some_slices_with_three_missing_append_steps(self):
+    #     target_dir = "memory://target.zarr"
+    #     slices = [
+    #         make_test_dataset(index=0, shape=(1, 50, 100)),
+    #         make_test_dataset(index=4, shape=(1, 50, 100)),
+    #     ]
+    #     zappend(slices, target_dir=target_dir, append_step="1D")
+    #     ds = xr.open_zarr(target_dir)
+    #     np.testing.assert_array_equal(
+    #         ds.time.values,
+    #         np.array(
+    #             [
+    #                 "2024-01-01",
+    #                 "2024-01-02",
+    #                 "2024-01-03",
+    #                 "2024-01-04",
+    #                 "2024-01-05",
+    #             ],
+    #             dtype="datetime64[ns]",
+    #         ),
+    #     )
+
+    def test_it_raises_for_wrong_append_step(self):
+        target_dir = "memory://target.zarr"
+        slices = [
+            make_test_dataset(index=0, shape=(1, 50, 100)),
+            make_test_dataset(index=1, shape=(1, 50, 100)),
+        ]
+        with pytest.raises(
+            ValueError,
+            match=(
+                "Cannot append slice because this would result in"
+                " an invalid step size."
+            ),
+        ):
+            zappend(slices, target_dir=target_dir, append_step="2D")
+
+    def test_some_slices_with_inc_append_labels(self):
+        append_step = "+"
+
         target_dir = "memory://target.zarr"
         slices = [
             make_test_dataset(index=0, shape=(1, 50, 100)),
+            make_test_dataset(index=1, shape=(1, 50, 100)),
             make_test_dataset(index=2, shape=(1, 50, 100)),
         ]
-        zappend(slices, target_dir=target_dir, append_step="1d")
-        ds = xr.open_zarr(target_dir)
-        np.testing.assert_array_equal(
-            ds.time.values,
-            np.array(["2024-01-01", "2024-01-02", "2024-01-03"], dtype=np.datetime64),
-        )
+        # OK!
+        zappend(slices, target_dir=target_dir, append_step=append_step)
 
-    def test_some_slices_with_three_missing_append_steps(self):
         target_dir = "memory://target.zarr"
         slices = [
+            make_test_dataset(index=1, shape=(1, 50, 100)),
             make_test_dataset(index=0, shape=(1, 50, 100)),
-            make_test_dataset(index=4, shape=(1, 50, 100)),
         ]
-        zappend(slices, target_dir=target_dir, append_step="1d")
-        ds = xr.open_zarr(target_dir)
-        np.testing.assert_array_equal(
-            ds.time.values,
-            np.array(
-                ["2024-01-01", "2024-01-02", "2024-01-03", "2024-01-04", "2024-01-05"],
-                dtype=np.datetime64,
+        with pytest.raises(
+            ValueError,
+            match=(
+                "Cannot append slice because labels must be monotonically increasing"
             ),
-        )
-
-    def test_it_raises_for_wrong_append_step(self):
-        # TODO: implement me
-        pass
-
-    def test_some_slices_with_inc_append_labels(self):
-        # TODO: implement me
-        pass
+        ):
+            zappend(slices, target_dir=target_dir, append_step=append_step)
 
     def test_some_slices_with_dec_append_labels(self):
-        # TODO: implement me
-        pass
+        append_step = "-"
 
-    def test_it_raises_for_none_inc_append_labels(self):
-        # TODO: implement me
-        pass
+        target_dir = "memory://target.zarr"
+        slices = [
+            make_test_dataset(index=2, shape=(1, 50, 100)),
+            make_test_dataset(index=1, shape=(1, 50, 100)),
+            make_test_dataset(index=0, shape=(1, 50, 100)),
+        ]
+        # OK!
+        zappend(slices, target_dir=target_dir, append_step=append_step)
+
+        target_dir = "memory://target.zarr"
+        slices = [
+            make_test_dataset(index=0, shape=(1, 50, 100)),
+            make_test_dataset(index=1, shape=(1, 50, 100)),
+        ]
+        with pytest.raises(
+            ValueError,
+            match=(
+                "Cannot append slice because labels must be monotonically decreasing"
+            ),
+        ):
+            zappend(slices, target_dir=target_dir, append_step=append_step)
 
     def test_some_slices_with_profiling(self):
         target_dir = "memory://target.zarr"
diff --git a/tests/test_context.py b/tests/test_context.py
@@ -45,16 +45,16 @@ def test_append_step(self):
         make_test_dataset(uri="memory://target.zarr")
         ctx = Context({"target_dir": "memory://target.zarr"})
         self.assertEqual(None, ctx.append_step_size)
-        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "1d"})
-        self.assertEqual("1d", ctx.append_step_size)
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "1D"})
+        self.assertEqual("1D", ctx.append_step_size)
 
     def test_last_append_label(self):
         make_test_dataset(uri="memory://target.zarr")
         ctx = Context({"target_dir": "memory://target.zarr"})
         self.assertEqual(None, ctx.last_append_label)
-        ctx = Context({"target_dir": "memory://TARGET.zarr", "append_step": "1d"})
+        ctx = Context({"target_dir": "memory://TARGET.zarr", "append_step": "1D"})
         self.assertEqual(None, ctx.last_append_label)
-        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "1d"})
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "1D"})
         self.assertEqual(np.datetime64("2024-01-03"), ctx.last_append_label)
 
     def test_slice_polling(self):
diff --git a/tests/test_processor.py b/tests/test_processor.py
@@ -3,15 +3,21 @@
 # https://opensource.org/licenses/MIT.
 
 import unittest
+
+import numpy as np
+import pytest
 import xarray as xr
 
 from zappend.fsutil.fileobj import FileObj
+from zappend.context import Context
 from zappend.processor import Processor
+from zappend.processor import to_timedelta
+from zappend.processor import verify_append_labels
 from .helpers import clear_memory_fs
 from .helpers import make_test_dataset
 
 
-class TestProcessor(unittest.TestCase):
+class ProcessorTest(unittest.TestCase):
     def setUp(self):
         clear_memory_fs()
 
@@ -142,3 +148,83 @@ def test_process_two_slices_with_chunk_overlap(self):
         self.assertEqual(None, ds.time.chunks)
         self.assertEqual(((3, 1), (5, 5), (10, 10)), ds.chl.chunks)
         self.assertEqual(((3, 1), (5, 5), (10, 10)), ds.tsm.chunks)
+
+
+# noinspection PyMethodMayBeStatic
+class AppendLabelValidationTest(unittest.TestCase):
+    def setUp(self):
+        clear_memory_fs()
+
+    def test_verify_append_labels_succeeds(self):
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "1D"})
+
+        # Ok, because we have no delta
+        slice_ds = make_test_dataset(shape=(1, 50, 100))
+        verify_append_labels(ctx, slice_ds)
+
+        # Ok, because we have 4 deltas that are 1D
+        slice_ds = make_test_dataset(shape=(5, 50, 100))
+        verify_append_labels(ctx, slice_ds)
+
+        # Ok, because after removing "time" coordinate variable,
+        # xarray will use numerical labels
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": 1})
+        slice_ds = make_test_dataset(shape=(3, 50, 100)).drop_vars(["time"])
+        verify_append_labels(ctx, slice_ds)
+
+        # Ok, because "foo" has no labels
+        ctx = Context(
+            {
+                "target_dir": "memory://target.zarr",
+                "append_dim": "foo",
+                "append_step": "1D",
+            }
+        )
+        slice_ds = make_test_dataset(shape=(3, 50, 100))
+        verify_append_labels(ctx, slice_ds)
+
+    def test_verify_append_labels_fails(self):
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "2D"})
+        slice_ds = make_test_dataset(shape=(3, 50, 100))
+        with pytest.raises(
+            ValueError,
+            match="Cannot append slice because this would result in an invalid step size.",
+        ):
+            verify_append_labels(ctx, slice_ds)
+
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "-"})
+        slice_ds = make_test_dataset(shape=(3, 50, 100))
+        with pytest.raises(
+            ValueError,
+            match="Cannot append slice because labels must be monotonically decreasing.",
+        ):
+            verify_append_labels(ctx, slice_ds)
+
+        ctx = Context({"target_dir": "memory://target.zarr", "append_step": "+"})
+        slice_ds = make_test_dataset(shape=(3, 50, 100))
+        time = slice_ds["time"]
+        slice_ds["time"] = xr.DataArray(
+            list(reversed(time.values)), dims=time.dims, attrs=time.attrs
+        )
+        with pytest.raises(
+            ValueError,
+            match="Cannot append slice because labels must be monotonically increasing.",
+        ):
+            verify_append_labels(ctx, slice_ds)
+
+
+class ToTimedeltaTest(unittest.TestCase):
+    def test_it(self):
+        self.assertEqual(np.timedelta64(1, "s"), to_timedelta("s"))
+        self.assertEqual(np.timedelta64(1, "m"), to_timedelta("m"))
+        self.assertEqual(np.timedelta64(1, "h"), to_timedelta("h"))
+        self.assertEqual(np.timedelta64(1, "h"), to_timedelta("1h"))
+        self.assertEqual(np.timedelta64(24, "h"), to_timedelta("24h"))
+        self.assertEqual(np.timedelta64(1, "D"), to_timedelta("24h"))
+        self.assertEqual(np.timedelta64(1, "D"), to_timedelta("D"))
+        self.assertEqual(np.timedelta64(1, "D"), to_timedelta("1D"))
+        self.assertEqual(np.timedelta64(7, "D"), to_timedelta("7D"))
+        self.assertEqual(np.timedelta64(1, "W"), to_timedelta("7D"))
+        self.assertEqual(np.timedelta64(12, "D"), to_timedelta("12D"))
+        self.assertEqual(np.timedelta64(60 * 60 * 24, "s"), to_timedelta(60 * 60 * 24))
+        self.assertEqual(np.timedelta64(1, "D"), to_timedelta(60 * 60 * 24))
diff --git a/zappend/config/markdown.py b/zappend/config/markdown.py
@@ -41,7 +41,7 @@ def _schema_to_md(
     if description:
         lines.append(description)
 
-    one_of = schema.get("oneOf")
+    one_of = schema.get("anyOf") or schema.get("oneOf")
     if one_of:
         if sequence_name:
             lines.append(f"The {sequence_name} must be one of the following.")
diff --git a/zappend/config/schema.py b/zappend/config/schema.py
@@ -479,7 +479,7 @@
                 "If set, enforces a step size in the append dimension between two"
                 " slices or just enforces a direction."
             ),
-            "oneOf": [
+            "anyOf": [
                 {
                     "description": "Arbitrary step size or not applicable.",
                     "const": None,
diff --git a/zappend/processor.py b/zappend/processor.py

Original file line number	Diff line number	Diff line change
`@@ -479,7 +479,7 @@`
`479`	`479`	`"If set, enforces a step size in the append dimension between two"`
`480`	`480`	`" slices or just enforces a direction."`
`481`	`481`	`),`
`482`		`- "oneOf": [`
	`482`	`+ "anyOf": [`
`483`	`483`	`{`
`484`	`484`	`"description": "Arbitrary step size or not applicable.",`
`485`	`485`	`"const": None,`