Merge pull request #94 from bcdev/forman-93-return_num_slices

Return the number of slices processed

Author: forman

CHANGES.md

@@ -1,10 +1,13 @@
 ## Version 0.7.1 (in development)
 
-* Fixed link to _slice sources_ in documentation main page.
+* The function `zappend.api.zappend()` now returns the number of slices 
+  processed. (#93)
 
 * Moved all project configuration to `pyproject.toml` and removed
   `setup.cfg` and requirements files. (#88)
 
+* Fixed link to _slice sources_ in documentation main page.
+
 ## Version 0.7.0 (from 2024-03-19)
 
 * Made writing custom slice sources easier and more flexible: (#82)

docs/guide.md

@@ -74,7 +74,7 @@ zappend(os.listdir("inputs"),
         dry_run=True)
 ```
 
-This remainder of this guide explains the how to use the various `zappend` 
+The remainder of this guide explains the how to use the various `zappend` 
 [configuration](config.md) settings.
 
 !!! note

tests/test_api.py

@@ -177,7 +177,8 @@ def drop_tsm(slice_ds: xr.Dataset) -> xr.Dataset:
 
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=3 * i) for i in range(3)]
-        zappend(slices, target_dir=target_dir, slice_source=drop_tsm)
+        count = zappend(slices, target_dir=target_dir, slice_source=drop_tsm)
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual({"time": 9, "y": 50, "x": 100}, ds.sizes)
         self.assertEqual({"chl"}, set(ds.data_vars))
@@ -196,7 +197,8 @@ def drop_tsm(slice_ds: xr.Dataset) -> xr.Dataset:
 
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=3 * i) for i in range(3)]
-        zappend(slices, target_dir=target_dir, slice_source=drop_tsm)
+        count = zappend(slices, target_dir=target_dir, slice_source=drop_tsm)
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual({"time": 9, "y": 50, "x": 100}, ds.sizes)
         self.assertEqual({"chl"}, set(ds.data_vars))
@@ -218,7 +220,8 @@ def crop_ds(slice_ds: xr.Dataset) -> xr.Dataset:
 
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=3 * i) for i in range(3)]
-        zappend(slices, target_dir=target_dir, slice_source=crop_ds)
+        count = zappend(slices, target_dir=target_dir, slice_source=crop_ds)
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual({"time": 9, "y": 40, "x": 90}, ds.sizes)
         self.assertEqual({"chl", "tsm"}, set(ds.data_vars))
@@ -258,9 +261,10 @@ def crop_ds(slice_ds: xr.Dataset) -> xr.Dataset:
 
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=3 * i) for i in range(3)]
-        zappend(
+        count = zappend(
             slices, target_dir=target_dir, slice_source=crop_ds, variables=variables
         )
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual({"time": 9, "y": 40, "x": 90}, ds.sizes)
         self.assertEqual({"chl", "tsm"}, set(ds.data_vars))
@@ -274,7 +278,8 @@ def crop_ds(slice_ds: xr.Dataset) -> xr.Dataset:
     def test_some_slices_with_inc_append_step(self):
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=i, shape=(1, 50, 100)) for i in range(3)]
-        zappend(slices, target_dir=target_dir, append_step="1D")
+        count = zappend(slices, target_dir=target_dir, append_step="1D")
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         np.testing.assert_array_equal(
             ds.time.values,
@@ -293,7 +298,8 @@ def test_some_slices_with_dec_append_step(self):
         slices = [
             make_test_dataset(index=i, shape=(1, 50, 100)) for i in reversed(range(3))
         ]
-        zappend(slices, target_dir=target_dir, append_step="-1D")
+        count = zappend(slices, target_dir=target_dir, append_step="-1D")
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         np.testing.assert_array_equal(
             ds.time.values,
@@ -416,7 +422,7 @@ def test_some_slices_with_dec_append_labels(self):
     def test_dynamic_attrs_with_one_slice(self):
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset()]
-        zappend(
+        count = zappend(
             slices,
             target_dir=target_dir,
             permit_eval=True,
@@ -426,6 +432,7 @@ def test_dynamic_attrs_with_one_slice(self):
                 "time_coverage_end": "{{ ds.time[-1] }}",
             },
         )
+        self.assertEqual(1, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual(
             {

tests/test_cli.py

@@ -66,7 +66,7 @@ def test_some_slices_and_target(self):
                 "memory://slice-3.zarr",
             ],
         )
-        self.assertEqual("", result.output)
+        self.assertEqual("3 slice datasets processed.\n", result.output)
         self.assertEqual(0, result.exit_code)
         self.assertTrue(FileObj("memory://target.zarr").exists())
 
@@ -88,7 +88,7 @@ def test_some_slices_and_target_dry_run(self):
                 "memory://slice-3.zarr",
             ],
         )
-        self.assertEqual("", result.output)
+        self.assertEqual("3 slice datasets processed.\n", result.output)
         self.assertEqual(0, result.exit_code)
         self.assertFalse(FileObj("memory://target.zarr").exists())
 

zappend/__init__.py

@@ -2,4 +2,4 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-__version__ = "0.8.0.dev0"
+__version__ = "0.7.1.dev0"

zappend/api.py

@@ -34,7 +34,7 @@ def zappend(
     slices: Iterable[Any],
     config: ConfigLike = None,
     **kwargs: Any,
-):
+) -> int:
     """Robustly create or update a Zarr dataset from dataset slices.
 
     The `zappend` function concatenates the dataset slices from given
@@ -62,11 +62,16 @@ def zappend(
     Args:
         slices: An iterable that yields slice items.
         config: Processor configuration.
-            May be a file path or URI, a `dict`, `None`, or a sequence of
+            Can be a file path or URI, a `dict`, `None`, or a sequence of
             the aforementioned. If a sequence is used, subsequent
             configurations are incremental to the previous ones.
         kwargs: Additional configuration parameters.
             Can be used to pass or override configuration values in *config*.
+
+    Returns:
+        The number of slices processed. The value can be useful if \
+        the number of items in `slices` is unknown.
+
     """
     processor = Processor(config, **kwargs)
-    processor.process_slices(slices)
+    return processor.process_slices(slices)

zappend/cli.py

@@ -93,13 +93,14 @@ def zappend(
 
     # noinspection PyBroadException
     try:
-        zappend(
+        count = zappend(
             slices,
             config=config,
             target_dir=target,
             force_new=force_new,
             dry_run=dry_run,
         )
+        click.echo(f"{count} slice dataset{'s' if count != 1 else ''} processed.")
     except BaseException as e:
         if traceback:
             import traceback as tb

zappend/processor.py

@@ -58,16 +58,21 @@ def __init__(self, config: ConfigLike = None, **kwargs):
         self._config = _config
         self._profiler = Profiler(_config.profiling)
 
-    def process_slices(self, slices: Iterable[SliceItem]):
+    def process_slices(self, slices: Iterable[SliceItem]) -> int:
         """Process the given `slices`.
         Passes each slice in `slices` to the `process_slice()` method.
 
         Args:
             slices: Iterable of slice items.
+        Returns:
+            The number of slices processed.
         """
+        slice_index = 0
         with self._profiler:
-            for slice_index, slice_item in enumerate(slices):
+            for slice_item in slices:
                 self.process_slice(slice_item, slice_index=slice_index)
+                slice_index += 1
+        return slice_index
 
     def process_slice(self, slice_item: SliceItem, slice_index: int = 0):
         """Process a single slice item *slice_item*.