Merge branch 'refs/heads/main' into forman-66-how_do_i_guide

forman · forman · commit ccf3c79d1652 · 2024-05-28T16:53:14.000+02:00
diff --git a/.flake8 b/.flake8
@@ -0,0 +1,7 @@
+[flake8]
+# Make line length conform to black's default
+max-line-length = 88
+# Workaround for https://github.com/PyCQA/pycodestyle/issues/373 ;
+# see also
+# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#pycodestyle
+extend-ignore = E203
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -27,8 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-dev.txt
-        pip install -r requirements.txt
+        pip install .[dev,doc]
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,4 +1,10 @@
-## Version 0.7.0 (in development)
+## Version 0.7.1 (in development)
+
+* The function `zappend.api.zappend()` now returns the number of slices 
+  processed. (#93)
+
+* Moved all project configuration to `pyproject.toml` and removed
+  `setup.cfg` and requirements files. (#88)
 
 * Fixed link to _slice sources_ in documentation main page.
 
diff --git a/docs/about.md b/docs/about.md
@@ -27,12 +27,11 @@ corresponding issue.
 
 ## Development
 
-Setup development environment:
+To set up development environment, with repository root as current
+working directory:
 
 ```bash
-pip install -r requirements.txt
-pip install -r requirements-dev.txt
-pip install -r requirements-docs.txt
+pip install .[dev,doc]
 ```
 
 ### Testing and Coverage
@@ -56,8 +55,10 @@ black zappend
 
 `zappend` documentation is built using the [mkdocs](https://www.mkdocs.org/) tool.
 
+With repository root as current working directory:
+
 ```bash
-pip install -r requirements-doc.txt
+pip install .[doc]
 
 mkdocs build
 mkdocs serve
diff --git a/docs/guide.md b/docs/guide.md
@@ -74,7 +74,7 @@ zappend(os.listdir("inputs"),
         dry_run=True)
 ```
 
-This remainder of this guide explains the how to use the various `zappend` 
+The remainder of this guide explains the how to use the various `zappend` 
 [configuration](config.md) settings.
 
 !!! note
diff --git a/docs/start.md b/docs/start.md
@@ -9,6 +9,11 @@ version from PyPI:
 pip install zappend
 ```
 
+You can replace `zappend` with `zappend[dev]` in the above command to also
+install development dependencies, with `zappend[doc]` to also install
+documentation-building dependencies, or with `zappend[dev,doc]` to get both
+additional dependency sets.
+
 To install the latest version for development, clone the
 [repository](https://github.com/bcdev/zappend), and with the repository’s root
 directory as the current working directory execute:
diff --git a/pyproject.toml b/pyproject.toml
@@ -3,3 +3,69 @@ requires = [
   "setuptools >= 40.9.0",
 ]
 build-backend = "setuptools.build_meta"
+
+[project]
+name = "zappend"
+dynamic = ["version"]
+authors = [
+  {name = "Norman Fomferra", email = "norman.fomferra@brockmann-consult.de"}
+]
+description = """\
+  Tool for robustly creating and updating Zarr datacubes from smaller slices\
+  """
+keywords = [
+  "analysis ready data", "data science", "datacube", "xarray", "zarr"
+]
+readme = {file = "README.md", content-type = "text/markdown"}
+license = {text = "MIT"}
+requires-python = ">=3.10"
+dependencies = [
+  "click",
+  "dask",
+  "fsspec",
+  "jsonschema",
+  "numcodecs",
+  "numpy",
+  "pyyaml",
+  "xarray",
+  "zarr"
+]
+
+[tool.setuptools.dynamic]
+version = {attr = "zappend.__version__"}
+
+[tool.setuptools.packages.find]
+exclude = [
+  "test*",
+  "doc*",
+  "site*",
+  "htmlcov*"
+]
+
+[project.scripts]
+zappend = "zappend.cli:zappend"
+
+[project.optional-dependencies]
+dev = [
+  "pytest",
+  "pytest-cov",
+  "black",
+  "flake8",
+  "flake8-bugbear",
+  "h5netcdf",
+  "s3fs",
+  "scipy",
+  "pyproj"
+]
+doc = [
+  "mkdocs",
+  "mkdocs-material",
+  "mkdocstrings",
+  "mkdocstrings-python"
+]
+
+[project.urls]
+Documentation = "https://bcdev.github.io/zappend/"
+Issues = "https://github.com/bcdev/zappend/issues"
+Changelog = "https://github.com/bcdev/zappend/blob/main/CHANGES.md"
+Repository = "https://github.com/bcdev/zappend"
diff --git a/requirements-dev.txt b/requirements-dev.txt
diff --git a/requirements-doc.txt b/requirements-doc.txt
diff --git a/requirements.txt b/requirements.txt
diff --git a/setup.cfg b/setup.cfg
diff --git a/tests/test_api.py b/tests/test_api.py
@@ -177,7 +177,8 @@ def drop_tsm(slice_ds: xr.Dataset) -> xr.Dataset:
 
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=3 * i) for i in range(3)]
-        zappend(slices, target_dir=target_dir, slice_source=drop_tsm)
+        count = zappend(slices, target_dir=target_dir, slice_source=drop_tsm)
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual({"time": 9, "y": 50, "x": 100}, ds.sizes)
         self.assertEqual({"chl"}, set(ds.data_vars))
@@ -196,7 +197,8 @@ def drop_tsm(slice_ds: xr.Dataset) -> xr.Dataset:
 
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=3 * i) for i in range(3)]
-        zappend(slices, target_dir=target_dir, slice_source=drop_tsm)
+        count = zappend(slices, target_dir=target_dir, slice_source=drop_tsm)
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual({"time": 9, "y": 50, "x": 100}, ds.sizes)
         self.assertEqual({"chl"}, set(ds.data_vars))
@@ -218,7 +220,8 @@ def crop_ds(slice_ds: xr.Dataset) -> xr.Dataset:
 
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=3 * i) for i in range(3)]
-        zappend(slices, target_dir=target_dir, slice_source=crop_ds)
+        count = zappend(slices, target_dir=target_dir, slice_source=crop_ds)
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual({"time": 9, "y": 40, "x": 90}, ds.sizes)
         self.assertEqual({"chl", "tsm"}, set(ds.data_vars))
@@ -258,9 +261,10 @@ def crop_ds(slice_ds: xr.Dataset) -> xr.Dataset:
 
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=3 * i) for i in range(3)]
-        zappend(
+        count = zappend(
             slices, target_dir=target_dir, slice_source=crop_ds, variables=variables
         )
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual({"time": 9, "y": 40, "x": 90}, ds.sizes)
         self.assertEqual({"chl", "tsm"}, set(ds.data_vars))
@@ -274,7 +278,8 @@ def crop_ds(slice_ds: xr.Dataset) -> xr.Dataset:
     def test_some_slices_with_inc_append_step(self):
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset(index=i, shape=(1, 50, 100)) for i in range(3)]
-        zappend(slices, target_dir=target_dir, append_step="1D")
+        count = zappend(slices, target_dir=target_dir, append_step="1D")
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         np.testing.assert_array_equal(
             ds.time.values,
@@ -293,7 +298,8 @@ def test_some_slices_with_dec_append_step(self):
         slices = [
             make_test_dataset(index=i, shape=(1, 50, 100)) for i in reversed(range(3))
         ]
-        zappend(slices, target_dir=target_dir, append_step="-1D")
+        count = zappend(slices, target_dir=target_dir, append_step="-1D")
+        self.assertEqual(3, count)
         ds = xr.open_zarr(target_dir)
         np.testing.assert_array_equal(
             ds.time.values,
@@ -416,7 +422,7 @@ def test_some_slices_with_dec_append_labels(self):
     def test_dynamic_attrs_with_one_slice(self):
         target_dir = "memory://target.zarr"
         slices = [make_test_dataset()]
-        zappend(
+        count = zappend(
             slices,
             target_dir=target_dir,
             permit_eval=True,
@@ -426,6 +432,7 @@ def test_dynamic_attrs_with_one_slice(self):
                 "time_coverage_end": "{{ ds.time[-1] }}",
             },
         )
+        self.assertEqual(1, count)
         ds = xr.open_zarr(target_dir)
         self.assertEqual(
             {
diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -66,7 +66,7 @@ def test_some_slices_and_target(self):
                 "memory://slice-3.zarr",
             ],
         )
-        self.assertEqual("", result.output)
+        self.assertEqual("3 slice datasets processed.\n", result.output)
         self.assertEqual(0, result.exit_code)
         self.assertTrue(FileObj("memory://target.zarr").exists())
 
@@ -88,7 +88,7 @@ def test_some_slices_and_target_dry_run(self):
                 "memory://slice-3.zarr",
             ],
         )
-        self.assertEqual("", result.output)
+        self.assertEqual("3 slice datasets processed.\n", result.output)
         self.assertEqual(0, result.exit_code)
         self.assertFalse(FileObj("memory://target.zarr").exists())
 
diff --git a/zappend/__init__.py b/zappend/__init__.py
@@ -2,4 +2,4 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-__version__ = "0.8.0.dev0"
+__version__ = "0.7.1.dev0"
diff --git a/zappend/api.py b/zappend/api.py
@@ -34,7 +34,7 @@ def zappend(
     slices: Iterable[Any],
     config: ConfigLike = None,
     **kwargs: Any,
-):
+) -> int:
     """Robustly create or update a Zarr dataset from dataset slices.
 
     The `zappend` function concatenates the dataset slices from given
@@ -62,11 +62,16 @@ def zappend(
     Args:
         slices: An iterable that yields slice items.
         config: Processor configuration.
-            May be a file path or URI, a `dict`, `None`, or a sequence of
+            Can be a file path or URI, a `dict`, `None`, or a sequence of
             the aforementioned. If a sequence is used, subsequent
             configurations are incremental to the previous ones.
         kwargs: Additional configuration parameters.
             Can be used to pass or override configuration values in *config*.
+
+    Returns:
+        The number of slices processed. The value can be useful if \
+        the number of items in `slices` is unknown.
+
     """
     processor = Processor(config, **kwargs)
-    processor.process_slices(slices)
+    return processor.process_slices(slices)
diff --git a/zappend/cli.py b/zappend/cli.py
@@ -93,13 +93,14 @@ def zappend(
 
     # noinspection PyBroadException
     try:
-        zappend(
+        count = zappend(
             slices,
             config=config,
             target_dir=target,
             force_new=force_new,
             dry_run=dry_run,
         )
+        click.echo(f"{count} slice dataset{'s' if count != 1 else ''} processed.")
     except BaseException as e:
         if traceback:
             import traceback as tb
diff --git a/zappend/processor.py b/zappend/processor.py
@@ -58,16 +58,21 @@ def __init__(self, config: ConfigLike = None, **kwargs):
         self._config = _config
         self._profiler = Profiler(_config.profiling)
 
-    def process_slices(self, slices: Iterable[SliceItem]):
+    def process_slices(self, slices: Iterable[SliceItem]) -> int:
         """Process the given `slices`.
         Passes each slice in `slices` to the `process_slice()` method.
 
         Args:
             slices: Iterable of slice items.
+        Returns:
+            The number of slices processed.
         """
+        slice_index = 0
         with self._profiler:
-            for slice_index, slice_item in enumerate(slices):
+            for slice_item in slices:
                 self.process_slice(slice_item, slice_index=slice_index)
+                slice_index += 1
+        return slice_index
 
     def process_slice(self, slice_item: SliceItem, slice_index: int = 0):
         """Process a single slice item *slice_item*.
