dropping variables without append dim before appending

Author: forman

tests/helpers.py

@@ -7,6 +7,7 @@
 
 import fsspec
 import numpy as np
+import pyproj
 import xarray as xr
 
 default_dims = ("time", "y", "x")
@@ -71,6 +72,7 @@ def make_test_dataset(
     dims: tuple[str, str, str] = default_dims,
     shape: tuple[int, int, int] = default_shape,
     chunks: tuple[int, int, int] = default_chunks,
+    crs: str | None = None,
     index: int = 0,
     uri: str | None = None,
     storage_options: dict[str, Any] | None = None
@@ -113,6 +115,9 @@ def make_test_dataset(
         }
     )
 
+    if crs:
+        ds["crs"] = xr.DataArray(np.array(0), attrs=pyproj.CRS(crs).to_cf())
+
     ds = ds.chunk(dict(tuple(zip(dims, chunks))))
 
     if not uri:

tests/test_rollbackstore.py

@@ -134,7 +134,8 @@ def handle_rollback_action(self, *args):
     def test_to_zarr(self):
         ds = make_test_dataset(
             shape=(1, 50, 100),
-            chunks=(2, 50, 50)
+            chunks=(2, 50, 50),
+            crs="epsg:4326"
         )
         ds.time.encoding.update(chunks=(10,))
         ds.chl.encoding.update(chunks=(2, 50, 50))
@@ -147,6 +148,7 @@ def test_to_zarr(self):
         self.assert_dataset_ok(
             {'x': 100, 'y': 50, 'time': 1},
             {
+                'crs': (),
                 'x': (100,), 'y': (50,), 'time': (10,),
                 'chl': (2, 50, 50),
                 'tsm': (2, 50, 50),
@@ -167,6 +169,9 @@ def test_to_zarr(self):
                 ('delete_file', 'time/.zarray'),
                 ('delete_file', 'time/.zattrs'),
                 ('delete_file', 'time/0'),
+                ('delete_file', 'crs/.zarray'),
+                ('delete_file', 'crs/.zattrs'),
+                ('delete_file', 'crs/0'),
                 ('delete_file', 'chl/.zarray'),
                 ('delete_file', 'chl/.zattrs'),
                 ('delete_file', 'chl/0.0.0'),
@@ -188,7 +193,8 @@ def test_to_zarr(self):
             chunks=(1, 50, 50)
         )
         # drop variables w.o. "time" dim
-        slice_1 = slice_1.drop_vars(["x", "y"])
+        slice_1 = slice_1.drop_vars([k for k, v in slice_1.variables.items()
+                                     if "time" not in v.dims])
         slice_1.attrs = {}
         for k, v in slice_1.variables.items():
             v.encoding = {}
@@ -205,6 +211,7 @@ def test_to_zarr(self):
         )
         self.assert_dataset_ok({'x': 100, 'y': 50, 'time': 2},
                                {
+                                   'crs': (),
                                    'x': (100,), 'y': (50,), 'time': (10,),
                                    'chl': (2, 50, 50),
                                    'tsm': (2, 50, 50),
@@ -234,7 +241,8 @@ def test_to_zarr(self):
             chunks=(1, 50, 50)
         )
         # drop variables w.o. "time" dim
-        slice_2 = slice_2.drop_vars(["x", "y"])
+        slice_2 = slice_2.drop_vars([k for k, v in slice_2.variables.items()
+                                     if "time" not in v.dims])
         for k, v in slice_2.variables.items():
             v.encoding = {}
             v.attrs = {}
@@ -250,6 +258,7 @@ def test_to_zarr(self):
         )
         self.assert_dataset_ok({'x': 100, 'y': 50, 'time': 3},
                                {
+                                   'crs': (),
                                    'x': (100,), 'y': (50,), 'time': (10,),
                                    'chl': (2, 50, 50),
                                    'tsm': (2, 50, 50),

tests/test_tailoring.py

@@ -5,6 +5,7 @@
 import unittest
 
 import numpy as np
+import pyproj
 import xarray as xr
 
 from zappend.metadata import DatasetMetadata
@@ -125,7 +126,37 @@ def test_it_completes_vars(self):
 
 class TailorSliceDatasetTest(unittest.TestCase):
 
-    def test_it_sets_metadata(self):
+    def test_it_drops_constant_variables(self):
+        ds = xr.Dataset(
+            {
+                "a": xr.DataArray(np.zeros((2, 3, 4)),
+                                  dims=("time", "y", "x")),
+                "b": xr.DataArray(np.zeros((2, 3, 4)),
+                                  dims=("time", "y", "x")),
+                "spatial_ref": xr.DataArray(
+                    np.array(0),
+                    attrs=pyproj.CRS("EPSG:4326").to_cf()
+                )
+            },
+            coords={
+                "x": xr.DataArray(np.linspace(0.0, 1.0, 4),
+                                  dims="x"),
+                "y": xr.DataArray(np.linspace(0.0, 1.0, 3),
+                                  dims="y"),
+            }
+        )
+        tailored_ds = tailor_slice_dataset(
+            ds,
+            DatasetMetadata.from_dataset(ds, {}),
+            "time"
+        )
+        self.assertIsInstance(tailored_ds, xr.Dataset)
+        self.assertEqual(
+            {"a", "b"},
+            set(tailored_ds.variables.keys())
+        )
+
+    def test_it_clears_encoding_and_attrs(self):
         ds = xr.Dataset({
             "a": xr.DataArray(np.zeros((2, 3, 4)),
                               dims=("time", "y", "x"),
@@ -148,18 +179,17 @@ def test_it_sets_metadata(self):
                                            "fill_value": -1}},
                     }
                 }
-            )
+            ),
+            "time"
         )
         self.assertIsInstance(tailored_ds, xr.Dataset)
-        self.assertEqual(
-            {"a", "b"},
-            set(tailored_ds.variables.keys())
-        )
 
+        self.assertIn("a", tailored_ds.variables)
         a = tailored_ds.a
         self.assertEqual({}, a.encoding)
         self.assertEqual({}, a.attrs)
 
+        self.assertIn("b", tailored_ds.variables)
         b = tailored_ds.b
         self.assertEqual({}, b.encoding)
         self.assertEqual({}, b.attrs)

zappend/processor.py

@@ -99,7 +99,9 @@ def update_target_from_slice(ctx: Context,
     target_dir = ctx.target_dir
     append_dim_name = ctx.append_dim_name
 
-    slice_ds = tailor_slice_dataset(slice_ds, ctx.target_metadata)
+    slice_ds = tailor_slice_dataset(slice_ds,
+                                    ctx.target_metadata,
+                                    append_dim_name)
 
     # TODO: adjust global attributes dependent on append_dim,
     #  e.g., time coverage

zappend/tailoring.py

@@ -29,11 +29,19 @@ def tailor_target_dataset(
 
 def tailor_slice_dataset(
     dataset: xr.Dataset,
-    target_metadata: DatasetMetadata
+    target_metadata: DatasetMetadata,
+    append_dim_name: str
 ) -> xr.Dataset:
     dataset = _strip_dataset(dataset, target_metadata)
     dataset = _complete_dataset(dataset, target_metadata)
 
+    const_variables = [k for k, v in dataset.variables.items() if
+                       append_dim_name not in v.dims]
+    if const_variables:
+        # Strip variables that do not have append_dim_name
+        # as dimension, e.g., "x", "y", "crs", ...
+        dataset = dataset.drop_vars(const_variables)
+
     # Remove any encoding and attributes from slice,
     # since both are prescribed by target
     dataset.attrs.clear()