fixes after system testing

Author: forman

zappend/context.py

@@ -35,13 +35,11 @@ def __init__(self, config: Dict[str, Any]):
                 target_uri,
                 storage_options=target_storage_options
             ) as target_dataset:
-                logger.info(f"Target dataset f{target_uri} found")
                 target_metadata = DatasetMetadata.from_dataset(
                     target_dataset,
                     config
                 )
         except FileNotFoundError:
-            logger.info(f"Target dataset {target_uri} not found")
             target_metadata = None
 
         self._target_metadata = target_metadata

zappend/fsutil/fileobj.py

@@ -77,8 +77,7 @@ def path(self) -> str:
     def close(self):
         """Close the filesystem used by this file object."""
         if self._fs is not None:
-            if hasattr(self._fs, "close") and callable(
-                self._fs.close):
+            if hasattr(self._fs, "close") and callable(self._fs.close):
                 self._fs.close()
             self._fs = None
 

zappend/fsutil/transaction.py

@@ -27,6 +27,8 @@
 ROLLBACK_ACTIONS = "delete_dir", "delete_file", "replace_file"
 
 
+# TODO: allow disabling rollbacks entirely
+
 class Transaction:
     """
     A filesystem transaction.
@@ -79,12 +81,14 @@ def __exit__(self, exc_type, exc_val, exc_tb):
                                     for line in rollback_txt.split("\n")
                                 ] if record]
 
-            for record in rollback_records:
-                logger.debug(f"Running rollback {record}")
-                action = record[0]
-                args = record[1:]
-                action_method = getattr(self, "_" + action)
-                action_method(*args)
+            if rollback_records:
+                logger.info(f"Rolling back {len(rollback_records)} action(s)")
+                for record in rollback_records:
+                    logger.debug(f"Running rollback {record}")
+                    action = record[0]
+                    args = record[1:]
+                    action_method = getattr(self, "_" + action)
+                    action_method(*args)
 
         self._rollback_dir.delete(recursive=True)
 
@@ -96,15 +100,17 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             logger.warning("Note, it should be save to delete it manually.")
 
     def _delete_dir(self, target_path):
-        self._target_dir.fs.rm(target_path, recursive=True)
+        _dir = self._target_dir / target_path
+        _dir.delete(recursive=True)
 
     def _delete_file(self, target_path):
-        self._target_dir.fs.rm(target_path)
+        _file = self._target_dir / target_path
+        _file.delete()
 
     def _replace_file(self, target_path, rollback_filename):
+        _file = self._target_dir / target_path
         data = (self._rollback_dir / rollback_filename).read()
-        with self._target_dir.fs.open(target_path, "wb") as f:
-            f.write(data)
+        _file.write(data)
 
     def _add_rollback_action(self,
                              action: RollbackAction,

zappend/metadata.py

@@ -123,8 +123,8 @@ def from_dataset(cls,
 
         variables = _get_effective_variables(
             dataset,
-            config.get("included_variables") or {},
-            config.get("excluded_variables") or {},
+            config.get("included_variables") or [],
+            config.get("excluded_variables") or [],
             config.get("variables") or {}
         )
 
@@ -166,11 +166,8 @@ def _get_effective_variables(
     config_variables: dict[str, dict[str, Any]] | None
 ) -> dict[str, VariableMetadata]:
     config_variables = dict(config_variables or {})
+    # Get and remove defaults for all variables
     defaults = config_variables.pop("*", {})
-    config_variables = {k: merge_configs(defaults, v)
-                        for k, v in config_variables.items()
-                        if k != "*"}
-
     all_var_names = (set(map(str, dataset.variables.keys()))
                      .union(set(config_variables.keys())))
     if config_included_variables:
@@ -189,7 +186,11 @@ def _get_effective_variables(
     variables = {}
 
     for var_name in selected_var_names:
-        config_var_def: dict = dict(config_variables.get(var_name) or {})
+        # "*" is default for all variables
+        config_var_def: dict = merge_configs(
+            defaults,
+            config_variables.get(var_name) or {}
+        )
         ds_var = dataset.variables.get(var_name)
         if ds_var is not None:
             # Variable found in dataset: use dataset variable to complement

zappend/processor.py

@@ -81,11 +81,11 @@ def process_slice(self, slice_obj: str | xr.Dataset):
 def create_target_from_slice(ctx: Context,
                              slice_ds: xr.Dataset,
                              rollback_cb: RollbackCallback):
-    logger.info(f"Creating target dataset")
+    target_dir = ctx.target_dir
+    logger.info(f"Creating target dataset {target_dir.uri}")
     target_ds = tailor_target_dataset(slice_ds, ctx.target_metadata)
     if ctx.dry_run:
         return
-    target_dir = ctx.target_dir
     # TODO: adjust global attributes dependent on append_dim,
     #  e.g., time coverage
     try:
@@ -102,8 +102,8 @@ def create_target_from_slice(ctx: Context,
 def update_target_from_slice(ctx: Context,
                              slice_ds: xr.Dataset,
                              rollback_cb: RollbackCallback):
-    logger.info(f"Updating target dataset")
     target_dir = ctx.target_dir
+    logger.info(f"Updating target dataset {target_dir.uri}")
     append_dim_name = ctx.append_dim_name
 
     slice_ds = tailor_slice_dataset(slice_ds,