Fix --ignore-merge-errors (#250)

* pass kwargs, rework strict_merge

* woops.

* Docs.

Author: PeterKraus

docs/source/version.7_0.rst

@@ -20,4 +20,5 @@ Breaking changes in ``yadg-next`` are:
 Bug fixes in ``yadg-next`` include:
 
   - The parameter ``Set I/C`` in :mod:`yadg.extractors.eclab.mpr` files should be ``C / N`` when set to 1, not ``C``.
+  - The command line argument ``--ignore-merge-errors`` was not being passed to the individual extractors when using the ``yadg extract`` syntax.
 

src/yadg/core.py

@@ -28,10 +28,6 @@ def process_schema(dataschema: DataSchema, strict_merge: bool = False) -> DataTr
         has to be identical. Defaults to ``False`` which means conflicts will be dropped.
 
     """
-    if strict_merge:
-        concatmode = "identical"
-    else:
-        concatmode = "drop_conflicts"
 
     while hasattr(dataschema, "update"):
         dataschema = dataschema.update()
@@ -85,7 +81,7 @@ def process_schema(dataschema: DataSchema, strict_merge: bool = False) -> DataTr
                     if k in fvals[name].attrs:
                         del fvals[name].attrs[k]
 
-            vals = dgutils.merge_dicttrees(vals, fvals, concatmode)
+            vals = dgutils.merge_dicttrees(vals, fvals, strict_merge)
 
         stepdt = DataTree.from_dict({} if vals is None else vals)
         stepdt.name = step.tag

src/yadg/dgutils/dsutils.py

@@ -69,7 +69,7 @@ def dicts_to_dataset(
     return xr.Dataset(data_vars=darrs, coords=coords, attrs=attrs)
 
 
-def merge_dicttrees(vals: dict, fvals: dict, mode: str) -> dict:
+def merge_dicttrees(vals: dict, fvals: dict, strict_merge: bool) -> dict:
     """
     A helper function that merges two ``DataTree.to_dict()`` objects by concatenating
     the new values in ``fvals`` to the existing ones in ``vals``.
@@ -79,8 +79,16 @@ def merge_dicttrees(vals: dict, fvals: dict, mode: str) -> dict:
         return fvals
     for k in fvals.keys():
         try:
-            vals[k] = xr.concat([vals[k], fvals[k]], dim="uts", combine_attrs=mode)
-        except xr.MergeError:
+            # vals[k] = xr.concat([vals[k], fvals[k]], dim="uts", combine_attrs=mode)
+            vals[k] = xr.concat(
+                [vals[k], fvals[k]],
+                dim="uts",
+                data_vars="different",
+                compat="identical" if strict_merge else "equals",
+                join="outer",
+                combine_attrs="identical" if strict_merge else "drop_conflicts",
+            )
+        except (xr.MergeError, ValueError) as e:
             raise RuntimeError(
                 "Merging metadata from multiple files has failed, as some of the "
                 "values differ between files. This might be caused by trying to "
@@ -89,7 +97,7 @@ def merge_dicttrees(vals: dict, fvals: dict, mode: str) -> dict:
                 "yadg with the '--ignore-merge-errors' option."
                 f"\n{vals[k].attrs=}"
                 f"\n{fvals[k].attrs=}"
-            )
+            ) from e
     return vals
 
 

src/yadg/extractors/__init__.py

@@ -77,13 +77,14 @@ def extract(
         }
     ).extractor
 
-    return extract_from_path(Path(path), extractor)
+    return extract_from_path(Path(path), extractor, **kwargs)
 
 
 @deprecate_fn_path
 def extract_from_path(
     source: Path,
     extractor: FileType,
+    **kwargs: dict,
 ) -> DataTree:
     """
     Extracts data and metadata from the provided path using the supplied extractor.
@@ -108,7 +109,7 @@ def extract_from_path(
     func = getattr(m, "extract")
 
     # Func should always return a xarray.DataTree
-    ret: DataTree = func(source=source, **vars(extractor))
+    ret: DataTree = func(source=source, **vars(extractor), **kwargs)
     jsonize_orig_meta(ret)
 
     ret.attrs.update(
@@ -127,6 +128,7 @@ def extract_from_path(
 def extract_from_bytes(
     source: bytes,
     extractor: FileType,
+    **kwargs: dict,
 ) -> DataTree:
     """
     Extracts data and metadata from the provided raw bytes using the supplied extractor.

src/yadg/extractors/agilent/dx.py

@@ -68,5 +68,5 @@ def extract_from_path(
         for ffn in sorted(filenames):
             path = Path(tempdir) / ffn
             fdt = extract_ch(source=path, timezone=timezone, **kwargs)
-            dt = dgutils.merge_dicttrees(dt, fdt.to_dict(), "identical")
+            dt = dgutils.merge_dicttrees(dt, fdt.to_dict(), True)
     return DataTree.from_dict(dt)

src/yadg/extractors/fusion/zip.py

@@ -50,8 +50,10 @@
 from yadg import dgutils
 from pathlib import Path
 from yadg.extractors import get_extract_dispatch
+import logging
 
 extract = get_extract_dispatch()
+logger = logging.getLogger(__name__)
 
 
 @extract.register(Path)
@@ -63,14 +65,19 @@ def extract_from_path(
     **kwargs: dict,
 ) -> DataTree:
     zf = zipfile.ZipFile(source)
+    strict_merge = not kwargs.get("ignore_merge_errors", False)
+    if strict_merge is False:
+        logger.info("Will drop metadata conflicts in individual fusion-data files.")
+
     with tempfile.TemporaryDirectory() as tempdir:
         zf.extractall(tempdir)
         dt = None
         filenames = [ffn for ffn in os.listdir(tempdir) if ffn.endswith("fusion-data")]
         for ffn in sorted(filenames):
+            logger.debug("Processing filename '%s'", ffn)
             path = Path(tempdir) / ffn
             fdt = extract_json(
                 source=path, timezone=timezone, encoding=encoding, **kwargs
             )
-            dt = dgutils.merge_dicttrees(dt, fdt.to_dict(), "identical")
+            dt = dgutils.merge_dicttrees(dt, fdt.to_dict(), strict_merge)
     return DataTree.from_dict(dt)

src/yadg/main.py

@@ -177,6 +177,13 @@ def run_with_arguments():
         default=None,
         type=str,
     )
+    extract.add_argument(
+        "--ignore-merge-errors",
+        dest="ignore_merge_errors",
+        action="store_true",
+        help="Ignore metadata merge errors while processing multiple files in a step.",
+        default=False,
+    )
     extract.set_defaults(func=subcommands.extract)
 
     # parse subparser args