Fix or suppress known warnings (TGSAI#655)

* Fix warnings

* Address PR review comments

* remove tests irrelevant to MDIO

* refactor warning handling and remove unnecessary ones

* remove warning filter from quickstart

---------

Co-authored-by: Altay Sansal <[email protected]>

Author: dmitriyrepin

docs/tutorials/quickstart.ipynb

@@ -46,25 +46,18 @@
    ]
   },
   {
+   "metadata": {},
    "cell_type": "code",
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "outputs": [],
+   "execution_count": null,
    "source": [
     "import os\n",
-    "import warnings\n",
-    "\n",
-    "warnings.filterwarnings(\"ignore\")\n",
     "\n",
     "os.environ[\"MDIO__IMPORT__CLOUD_NATIVE\"] = \"true\"\n",
     "os.environ[\"MDIO__IMPORT__SAVE_SEGY_FILE_HEADER\"] = \"true\"\n",
     "\n",
     "input_url = \"http://s3.amazonaws.com/teapot/filt_mig.sgy\""
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "markdown",

src/mdio/__main__.py

@@ -16,7 +16,7 @@
 KNOWN_MODULES = ["segy.py", "copy.py", "info.py"]
 
 
-class MyCLI(click.MultiCommand):
+class MyCLI(click.Group):
     """CLI generator via plugin design pattern.
 
     This class dynamically loads command modules from the specified `plugin_folder`. If the
@@ -26,8 +26,8 @@ class MyCLI(click.MultiCommand):
 
     Args:
         plugin_folder: Path to the directory containing command modules
-        *args: Variable length argument list passed to the click.MultiCommand.
-        **kwargs: Arbitrary keyword arguments passed to the click.MultiCommand.
+        *args: Variable length argument list passed to the click.Group.
+        **kwargs: Arbitrary keyword arguments passed to the click.Group.
     """
 
     def __init__(self, plugin_folder: Path, *args: Any, **kwargs: Any):  # noqa: ANN401

src/mdio/api/io.py

@@ -13,6 +13,7 @@
 from xarray.backends.api import to_zarr as xr_to_zarr
 
 from mdio.constants import ZarrFormat
+from mdio.core.zarr_io import zarr_warnings_suppress_unstable_structs_v3
 
 if TYPE_CHECKING:
     from collections.abc import Mapping
@@ -51,11 +52,13 @@ def open_mdio(input_path: UPath | Path | str, chunks: T_Chunks = None) -> xr_Dat
     input_path = _normalize_path(input_path)
     storage_options = _normalize_storage_options(input_path)
     zarr_format = zarr.config.get("default_zarr_format")
+
     return xr_open_zarr(
         input_path.as_posix(),
         chunks=chunks,
         storage_options=storage_options,
         mask_and_scale=zarr_format == ZarrFormat.V3,  # off for v2, on for v3
+        consolidated=zarr_format == ZarrFormat.V2,  # on for v2, off for v3
     )
 
 
@@ -86,13 +89,15 @@ def to_mdio(  # noqa: PLR0913
     output_path = _normalize_path(output_path)
     storage_options = _normalize_storage_options(output_path)
     zarr_format = zarr.config.get("default_zarr_format")
-    xr_to_zarr(
-        dataset,
-        store=output_path.as_posix(),  # xarray doesn't like URI when file:// is protocol
-        mode=mode,
-        compute=compute,
-        consolidated=zarr_format == ZarrFormat.V2,  # off for v3, on for v2
-        region=region,
-        storage_options=storage_options,
-        write_empty_chunks=False,
-    )
+
+    with zarr_warnings_suppress_unstable_structs_v3():
+        xr_to_zarr(
+            dataset,
+            store=output_path.as_posix(),  # xarray doesn't like URI when file:// is protocol
+            mode=mode,
+            compute=compute,
+            consolidated=zarr_format == ZarrFormat.V2,  # on for v2, off for v3
+            region=region,
+            storage_options=storage_options,
+            write_empty_chunks=False,
+        )

src/mdio/builder/schemas/v1/units.py

@@ -10,7 +10,7 @@
 from mdio.builder.schemas.units import create_unit_model
 
 ureg = UnitRegistry()
-ureg.default_format = "~C"  # compact, abbreviated (symbol).
+ureg.formatter.default_format = "~C"  # compact, abbreviated (symbol).
 
 
 class LengthUnitEnum(UnitEnum):

src/mdio/builder/xarray_builder.py

@@ -222,7 +222,7 @@ def to_xarray_dataset(mdio_ds: Dataset) -> xr_Dataset:  # noqa: PLR0912
 
         encoding = {
             "chunks": original_chunks,
-            "compressor": _convert_compressor(v.compressor),
+            "compressors": _convert_compressor(v.compressor),
             fill_value_key: fill_value,
         }
 

src/mdio/core/zarr_io.py

@@ -0,0 +1,23 @@
+"""Utilities to open/write Zarr files."""
+
+from __future__ import annotations
+
+import warnings
+from contextlib import contextmanager
+from typing import TYPE_CHECKING
+
+from zarr.errors import UnstableSpecificationWarning
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+
+@contextmanager
+def zarr_warnings_suppress_unstable_structs_v3() -> Generator[None, None, None]:
+    """Context manager to suppress Zarr V3 unstable structured array warning."""
+    warn = r"The data type \((.*?)\) does not have a Zarr V3 specification\."
+    warnings.filterwarnings("ignore", message=warn, category=UnstableSpecificationWarning)
+    try:
+        yield
+    finally:
+        pass

src/mdio/segy/utilities.py

@@ -112,6 +112,10 @@ def find_trailing_ones_index(dim_blocks: tuple[int, ...]) -> int:
     return total_dims - trailing_ones_count
 
 
+# TODO (Dmitriy Repin): Investigate the following warning generated at test_3d_export
+# https://github.com/TGSAI/mdio-python/issues/657
+# "The specified chunks separate the stored chunks along dimension "inline" starting at index 256.
+# This could degrade performance. Instead, consider rechunking after loading."
 def segy_export_rechunker(
     chunks: dict[str, int],
     sizes: dict[str, int],

tests/unit/test_indexing.py

@@ -91,9 +91,9 @@ def mock_trace_worker(
     # We used a 2D selection with 2D index_slices
     assert grid_map.shape == (3, 4, 20)
     # We used a 3D selection with isel()
-    assert tuple(dataset.dims[d] for d in region) == (3, 4, 5)
+    assert tuple(dataset.sizes[d] for d in region) == (3, 4, 5)
 
-    dimension_names = list(dataset.dims)
+    dimension_names = list(dataset.sizes)
 
     slice0 = region[dimension_names[0]]
     slice1 = region[dimension_names[1]]

tests/unit/v1/test_dataset_serializer.py

@@ -4,8 +4,6 @@
 
 import numpy as np
 import pytest
-from xarray import DataArray as xr_DataArray
-from zarr import zeros as zarr_zeros
 from zarr.codecs import BloscCodec
 
 from mdio import to_mdio
@@ -295,7 +293,7 @@ def test_to_xarray_dataset(tmp_path: Path) -> None:
     xr_ds = to_xarray_dataset(dataset)
 
     file_path = output_path(tmp_path, f"{xr_ds.attrs['name']}", debugging=False)
-    xr_ds.to_zarr(store=file_path, mode="w", compute=False)
+    to_mdio(dataset=xr_ds, output_path=file_path, mode="w", compute=False)
 
 
 def test_seismic_poststack_3d_acceptance_to_xarray_dataset(tmp_path: Path) -> None:
@@ -306,62 +304,3 @@ def test_seismic_poststack_3d_acceptance_to_xarray_dataset(tmp_path: Path) -> No
 
     file_path = output_path(tmp_path, f"{xr_ds.attrs['name']}", debugging=False)
     to_mdio(xr_ds, output_path=file_path, mode="w-", compute=False)
-
-
-def test_to_zarr_from_zarr_zeros_1(tmp_path: Path) -> None:
-    """Test writing XArray dataset with data as Zarr zero array to Zarr.
-
-    Set encoding in as DataArray attributes
-    """
-    # Create a data type and the fill value
-    dtype = np.dtype([("inline", "int32"), ("cdp_x", "float64")])
-
-    my_attr_encoding = {"fill_value": np.void((0, 0), dtype=dtype)}
-
-    # Create a zarr array using the data type,
-    # Specify encoding as the array attribute
-    data = zarr_zeros((36, 36), dtype=dtype)
-    aa = xr_DataArray(name="myattr", data=data)
-    aa.encoding = my_attr_encoding
-
-    file_path = output_path(tmp_path, "to_zarr/zarr_zarr_zerros_1", debugging=False)
-    aa.to_zarr(file_path, mode="w", compute=False)
-
-
-def test_to_zarr_from_zarr_zeros_2(tmp_path: Path) -> None:
-    """Test writing XArray dataset with data as Zarr zero array to Zarr.
-
-    Set encoding in the to_zar method
-    """
-    # Create a data type and the fill value
-    dtype = np.dtype([("inline", "int32"), ("cdp_x", "float64")])
-
-    my_attr_encoding = {"fill_value": np.void((0, 0), dtype=dtype)}
-
-    # Create a zarr array using the data type,
-    # Do not specify encoding as the array attribute
-    data = zarr_zeros((36, 36), dtype=dtype)
-    aa = xr_DataArray(name="myattr", data=data)
-
-    file_path = output_path(tmp_path, "to_zarr/zarr_zarr_zerros_2", debugging=False)
-    # Specify encoding per array
-    encoding = {"myattr": my_attr_encoding}
-    aa.to_zarr(file_path, mode="w", encoding=encoding, compute=False)
-
-
-def test_to_zarr_from_np(tmp_path: Path) -> None:
-    """Test writing XArray dataset with data as NumPy array to Zarr."""
-    # Create a data type and the fill value
-    dtype = np.dtype([("inline", "int32"), ("cdp_x", "float64")])
-
-    my_attr_encoding = {"fill_value": np.void((0, 0), dtype=dtype)}
-
-    # Create a zarr array using the data type
-    # Do not specify encoding as the array attribute
-    data = np.zeros((36, 36), dtype=dtype)
-    aa = xr_DataArray(name="myattr", data=data)
-
-    file_path = output_path(tmp_path, "to_zarr/zarr_np", debugging=False)
-    # Specify encoding per array
-    encoding = {"myattr": my_attr_encoding}
-    aa.to_zarr(file_path, mode="w", encoding=encoding, compute=False)