Merge branch 'main' into zarrv3_with_uv -- Probably broken

Author: BrianMichell

docs/index.md

@@ -17,6 +17,7 @@ maxdepth: 1
 ---
 installation
 notebooks/quickstart
+notebooks/creation
 notebooks/compression
 notebooks/rechunking
 usage

docs/notebooks/creation.ipynb

@@ -7,6 +7,12 @@
 from mdio.api.convenience import copy_mdio
 from mdio.converters import mdio_to_segy
 from mdio.converters import segy_to_mdio
+from mdio.core.dimension import Dimension
+from mdio.core.factory import MDIOCreateConfig
+from mdio.core.factory import MDIOVariableConfig
+from mdio.core.factory import create_empty
+from mdio.core.factory import create_empty_like
+from mdio.core.grid import Grid
 
 
 __all__ = [
@@ -15,6 +21,12 @@
     "copy_mdio",
     "mdio_to_segy",
     "segy_to_mdio",
+    "Dimension",
+    "MDIOCreateConfig",
+    "MDIOVariableConfig",
+    "create_empty",
+    "create_empty_like",
+    "Grid",
 ]
 
 

src/mdio/__init__.py

@@ -129,6 +129,8 @@ class MDIOAccessor:
         we can directly unpack it and use it further down our code.
     """
 
+    _stats_keys = {"mean", "std", "rms", "min", "max"}
+
     _array_load_function_mapper = {
         "zarr": open_zarr_array,
         "dask": open_zarr_array_dask,
@@ -154,16 +156,13 @@ def __init__(
 
         # Set private attributes for public interface.
         # Pep8 complains because they are defined outside __init__
-        self._binary_header = None
         self._chunks = None
         self._live_mask = None
         self._root = None
         self._n_dim = None
         self._orig_chunks = None
         self._store = None
         self._shape = None
-        self._stats = None
-        self._text_header = None
         self._trace_count = None
 
         # Private attributes
@@ -205,13 +204,25 @@ def _validate_store(self, storage_options):
 
     def _connect(self):
         """Open the zarr root."""
-        if self.mode in {"r", "r+"}:
-            self.root = zarr.open_consolidated(store=self.store, mode=self.mode)
-        elif self.mode == "w":
-            self.root = zarr.open(store=self.store, mode="r+")
-        else:
-            msg = f"Invalid mode: {self.mode}"
-            raise ValueError(msg)
+        try:
+            if self.mode in {"r", "r+"}:
+                self.root = zarr.open_consolidated(store=self.store, mode=self.mode)
+            elif self.mode == "w":
+                self.root = zarr.open(store=self.store, mode="r+")
+            else:
+                msg = f"Invalid mode: {self.mode}"
+                raise ValueError(msg)
+        except KeyError as e:
+            msg = (
+                f"MDIO file not found or corrupt at {self.store.path}. "
+                "Please check the URL or ensure it is not a deprecated "
+                "version of MDIO file."
+            )
+            raise MDIONotFoundError(msg) from e
+
+    def _consolidate_metadata(self) -> None:
+        """Flush optimized MDIO metadata, run after modifying it."""
+        zarr.consolidate_metadata(self.root.store)
 
     def _deserialize_grid(self):
         """Deserialize grid from Zarr metadata."""
@@ -220,12 +231,6 @@ def _deserialize_grid(self):
     def _set_attributes(self):
         """Deserialize attributes from Zarr metadata."""
         self.trace_count = self.root.attrs["trace_count"]
-        self.stats = {
-            key: self.root.attrs[key] for key in ["mean", "std", "rms", "min", "max"]
-        }
-
-        self.text_header = self._metadata_group.attrs["text_header"]
-        self.binary_header = self._metadata_group.attrs["binary_header"]
 
         # Grid based attributes
         self.shape = self.grid.shape
@@ -332,26 +337,28 @@ def trace_count(self, value: int) -> None:
     @property
     def text_header(self) -> list:
         """Get seismic text header."""
-        return self._text_header
+        return self._metadata_group.attrs["text_header"]
 
     @text_header.setter
     def text_header(self, value: list) -> None:
         """Validate and set seismic text header."""
-        if not isinstance(value, list):
+        if not isinstance(value, list) or len(value) != 40:
             raise AttributeError("Text header must be a list of str with 40 elements")
-        self._text_header = value
+        self._metadata_group.attrs["text_header"] = value
+        self._consolidate_metadata()
 
     @property
     def binary_header(self) -> dict:
         """Get seismic binary header metadata."""
-        return self._binary_header
+        return self._metadata_group.attrs["binary_header"]
 
     @binary_header.setter
     def binary_header(self, value: dict) -> None:
         """Validate and set seismic binary header metadata."""
         if not isinstance(value, dict):
             raise AttributeError("Binary header has to be a dictionary type collection")
-        self._binary_header = value
+        self._metadata_group.attrs["binary_header"] = value
+        self._consolidate_metadata()
 
     @property
     def chunks(self) -> tuple[int, ...]:
@@ -366,12 +373,16 @@ def chunks(self, value: tuple[int, ...]) -> None:
     @property
     def stats(self) -> dict:
         """Get global statistics like min/max/rms/std."""
-        return self._stats
+        return {key: self.root.attrs[key] for key in self._stats_keys}
 
     @stats.setter
     def stats(self, value: dict) -> None:
         """Set global statistics like min/max/rms/std."""
-        self._stats = value
+        if not isinstance(value, dict) or not self._stats_keys.issubset(value.keys()):
+            msg = f"For settings status, you must provide keys: {self._stats_keys}"
+            raise AttributeError(msg)
+        self.root.attrs.update(value)
+        self._consolidate_metadata()
 
     @property
     def _metadata_group(self) -> zarr.Group:
@@ -404,6 +415,7 @@ def __getitem__(self, item: int | tuple) -> npt.ArrayLike | da.Array | tuple:
     def __setitem__(self, key: int | tuple, value: npt.ArrayLike) -> None:
         """Data setter."""
         self._traces[key] = value
+        self._live_mask[key] = True
 
     def coord_to_index(
         self,
@@ -644,10 +656,10 @@ def __init__(
         memory_cache_size: int = 0,
         disk_cache: bool = False,
     ):  # TODO: Disabled all caching by default, sometimes causes performance issues
-        """Initialize super class with `r+` permission."""
+        """Initialize accessor class with `w` permission."""
         super().__init__(
             mdio_path_or_buffer=mdio_path_or_buffer,
-            mode="r+",
+            mode="w",
             access_pattern=access_pattern,
             storage_options=storage_options,
             return_metadata=return_metadata,

src/mdio/api/accessor.py

@@ -12,20 +12,26 @@
 FLOAT64_MIN = np.finfo("float64").min
 FLOAT64_MAX = np.finfo("float64").max
 
-INT8_MIN = -0x80
-INT8_MAX = 0x7F
+INT8_MIN = np.iinfo("int8").min
+INT8_MAX = np.iinfo("int8").max
 
-INT16_MIN = -0x8000
-INT16_MAX = 0x7FFF
+INT16_MIN = np.iinfo("int16").min
+INT16_MAX = np.iinfo("int16").max
 
-INT32_MIN = -0x80000000
-INT32_MAX = 0x7FFFFFFF
+INT32_MIN = np.iinfo("int32").min
+INT32_MAX = np.iinfo("int32").max
 
-UINT8_MIN = 0x0
-UINT8_MAX = 0xFF
+INT64_MIN = np.iinfo("int64").min
+INT64_MAX = np.iinfo("int64").max
 
-UINT16_MIN = 0x0
-UINT16_MAX = 0xFFFF
+UINT8_MIN = 0
+UINT8_MAX = np.iinfo("uint8").max
 
-UINT32_MIN = 0x0
-UINT32_MAX = 0xFFFFFFFF
+UINT16_MIN = 0
+UINT16_MAX = np.iinfo("uint16").max
+
+UINT32_MIN = 0
+UINT32_MAX = np.iinfo("uint32").max
+
+UINT64_MIN = 0
+UINT64_MAX = np.iinfo("uint64").max