Change default netCDF engine to use h5netcdf and add netcdf_engine_order (#10755)

* Add option for netcdf_engine_order

The default `engine` when reading/writing netCDF files is now h5netcdf
or scipy, which are typically faster than the prior default of netCDF4-python.
You can control this default behavior explicitly via the new
`netcdf_engine_order` parameter in `set_options()`, e.g.,
`xr.set_options(netcdf_engine_order=['netcdf4', 'scipy', 'h5netcdf'])` to
restore the prior defaults.

I've also updated the documentation page which misled @lesserwhirls
about Xarray supporting invalid netCDF files without
`invalid_netcdf=True`.

Fixes #10657

* Fix test failures

* Automatically support NCZarr

* Revert "Automatically support NCZarr"

This reverts commit 18fe84f.

* Reapply "Automatically support NCZarr"

This reverts commit 4131449.

* Fix mypy

* spelling

* Improve typing for _normalize_path()

* hard code engine="netcdf4" for test_encoding_enum__no_fill_value

* Fix reading netcdf3 files with open_datatree

* Set engine in test_encoding_enum__multiple_variable_with_enum

* set yet another test to only use netcdf4

Author: shoyer

doc/user-guide/io.rst

@@ -591,8 +591,8 @@ The library ``h5netcdf`` allows writing some dtypes that aren't
 allowed in netCDF4 (see
 `h5netcdf documentation <https://github.com/h5netcdf/h5netcdf#invalid-netcdf-files>`_).
 This feature is available through :py:meth:`DataArray.to_netcdf` and
-:py:meth:`Dataset.to_netcdf` when used with ``engine="h5netcdf"``
-and currently raises a warning unless ``invalid_netcdf=True`` is set.
+:py:meth:`Dataset.to_netcdf` when used with ``engine="h5netcdf"``, only if
+``invalid_netcdf=True`` is explicitly set.
 
 .. warning::
 

doc/whats-new.rst

@@ -29,9 +29,12 @@ Breaking changes
   dataset in-place. (:issue:`10167`)
   By `Maximilian Roos <https://github.com/max-sixty>`_.
 
-- The default ``engine`` when reading/writing netCDF files in-memory is now
-  netCDF4, consistent with Xarray's default ``engine`` when read/writing netCDF
-  files to disk (:pull:`10624`).
+- The default ``engine`` when reading/writing netCDF files is now h5netcdf
+  or scipy, which are typically faster than the prior default of netCDF4-python.
+  You can control this default behavior explicitly via the new
+  ``netcdf_engine_order`` parameter in :py:func:`~xarray.set_options`, e.g.,
+  ``xr.set_options(netcdf_engine_order=['netcdf4', 'scipy', 'h5netcdf'])`` to
+  restore the prior defaults (:issue:`10657`).
   By `Stephan Hoyer <https://github.com/shoyer>`_.
 
 Deprecations

xarray/backends/api.py

@@ -415,9 +415,10 @@ def open_dataset(
         , installed backend \
         or subclass of xarray.backends.BackendEntrypoint, optional
         Engine to use when reading files. If not provided, the default engine
-        is chosen based on available dependencies, with a preference for
-        "netcdf4". A custom backend class (a subclass of ``BackendEntrypoint``)
-        can also be used.
+        is chosen based on available dependencies, by default preferring
+        "h5netcdf" over "scipy" over "netcdf4" (customizable via
+        ``netcdf_engine_order`` in ``xarray.set_options()``). A custom backend
+        class (a subclass of ``BackendEntrypoint``) can also be used.
     chunks : int, dict, 'auto' or None, default: None
         If provided, used to load the data into dask arrays.
 
@@ -658,8 +659,10 @@ def open_dataarray(
         , installed backend \
         or subclass of xarray.backends.BackendEntrypoint, optional
         Engine to use when reading files. If not provided, the default engine
-        is chosen based on available dependencies, with a preference for
-        "netcdf4".
+        is chosen based on available dependencies, by default preferring
+        "h5netcdf" over "scipy" over "netcdf4" (customizable via
+        ``netcdf_engine_order`` in ``xarray.set_options()``). A custom backend
+        class (a subclass of ``BackendEntrypoint``) can also be used.
     chunks : int, dict, 'auto' or None, default: None
         If provided, used to load the data into dask arrays.
 
@@ -882,9 +885,10 @@ def open_datatree(
     engine : {"netcdf4", "h5netcdf", "zarr", None}, \
              installed backend or xarray.backends.BackendEntrypoint, optional
         Engine to use when reading files. If not provided, the default engine
-        is chosen based on available dependencies, with a preference for
-        "netcdf4". A custom backend class (a subclass of ``BackendEntrypoint``)
-        can also be used.
+        is chosen based on available dependencies, by default preferring
+        "h5netcdf" over "netcdf4" (customizable via ``netcdf_engine_order`` in
+        ``xarray.set_options()``). A custom backend class (a subclass of
+        ``BackendEntrypoint``) can also be used.
     chunks : int, dict, 'auto' or None, default: None
         If provided, used to load the data into dask arrays.
 
@@ -1040,7 +1044,7 @@ def open_datatree(
         kwargs.update(backend_kwargs)
 
     if engine is None:
-        engine = plugins.guess_engine(filename_or_obj)
+        engine = plugins.guess_engine(filename_or_obj, must_support_groups=True)
 
     if from_array_kwargs is None:
         from_array_kwargs = {}
@@ -1126,8 +1130,10 @@ def open_groups(
     engine : {"netcdf4", "h5netcdf", "zarr", None}, \
              installed backend or xarray.backends.BackendEntrypoint, optional
         Engine to use when reading files. If not provided, the default engine
-        is chosen based on available dependencies, with a preference for
-        "netcdf4". A custom backend class (a subclass of ``BackendEntrypoint``)
+        is chosen based on available dependencies, by default preferring
+        "h5netcdf" over "netcdf4" (customizable via ``netcdf_engine_order`` in
+        ``xarray.set_options()``). A custom backend class (a subclass of
+        ``BackendEntrypoint``) can also be used.
         can also be used.
     chunks : int, dict, 'auto' or None, default: None
         If provided, used to load the data into dask arrays.
@@ -1283,7 +1289,7 @@ def open_groups(
         kwargs.update(backend_kwargs)
 
     if engine is None:
-        engine = plugins.guess_engine(filename_or_obj)
+        engine = plugins.guess_engine(filename_or_obj, must_support_groups=True)
 
     if from_array_kwargs is None:
         from_array_kwargs = {}
@@ -1443,8 +1449,10 @@ def open_mfdataset(
         , installed backend \
         or subclass of xarray.backends.BackendEntrypoint, optional
         Engine to use when reading files. If not provided, the default engine
-        is chosen based on available dependencies, with a preference for
-        "netcdf4".
+        is chosen based on available dependencies, by default preferring
+        "h5netcdf" over "scipy" over "netcdf4" (customizable via
+        ``netcdf_engine_order`` in ``xarray.set_options()``). A custom backend
+        class (a subclass of ``BackendEntrypoint``) can also be used.
     data_vars : {"minimal", "different", "all"} or list of str, default: "all"
         These data variables will be concatenated together:
           * "minimal": Only data variables in which the dimension already

xarray/backends/common.py

@@ -53,14 +53,18 @@
 
 
 @overload
-def _normalize_path(path: str | os.PathLike) -> str: ...
+def _normalize_path(path: os.PathLike) -> str: ...
+
+
+@overload
+def _normalize_path(path: str) -> str: ...
 
 
 @overload
 def _normalize_path(path: T) -> T: ...
 
 
-def _normalize_path(path: str | os.PathLike | T) -> str | T:
+def _normalize_path(path: os.PathLike | str | T) -> str | T:
     """
     Normalize pathlikes to string.
 
@@ -85,7 +89,7 @@ def _normalize_path(path: str | os.PathLike | T) -> str | T:
     if isinstance(path, str) and not is_remote_uri(path):
         path = os.path.abspath(os.path.expanduser(path))
 
-    return path  # type:ignore [return-value]
+    return path  # type: ignore[return-value]
 
 
 @overload
@@ -749,11 +753,15 @@ class BackendEntrypoint:
     url : str, default: ""
         A string with the URL to the backend's documentation.
         The setting of this attribute is not mandatory.
+    supports_groups : bool, default: False
+        Whether the backend supports opening groups (via open_datatree and
+        open_groups_as_dict) or not.
     """
 
     open_dataset_parameters: ClassVar[tuple | None] = None
     description: ClassVar[str] = ""
     url: ClassVar[str] = ""
+    supports_groups: ClassVar[bool] = False
 
     def __repr__(self) -> str:
         txt = f"<{type(self).__name__}>"
@@ -808,6 +816,8 @@ def open_datatree(
     ) -> DataTree:
         """
         Backend open_datatree method used by Xarray in :py:func:`~xarray.open_datatree`.
+
+        If implemented, set the class variable supports_groups to True.
         """
 
         raise NotImplementedError()
@@ -830,6 +840,8 @@ def open_groups_as_dict(
         This function exists to provide a universal way to open all groups in a file,
         before applying any additional consistency checks or requirements necessary
         to create a `DataTree` object (typically done using :py:meth:`~xarray.DataTree.from_dict`).
+
+        If implemented, set the class variable supports_groups to True.
         """
 
         raise NotImplementedError()

xarray/backends/h5netcdf_.py

@@ -459,6 +459,7 @@ class H5netcdfBackendEntrypoint(BackendEntrypoint):
         "Open netCDF (.nc, .nc4 and .cdf) and most HDF5 files using h5netcdf in Xarray"
     )
     url = "https://docs.xarray.dev/en/stable/generated/xarray.backends.H5netcdfBackendEntrypoint.html"
+    supports_groups = True
 
     def guess_can_open(self, filename_or_obj: T_PathFileOrDataStore) -> bool:
         filename_or_obj = _normalize_filename_or_obj(filename_or_obj)

xarray/backends/netCDF4_.py

@@ -698,6 +698,7 @@ class NetCDF4BackendEntrypoint(BackendEntrypoint):
         "Open netCDF (.nc, .nc4 and .cdf) and most HDF5 files using netCDF4 in Xarray"
     )
     url = "https://docs.xarray.dev/en/stable/generated/xarray.backends.NetCDF4BackendEntrypoint.html"
+    supports_groups = True
 
     def guess_can_open(self, filename_or_obj: T_PathFileOrDataStore) -> bool:
         if isinstance(filename_or_obj, str) and is_remote_uri(filename_or_obj):

xarray/backends/plugins.py

@@ -9,6 +9,7 @@
 from typing import TYPE_CHECKING, Any
 
 from xarray.backends.common import BACKEND_ENTRYPOINTS, BackendEntrypoint
+from xarray.core.options import OPTIONS
 from xarray.core.utils import module_available
 
 if TYPE_CHECKING:
@@ -18,8 +19,6 @@
     from xarray.backends.common import AbstractDataStore
     from xarray.core.types import ReadBuffer
 
-NETCDF_BACKENDS_ORDER = ["netcdf4", "h5netcdf", "scipy"]
-
 
 def remove_duplicates(entrypoints: EntryPoints) -> list[EntryPoint]:
     # sort and group entrypoints by name
@@ -91,8 +90,8 @@ def set_missing_parameters(
 def sort_backends(
     backend_entrypoints: dict[str, type[BackendEntrypoint]],
 ) -> dict[str, type[BackendEntrypoint]]:
-    ordered_backends_entrypoints = {}
-    for be_name in NETCDF_BACKENDS_ORDER:
+    ordered_backends_entrypoints: dict[str, type[BackendEntrypoint]] = {}
+    for be_name in OPTIONS["netcdf_engine_order"]:
         if backend_entrypoints.get(be_name) is not None:
             ordered_backends_entrypoints[be_name] = backend_entrypoints.pop(be_name)
     ordered_backends_entrypoints.update(
@@ -144,10 +143,13 @@ def guess_engine(
     | bytes
     | memoryview
     | AbstractDataStore,
+    must_support_groups: bool = False,
 ) -> str | type[BackendEntrypoint]:
     engines = list_engines()
 
     for engine, backend in engines.items():
+        if must_support_groups and not backend.supports_groups:
+            continue
         try:
             if backend.guess_can_open(store_spec):
                 return engine
@@ -162,6 +164,8 @@ def guess_engine(
     for engine, (_, backend_cls) in BACKEND_ENTRYPOINTS.items():
         try:
             backend = backend_cls()
+            if must_support_groups and not backend.supports_groups:
+                continue
             if backend.guess_can_open(store_spec):
                 compatible_engines.append(engine)
         except Exception:
@@ -180,6 +184,15 @@ def guess_engine(
                 "https://docs.xarray.dev/en/stable/getting-started-guide/installing.html\n"
                 "https://docs.xarray.dev/en/stable/user-guide/io.html"
             )
+        elif must_support_groups:
+            error_msg = (
+                "xarray is unable to open this file because it has no currently "
+                "installed IO backends that support reading groups (e.g., h5netcdf "
+                "or netCDF4-python). Xarray's read/write support requires "
+                "installing optional IO dependencies, see:\n"
+                "https://docs.xarray.dev/en/stable/getting-started-guide/installing.html\n"
+                "https://docs.xarray.dev/en/stable/user-guide/io"
+            )
         else:
             error_msg = (
                 "xarray is unable to open this file because it has no currently "

xarray/backends/scipy_.py

@@ -323,7 +323,7 @@ def _normalize_filename_or_obj(
     if isinstance(filename_or_obj, bytes | memoryview):
         return io.BytesIO(filename_or_obj)
     else:
-        return _normalize_path(filename_or_obj)  # type: ignore[return-value]
+        return _normalize_path(filename_or_obj)
 
 
 class ScipyBackendEntrypoint(BackendEntrypoint):