allow refreshing of backends (#7523)

* allow refreshing of backends

* fix forgotten inheritance

* fix typing

* fix typing in tests

* add refresh func to xr.backends

* add to whats-new

* add func to api

* add tests for list_engines and refresh_engines

* try fixing test issues

* fallback for pyhton 3.9

* use sys.version instead of try

* use sys.version in TYPE_CHECKING as well

* try fixing type errors in python 3.9

* check if clearing cache makes test pass

* add sys.version logic to test

* try fixing parallel test fail

* refactor BackEntrypoint available logic

* remove obsolete code

* fix typing

* fix typing2

* update api-hidden

* remove comma

* update whats-new to 2023.04

* Delete cfgrib_.py

* Update xarray/tests/test_plugins.py

Co-authored-by: Joe Hamman <[email protected]>

* Update xarray/tests/test_plugins.py

Co-authored-by: Joe Hamman <[email protected]>

---------

Co-authored-by: Deepak Cherian <[email protected]>
Co-authored-by: Joe Hamman <[email protected]>

Author: 3 people

doc/api-hidden.rst

@@ -484,7 +484,6 @@
    backends.NetCDF4DataStore.is_remote
    backends.NetCDF4DataStore.lock
 
-   backends.NetCDF4BackendEntrypoint.available
    backends.NetCDF4BackendEntrypoint.description
    backends.NetCDF4BackendEntrypoint.url
    backends.NetCDF4BackendEntrypoint.guess_can_open
@@ -517,7 +516,6 @@
    backends.H5NetCDFStore.sync
    backends.H5NetCDFStore.ds
 
-   backends.H5netcdfBackendEntrypoint.available
    backends.H5netcdfBackendEntrypoint.description
    backends.H5netcdfBackendEntrypoint.url
    backends.H5netcdfBackendEntrypoint.guess_can_open
@@ -532,7 +530,6 @@
    backends.PseudoNetCDFDataStore.open_store_variable
    backends.PseudoNetCDFDataStore.ds
 
-   backends.PseudoNetCDFBackendEntrypoint.available
    backends.PseudoNetCDFBackendEntrypoint.description
    backends.PseudoNetCDFBackendEntrypoint.url
    backends.PseudoNetCDFBackendEntrypoint.guess_can_open
@@ -547,7 +544,6 @@
    backends.PydapDataStore.open
    backends.PydapDataStore.open_store_variable
 
-   backends.PydapBackendEntrypoint.available
    backends.PydapBackendEntrypoint.description
    backends.PydapBackendEntrypoint.url
    backends.PydapBackendEntrypoint.guess_can_open
@@ -575,7 +571,6 @@
    backends.ScipyDataStore.sync
    backends.ScipyDataStore.ds
 
-   backends.ScipyBackendEntrypoint.available
    backends.ScipyBackendEntrypoint.description
    backends.ScipyBackendEntrypoint.url
    backends.ScipyBackendEntrypoint.guess_can_open
@@ -596,13 +591,11 @@
    backends.ZarrStore.sync
    backends.ZarrStore.ds
 
-   backends.ZarrBackendEntrypoint.available
    backends.ZarrBackendEntrypoint.description
    backends.ZarrBackendEntrypoint.url
    backends.ZarrBackendEntrypoint.guess_can_open
    backends.ZarrBackendEntrypoint.open_dataset
 
-   backends.StoreBackendEntrypoint.available
    backends.StoreBackendEntrypoint.description
    backends.StoreBackendEntrypoint.url
    backends.StoreBackendEntrypoint.guess_can_open

doc/api.rst

@@ -1096,6 +1096,7 @@ Advanced API
    backends.BackendArray
    backends.BackendEntrypoint
    backends.list_engines
+   backends.refresh_engines
 
 Default, pandas-backed indexes built-in Xarray:
 

doc/whats-new.rst

@@ -26,6 +26,8 @@ New Features
   (:issue:`7686`, :pull:`7689`).
   By `Joe Hamman <https://github.com/jhamman>`_.
 
+- Allow refreshing backend engines with :py:meth:`xarray.backends.refresh_engines` (:issue:`7478`, :pull:`7523`).
+  By `Michael Niklas <https://github.com/headtr1ck>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~

xarray/backends/__init__.py

@@ -12,7 +12,7 @@
 from xarray.backends.h5netcdf_ import H5netcdfBackendEntrypoint, H5NetCDFStore
 from xarray.backends.memory import InMemoryDataStore
 from xarray.backends.netCDF4_ import NetCDF4BackendEntrypoint, NetCDF4DataStore
-from xarray.backends.plugins import list_engines
+from xarray.backends.plugins import list_engines, refresh_engines
 from xarray.backends.pseudonetcdf_ import (
     PseudoNetCDFBackendEntrypoint,
     PseudoNetCDFDataStore,
@@ -46,4 +46,5 @@
     "StoreBackendEntrypoint",
     "ZarrBackendEntrypoint",
     "list_engines",
+    "refresh_engines",
 ]

xarray/backends/common.py

@@ -17,6 +17,8 @@
 if TYPE_CHECKING:
     from io import BufferedIOBase
 
+    from xarray.core.dataset import Dataset
+
 # Create a logger object, but don't add any handlers. Leave that to user code.
 logger = logging.getLogger(__name__)
 
@@ -377,9 +379,6 @@ class BackendEntrypoint:
     Attributes
     ----------
 
-    available : bool, default: True
-        Indicate wether this backend is available given the installed packages.
-        The setting of this attribute is not mandatory.
     open_dataset_parameters : tuple, default: None
         A list of ``open_dataset`` method parameters.
         The setting of this attribute is not mandatory.
@@ -391,8 +390,6 @@ class BackendEntrypoint:
         The setting of this attribute is not mandatory.
     """
 
-    available: ClassVar[bool] = True
-
     open_dataset_parameters: ClassVar[tuple | None] = None
     description: ClassVar[str] = ""
     url: ClassVar[str] = ""
@@ -408,9 +405,10 @@ def __repr__(self) -> str:
     def open_dataset(
         self,
         filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        *,
         drop_variables: str | Iterable[str] | None = None,
         **kwargs: Any,
-    ):
+    ) -> Dataset:
         """
         Backend open_dataset method used by Xarray in :py:func:`~xarray.open_dataset`.
         """
@@ -420,12 +418,13 @@ def open_dataset(
     def guess_can_open(
         self,
         filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
-    ):
+    ) -> bool:
         """
         Backend open_dataset method used by Xarray in :py:func:`~xarray.open_dataset`.
         """
 
         return False
 
 
-BACKEND_ENTRYPOINTS: dict[str, type[BackendEntrypoint]] = {}
+# mapping of engine name to (module name, BackendEntrypoint Class)
+BACKEND_ENTRYPOINTS: dict[str, tuple[str | None, type[BackendEntrypoint]]] = {}

xarray/backends/h5netcdf_.py

@@ -3,6 +3,8 @@
 import functools
 import io
 import os
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any
 
 from packaging.version import Version
 
@@ -27,12 +29,17 @@
 from xarray.core.utils import (
     FrozenDict,
     is_remote_uri,
-    module_available,
     read_magic_number_from_file,
     try_read_magic_number_from_file_or_path,
 )
 from xarray.core.variable import Variable
 
+if TYPE_CHECKING:
+    from io import BufferedIOBase
+
+    from xarray.backends.common import AbstractDataStore
+    from xarray.core.dataset import Dataset
+
 
 class H5NetCDFArrayWrapper(BaseNetCDF4Array):
     def get_array(self, needs_lock=True):
@@ -365,33 +372,34 @@ class H5netcdfBackendEntrypoint(BackendEntrypoint):
     backends.ScipyBackendEntrypoint
     """
 
-    available = module_available("h5netcdf")
     description = (
         "Open netCDF (.nc, .nc4 and .cdf) and most HDF5 files using h5netcdf in Xarray"
     )
     url = "https://docs.xarray.dev/en/stable/generated/xarray.backends.H5netcdfBackendEntrypoint.html"
 
-    def guess_can_open(self, filename_or_obj):
+    def guess_can_open(
+        self,
+        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+    ) -> bool:
         magic_number = try_read_magic_number_from_file_or_path(filename_or_obj)
         if magic_number is not None:
             return magic_number.startswith(b"\211HDF\r\n\032\n")
 
-        try:
+        if isinstance(filename_or_obj, (str, os.PathLike)):
             _, ext = os.path.splitext(filename_or_obj)
-        except TypeError:
-            return False
+            return ext in {".nc", ".nc4", ".cdf"}
 
-        return ext in {".nc", ".nc4", ".cdf"}
+        return False
 
-    def open_dataset(
+    def open_dataset(  # type: ignore[override]  # allow LSP violation, not supporting **kwargs
         self,
-        filename_or_obj,
+        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
         *,
         mask_and_scale=True,
         decode_times=True,
         concat_characters=True,
         decode_coords=True,
-        drop_variables=None,
+        drop_variables: str | Iterable[str] | None = None,
         use_cftime=None,
         decode_timedelta=None,
         format=None,
@@ -400,7 +408,7 @@ def open_dataset(
         invalid_netcdf=None,
         phony_dims=None,
         decode_vlen_strings=True,
-    ):
+    ) -> Dataset:
         filename_or_obj = _normalize_path(filename_or_obj)
         store = H5NetCDFStore.open(
             filename_or_obj,
@@ -427,4 +435,4 @@ def open_dataset(
         return ds
 
 
-BACKEND_ENTRYPOINTS["h5netcdf"] = H5netcdfBackendEntrypoint
+BACKEND_ENTRYPOINTS["h5netcdf"] = ("h5netcdf", H5netcdfBackendEntrypoint)

xarray/backends/netCDF4_.py

@@ -3,7 +3,9 @@
 import functools
 import operator
 import os
+from collections.abc import Iterable
 from contextlib import suppress
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
@@ -33,11 +35,16 @@
     FrozenDict,
     close_on_error,
     is_remote_uri,
-    module_available,
     try_read_magic_number_from_path,
 )
 from xarray.core.variable import Variable
 
+if TYPE_CHECKING:
+    from io import BufferedIOBase
+
+    from xarray.backends.common import AbstractDataStore
+    from xarray.core.dataset import Dataset
+
 # This lookup table maps from dtype.byteorder to a readable endian
 # string used by netCDF4.
 _endian_lookup = {"=": "native", ">": "big", "<": "little", "|": "native"}
@@ -535,33 +542,37 @@ class NetCDF4BackendEntrypoint(BackendEntrypoint):
     backends.ScipyBackendEntrypoint
     """
 
-    available = module_available("netCDF4")
     description = (
         "Open netCDF (.nc, .nc4 and .cdf) and most HDF5 files using netCDF4 in Xarray"
     )
     url = "https://docs.xarray.dev/en/stable/generated/xarray.backends.NetCDF4BackendEntrypoint.html"
 
-    def guess_can_open(self, filename_or_obj):
+    def guess_can_open(
+        self,
+        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+    ) -> bool:
         if isinstance(filename_or_obj, str) and is_remote_uri(filename_or_obj):
             return True
         magic_number = try_read_magic_number_from_path(filename_or_obj)
         if magic_number is not None:
             # netcdf 3 or HDF5
             return magic_number.startswith((b"CDF", b"\211HDF\r\n\032\n"))
-        try:
+
+        if isinstance(filename_or_obj, (str, os.PathLike)):
             _, ext = os.path.splitext(filename_or_obj)
-        except TypeError:
-            return False
-        return ext in {".nc", ".nc4", ".cdf"}
+            return ext in {".nc", ".nc4", ".cdf"}
+
+        return False
 
-    def open_dataset(
+    def open_dataset(  # type: ignore[override]  # allow LSP violation, not supporting **kwargs
         self,
-        filename_or_obj,
+        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        *,
         mask_and_scale=True,
         decode_times=True,
         concat_characters=True,
         decode_coords=True,
-        drop_variables=None,
+        drop_variables: str | Iterable[str] | None = None,
         use_cftime=None,
         decode_timedelta=None,
         group=None,
@@ -572,7 +583,7 @@ def open_dataset(
         persist=False,
         lock=None,
         autoclose=False,
-    ):
+    ) -> Dataset:
         filename_or_obj = _normalize_path(filename_or_obj)
         store = NetCDF4DataStore.open(
             filename_or_obj,
@@ -601,4 +612,4 @@ def open_dataset(
         return ds
 
 
-BACKEND_ENTRYPOINTS["netcdf4"] = NetCDF4BackendEntrypoint
+BACKEND_ENTRYPOINTS["netcdf4"] = ("netCDF4", NetCDF4BackendEntrypoint)