Merge pull request #805 from davidhassell/active-storage-new

Optimsed lazy indexing - h5netcdf backend - Active storage reductions

Author: davidhassell

Changelog.rst

@@ -1,3 +1,22 @@
+version NEXTVERSION + 1
+-----------------------
+
+**2024-??-??**
+
+* Allow access to netCDF-4 files in S3 object stores
+  (https://github.com/NCAS-CMS/cf-python/issues/712)
+* New class `cf.H5netcdfArray`
+* New class `cf.NetCDF4Array`
+* New class `cf.CFAH5netcdfArray`
+* New class `cf.CFANetCDF4Array`
+* New dependency: ``h5netcdf>=1.3.0``
+* New dependency: ``h5py>=3.10.0``
+* New dependency: ``s3fs>=2024.2.0``
+* Changed dependency: ``1.11.2.0<=cfdm<1.11.3.0``
+* Changed dependency: ``cfunits>=3.3.7``
+
+----
+
 version NEXTVERSION
 -------------------
 
@@ -141,6 +160,8 @@ version 3.16.0
 * Changed dependency: ``1.11.0.0<=cfdm<1.11.1.0``
 * New dependency: ``scipy>=1.10.0``
 
+----
+
 version 3.15.4
 --------------
 
@@ -279,7 +300,7 @@ version 3.14.1
 
 ----
 
-version 3.14.0 (*first Dask release*)
+version 3.14.0 (*first Dask version*)
 -------------------------------------
 
 **2023-01-31**
@@ -314,7 +335,7 @@ version 3.14.0 (*first Dask release*)
 
 ----
 
-version 3.13.1 (*last LAMA release*)
+version 3.13.1 (*last LAMA version*)
 ------------------------------------
 
 **2022-10-17**

cf/__init__.py

@@ -13,10 +13,17 @@
 
 * read field constructs from netCDF, CDL, PP and UM datasets,
 
+* read field constructs and domain constructs from netCDF, CDL, PP and
+  UM datasets with a choice of netCDF backends,
+
+* read files from OPeNDAP servers and S3 object stores,
+
 * create new field constructs in memory,
 
 * write and append field constructs to netCDF datasets on disk,
 
+* read, write, and manipulate UGRID mesh topologies,
+
 * read, write, and create coordinates defined by geometry cells,
 
 * read netCDF and CDL datasets containing hierarchical groups,
@@ -74,8 +81,8 @@
 """
 
 __Conventions__ = "CF-1.11"
-__date__ = "2024-04-26"
-__version__ = "3.16.2"
+__date__ = "2024-??-??"
+__version__ = "3.17.0"
 
 _requires = (
     "numpy",
@@ -199,8 +206,8 @@
     )
 
 # Check the version of cfdm
-_minimum_vn = "1.11.1.0"
-_maximum_vn = "1.11.2.0"
+_minimum_vn = "1.11.2.0"
+_maximum_vn = "1.11.3.0"
 _cfdm_version = Version(cfdm.__version__)
 if not Version(_minimum_vn) <= _cfdm_version < Version(_maximum_vn):
     raise RuntimeError(
@@ -209,12 +216,6 @@
     )
 
 # Check the version of dask
-_minimum_vn = "2022.12.1"
-if Version(dask.__version__) < Version(_minimum_vn):
-    raise RuntimeError(
-        f"Bad dask version: cf requires dask>={_minimum_vn}. "
-        f"Got {dask.__version__} at {dask.__file__}"
-    )
 
 # Check the version of Python
 _minimum_vn = "3.8.0"
@@ -274,15 +275,19 @@
 from .data.array import (
     BoundsFromNodesArray,
     CellConnectivityArray,
-    CFANetCDFArray,
+    CFAH5netcdfArray,
+    CFANetCDF4Array,
     FullArray,
     GatheredArray,
+    H5netcdfArray,
     NetCDFArray,
+    NetCDF4Array,
     PointTopologyArray,
     RaggedContiguousArray,
     RaggedIndexedArray,
     RaggedIndexedContiguousArray,
     SubsampledArray,
+    UMArray,
 )
 
 from .data.fragment import (

cf/cellmethod.py

@@ -56,7 +56,7 @@ class CellMethod(cfdm.CellMethod):
     def __new__(cls, *args, **kwargs):
         """This must be overridden in subclasses.
 
-        .. versionadded:: (cfdm) 3.7.0
+        .. versionadded:: 3.7.0
 
         """
         instance = super().__new__(cls)

cf/cfimplementation.py

@@ -26,12 +26,16 @@
     TiePointIndex,
 )
 from .data import Data
+
+# REVIEW: h5: `cfimplementation.py`: import `CFAH5netcdfArray`, `CFANetCDF4Array`, `H5netcdfArray`,`NetCDF4Array`
 from .data.array import (
     BoundsFromNodesArray,
     CellConnectivityArray,
-    CFANetCDFArray,
+    CFAH5netcdfArray,
+    CFANetCDF4Array,
     GatheredArray,
-    NetCDFArray,
+    H5netcdfArray,
+    NetCDF4Array,
     PointTopologyArray,
     RaggedContiguousArray,
     RaggedIndexedArray,
@@ -112,65 +116,41 @@ def set_construct(self, parent, construct, axes=None, copy=True, **kwargs):
             parent, construct, axes=axes, copy=copy, **kwargs
         )
 
-    def initialise_CFANetCDFArray(
-        self,
-        filename=None,
-        address=None,
-        dtype=None,
-        mask=True,
-        units=False,
-        calendar=False,
-        instructions=None,
-        substitutions=None,
-        term=None,
-        x=None,
-        **kwargs,
-    ):
-        """Return a `CFANetCDFArray` instance.
+    # REVIEW: h5: `initialise_CFANetCDF4Array`: new method to initialise `CFANetCDF4Array`
+    def initialise_CFANetCDF4Array(self, **kwargs):
+        """Return a `CFANetCDF4Array` instance.
 
         :Parameters:
 
-            filename: `str`
-
-            address: (sequence of) `str` or `int`
-
-            dytpe: `numpy.dtype`
-
-            mask: `bool`, optional
+            kwargs: optional
+                Initialisation parameters to pass to the new instance.
 
-            units: `str` or `None`, optional
+        :Returns:
 
-            calendar: `str` or `None`, optional
+            `CFANetCDF4Array`
 
-            instructions: `str`, optional
+        """
+        cls = self.get_class("CFANetCDF4Array")
+        return cls(**kwargs)
 
-            substitutions: `dict`, optional
+    # REVIEW: h5: `initialise_CFAH5netcdfArray`: new method to initialise `CFAH5netcdfArray`
+    def initialise_CFAH5netcdfArray(self, **kwargs):
+        """Return a `CFAH5netcdfArray` instance.
 
-            term: `str`, optional
+        .. versionadded:: NEXTVERSION
 
-            x: `dict`, optional
+        :Parameters:
 
             kwargs: optional
-                Ignored.
+                Initialisation parameters to pass to the new instance.
 
         :Returns:
 
-            `CFANetCDFArray`
+            `CFAH5netcdfArray`
 
         """
-        cls = self.get_class("CFANetCDFArray")
-        return cls(
-            filename=filename,
-            address=address,
-            dtype=dtype,
-            mask=mask,
-            units=units,
-            calendar=calendar,
-            instructions=instructions,
-            substitutions=substitutions,
-            term=term,
-            x=x,
-        )
+        cls = self.get_class("CFAH5netcdfArray")
+        return cls(**kwargs)
 
 
 _implementation = CFImplementation(
@@ -179,7 +159,8 @@ def initialise_CFANetCDFArray(
     CellConnectivity=CellConnectivity,
     CellMeasure=CellMeasure,
     CellMethod=CellMethod,
-    CFANetCDFArray=CFANetCDFArray,
+    CFAH5netcdfArray=CFAH5netcdfArray,
+    CFANetCDF4Array=CFANetCDF4Array,
     CoordinateReference=CoordinateReference,
     DimensionCoordinate=DimensionCoordinate,
     Domain=Domain,
@@ -202,7 +183,8 @@ def initialise_CFANetCDFArray(
     BoundsFromNodesArray=BoundsFromNodesArray,
     CellConnectivityArray=CellConnectivityArray,
     GatheredArray=GatheredArray,
-    NetCDFArray=NetCDFArray,
+    H5netcdfArray=H5netcdfArray,
+    NetCDF4Array=NetCDF4Array,
     PointTopologyArray=PointTopologyArray,
     RaggedContiguousArray=RaggedContiguousArray,
     RaggedIndexedArray=RaggedIndexedArray,
@@ -236,7 +218,8 @@ def implementation():
      'CellConnectivityArray': cf.data.array.cellconnectivityarray.CellConnectivityArray,
      'CellMeasure': cf.cellmeasure.CellMeasure,
      'CellMethod': cf.cellmethod.CellMethod,
-     'CFANetCDFArray': cf.data.array.cfanetcdfarray.CFANetCDFArray,
+     'CFAH5netcdfArray': cf.data.array.cfah5netcdfarray.CFAH5netcdfArray,
+     'CFANetCDF4Array': cf.data.array.cfanetcdf4array.CFANetCDF4Array,
      'CoordinateReference': cf.coordinatereference.CoordinateReference,
      'DimensionCoordinate': cf.dimensioncoordinate.DimensionCoordinate,
      'Domain': cf.domain.Domain,
@@ -257,7 +240,8 @@ def implementation():
      'PartNodeCountProperties': cf.partnodecountproperties.PartNodeCountProperties,
      'Data': cf.data.data.Data,
      'GatheredArray': cf.data.array.gatheredarray.GatheredArray,
-     'NetCDFArray': cf.data.array.netcdfarray.NetCDFArray,
+     'H5netcdfArray': cf.data.array.h5netcdfarray.H5netcdfArray,
+     'NetCDF4Array': cf.data.array.netcdf4array.NetCDF4Array,
      'PointTopologyArray': <class 'cf.data.array.pointtopologyarray.PointTopologyArray'>,
      'RaggedContiguousArray': cf.data.array.raggedcontiguousarray.RaggedContiguousArray,
      'RaggedIndexedArray': cf.data.array.raggedindexedarray.RaggedIndexedArray,

cf/constants.py

@@ -63,6 +63,10 @@
     "LOG_LEVEL": logging.getLevelName(logging.getLogger().level),
     "BOUNDS_COMBINATION_MODE": "AND",
     "CHUNKSIZE": parse_bytes(_CHUNKSIZE),
+    # REVIEW: active: `CONSTANTS`: new constants 'active_storage', 'active_storage_url', 'active_storage_max_requests'
+    "active_storage": False,
+    "active_storage_url": None,
+    "active_storage_max_requests": 100,
 }
 
 masked = np.ma.masked

cf/data/array/__init__.py

@@ -1,9 +1,20 @@
 from .boundsfromnodesarray import BoundsFromNodesArray
 from .cellconnectivityarray import CellConnectivityArray
-from .cfanetcdfarray import CFANetCDFArray
+
+# REVIEW: h5: `__init__.py`: import `CFAH5netcdfArray`
+from .cfah5netcdfarray import CFAH5netcdfArray
+
+# REVIEW: h5: `__init__.py`: import `CFAH5netcdfArray`
+from .cfanetcdf4array import CFANetCDF4Array
 from .fullarray import FullArray
 from .gatheredarray import GatheredArray
+
+# REVIEW: h5: `__init__.py`: import `H5netcdfArray`
+from .h5netcdfarray import H5netcdfArray
 from .netcdfarray import NetCDFArray
+
+# REVIEW: h5: `__init__.py`: import `NetCDF4Array`
+from .netcdf4array import NetCDF4Array
 from .pointtopologyarray import PointTopologyArray
 from .raggedcontiguousarray import RaggedContiguousArray
 from .raggedindexedarray import RaggedIndexedArray

cf/data/array/cfah5netcdfarray.py

@@ -0,0 +1,11 @@
+# REVIEW: h5: `CFAH5netcdfArray`: New class for accessing CFA with `h5netcdf`
+from .h5netcdfarray import H5netcdfArray
+from .mixin import CFAMixin
+
+
+class CFAH5netcdfArray(CFAMixin, H5netcdfArray):
+    """A CFA-netCDF array accessed with `h5netcdf`
+
+    .. versionadded:: NEXTVERSION
+
+    """

cf/data/array/cfanetcdf4array.py

@@ -0,0 +1,11 @@
+# REVIEW: h5: `CFAnetCDF4Array`: New class for accessing CFA with `netCDF4`
+from .mixin import CFAMixin
+from .netcdf4array import NetCDF4Array
+
+
+class CFANetCDF4Array(CFAMixin, NetCDF4Array):
+    """A CFA-netCDF array accessed with `netCDF4`.
+
+    .. versionadded:: NEXTVERSION
+
+    """