Merge branch 'main' into cfa-write

Author: davidhassell

Changelog.rst

@@ -9,8 +9,12 @@ version NEXT
   to regrid the vertical axis in logarithmic coordinates to
   `cf.Field.regrids` and `cf.Field.regridc`
   (https://github.com/NCAS-CMS/cf-python/issues/715)
+* Improve the performance of reading and accessing the data of PP and
+  UM fields files (https://github.com/NCAS-CMS/cf-python/issues/746)
 * Improve `cf.Field.collapse` performance by lazily computing reduced
   axis coordinates (https://github.com/NCAS-CMS/cf-python/issues/741)
+* Improve `cf.Field.__getitem__` performance by not re-calculating
+  axis cyclicity (https://github.com/NCAS-CMS/cf-python/issues/744)
 * Reduce output CFA netCDF file size by setting the HDF5 chunksizes of
   CFA variables to be no larger than required
   (https://github.com/NCAS-CMS/cf-python/issues/739)

DOCUMENTATION.md

@@ -1,12 +1,14 @@
 * The generation of recipes using Sphinx-Gallery requires:
 
   ```txt
-  sphinx-gallery==0.11.0
-  sphinx-copybutton==0.5.1
-  sphinx-toggleprompt==0.2.0
-  sphinxcontrib-spelling==4.3.0
-  sphinxcontrib.serializinghtml==1.1.5
-  sphinx==2.4.5
+  pip install sphinx-gallery==0.11.0
+  pip install sphinx-copybutton==0.5.1
+  pip install sphinx-toggleprompt==0.2.0
+  pip install sphinxcontrib-spelling==4.3.0
+  pip install sphinxcontrib.serializinghtml==1.1.5
+  pip install sphinxcontrib.htmlhelp==1.0.3
+  pip install sphinx==2.4.5
+  ```
 
 * The `.py` files to generate recipes are stored in `docs/source/recipes/`.
 

README.md

@@ -107,9 +107,10 @@ of its array manipulation and can:
   including those with geometry cells and UGRID mesh topologies,
 * perform histogram, percentile and binning operations on field
   constructs,
-* regrid field constructs with (multi-)linear, nearest neighbour,
-  first- and second-order conservative and higher order patch recovery
-  methods, to and from structured and unstructured grids,
+* regrid structured grid, mesh and DSG field constructs with
+  (multi-)linear, nearest neighbour, first- and second-order
+  conservative and higher order patch recovery methods, including 3-d
+  regridding,
 * apply convolution filters to field constructs,
 * create running means from field constructs,
 * apply differential operators to field constructs,

RELEASE.md

@@ -17,6 +17,9 @@
 * Make sure that `Changelog.rst` is up to date (version, date and
   changes).
 
+* Search for occurences of `NEXTRELEASE` and replace them with the
+  release version (i.e. the value of the now correct `cf.__version__`)
+
 * Deprecated methods and keyword arguments: Check the code for
   deprecated methods and keyword arguments that can be completely
   removed, i.e. those with a ``removed_at`` version that is at or

cf/data/array/umarray.py

@@ -8,7 +8,7 @@
     load_stash2standard_name,
     parse_indices,
 )
-from ...umread_lib.umfile import File
+from ...umread_lib.umfile import File, Rec
 from .abstract import Array
 from .mixin import FileArrayMixin
 
@@ -272,13 +272,22 @@ def _get_rec(self, f, header_offset):
                 The record container.
 
         """
-        # TODOCFA: This method doesn't require data_offset and disk_length,
-        # so plays nicely with CFA. Is it fast enough that we can
-        # use this method always?
-        for v in f.vars:
-            for r in v.recs:
-                if r.hdr_offset == header_offset:
-                    return r
+        return Rec.from_file_and_offsets(f, header_offset)
+
+        # ------------------------------------------------------------
+        # Leave the following commented code here for debugging
+        # purposes. Replacing the above line with this code moves the
+        # calculation of the data offset and disk length from pure
+        # Python to the C library, at the expense of completely
+        # parsing the file. Note: If you do replace the above line
+        # with the commented code, then you *must* also set
+        # 'parse=True' in the `open` method.
+        # ------------------------------------------------------------
+
+        # for v in f.vars:
+        #     for r in v.recs:
+        #         if r.hdr_offset == header_offset:
+        #             return r
 
     def _set_units(self, int_hdr):
         """The units and calendar properties.
@@ -666,21 +675,24 @@ def get_word_size(self):
         return self._get_component("word_size", None)
 
     def open(self):
-        """Returns an open dataset containing the data array.
+        """Returns an open dataset and the address of the data.
 
         :Returns:
 
-            `umfile_lib.File`, `int`
+            `umfile_lib.umfile.File`, `int`
+                The open file object, and the start address in bytes
+                of the lookup header.
 
         **Examples**
 
         >>> f.open()
-        (<cf.umread_lib.umfile.File object at 0x7fdc25056380>, 44567)
+        (<cf.umread_lib.umfile.File object at 0x7fdc25056380>, 4)
 
         """
         return super().open(
             File,
             byte_ordering=self.get_byte_ordering(),
             word_size=self.get_word_size(),
             fmt=self.get_fmt(),
+            parse=False,
         )

cf/docstring/docstring.py

@@ -621,30 +621,33 @@
                 True, or a tuple of both if *item* is True.""",
     # regrid RegridOperator
     "{{regrid RegridOperator}}": """* `RegridOperator`: The grid is defined by a regrid
-                operator that has been returned by a previous call
-                with the *return_operator* parameter set to True.
-
-                Unlike the other options, for which the regrid weights
-                need to be calculated, the regrid operator already
-                contains the weights. Therefore, for cases where
-                multiple fields with the same source grids need to be
-                regridded to the same destination grid, using a regrid
-                operator can give performance improvements by avoiding
-                having to calculate the weights for each source
-                field. Note that for the other types of *dst*
-                parameter, the calculation of the regrid weights is
-                not a lazy operation.
-
-                .. note:: The source grid of the regrid operator is
-                          immediately checked for compatibility with
-                          the grid of the source field. By default
-                          only the computationally cheap tests are
-                          performed (checking that the coordinate
-                          system, cyclicity and grid shape are the
-                          same), with the grid coordinates not being
-                          checked. The coordinates check will be
-                          carried out, however, if the
-                          *check_coordinates* parameter is True.""",
+                  operator that has been returned by a previous call
+                  with the *return_operator* parameter set to True.
+
+                  Unlike the other options, for which the regrid
+                  weights need to be calculated, the regrid operator
+                  already contains the weights. Therefore, for cases
+                  where multiple fields with the same source grids
+                  need to be regridded to the same destination grid,
+                  using a regrid operator can give performance
+                  improvements by avoiding having to calculate the
+                  weights for each source field. Note that for the
+                  other types of *dst* parameter, the calculation of
+                  the regrid weights is not a lazy operation.
+
+                   .. note:: When *dst* is a `RegridOperator`, the
+                            source grid of the regrid operator is
+                            immediately checked for compatibility with
+                            the grid of the source field. By default
+                            only the computationally cheap tests are
+                            performed (checking that the coordinate
+                            system, cyclicity, grid shape, regridding
+                            dimesionality, mesh location, and feature
+                            type are the same), with the grid
+                            coordinates not being checked. The
+                            coordinates check will be carried out,
+                            however, if the *check_coordinates*
+                            parameter is True.""",
     # Returns cfa_file_substitutions
     "{{Returns cfa_file_substitutions}}": """The CFA-netCDF file name substitutions in a dictionary
                 whose key/value pairs are the file name parts to be

cf/field.py

@@ -441,6 +441,25 @@ def __getitem__(self, indices):
         for axis, size in zip(data_axes, new_data.shape):
             domain_axes[axis].set_size(size)
 
+        # Record which axes were cyclic before the subspace
+        org_cyclic = [data_axes.index(axis) for axis in new.cyclic()]
+
+        # Set the subspaced data
+        new.set_data(new_data, axes=data_axes, copy=False)
+
+        # Update axis cylcicity. Note that this can only entail
+        # setting an originally cyclic axis to be non-cyclic. Doing
+        # this now enables us to disable the (possibly very slow)
+        # automatic check for cyclicity on the 'set_construct' calls
+        # below.
+        if org_cyclic:
+            new_cyclic = new_data.cyclic()
+            [
+                new.cyclic(i, iscyclic=False)
+                for i in org_cyclic
+                if i not in new_cyclic
+            ]
+
         # ------------------------------------------------------------
         # Subspace constructs with data
         # ------------------------------------------------------------
@@ -507,6 +526,7 @@ def __getitem__(self, indices):
                         key=key,
                         axes=construct_axes,
                         copy=False,
+                        autocyclic={"no-op": True},
                     )
 
         new.set_data(new_data, axes=data_axes, copy=False)
@@ -13701,7 +13721,7 @@ def regrids(
 
                 Ignored if *dst* is a `RegridOperator`.
 
-                .. versionadded:: 3.17.0
+                .. versionadded:: NEXTRELEASE
 
             dst_z: optional
                 If `None`, the default, then the regridding is 2-d in
@@ -13715,7 +13735,7 @@ def regrids(
 
                 Ignored if *dst* is a `RegridOperator`.
 
-                .. versionadded:: 3.17.0
+                .. versionadded:: NEXTRELEASE
 
             z: optional
                 The *z* parameter is a convenience that may be used to
@@ -13729,11 +13749,11 @@ def regrids(
                 *Example:*
                   ``z='Z'`` is equivalent to ``src_z='Z', dst_z='Z'``.
 
-                .. versionadded:: 3.17.0
+                .. versionadded:: NEXTRELEASE
 
             {{ln_z: `bool` or `None`, optional}}
 
-                .. versionadded:: 3.17.0
+                .. versionadded:: NEXTRELEASE
 
             {{verbose: `int` or `str` or `None`, optional}}
 
@@ -14000,7 +14020,7 @@ def regridc(
 
                 Ignored if *dst* is a `RegridOperator`.
 
-                .. versionadded:: 3.17.0
+                .. versionadded:: NEXTRELEASE
 
             dst_z: optional
                 If not `None` then *dst_z* specifies the identity of a
@@ -14010,7 +14030,7 @@ def regridc(
 
                 Ignored if *dst* is a `RegridOperator`.
 
-                .. versionadded:: 3.17.0
+                .. versionadded:: NEXTRELEASE
 
             z: optional
                 The *z* parameter is a convenience that may be used to
@@ -14022,11 +14042,11 @@ def regridc(
                 *Example:*
                   ``z='Z'`` is equivalent to ``src_z='Z', dst_z='Z'``.
 
-                .. versionadded:: 3.17.0
+                .. versionadded:: NEXTRELEASE
 
             {{ln_z: `bool` or `None`, optional}}
 
-                .. versionadded:: 3.17.0
+                .. versionadded:: NEXTRELEASE
 
             {{inplace: `bool`, optional}}
 

cf/mixin/fielddomain.py

@@ -1109,10 +1109,15 @@ def autocyclic(self, key=None, coord=None, verbose=None, config={}):
 
         :Returns:
 
-           `bool`
+           `bool` or `None`
+               `True` if the dimension is cyclic, `False` if it isn't,
+               or `None` if no checks were done.
 
         """
         noop = config.get("no-op")
+        if noop:
+            # Don't do anything
+            return
 
         if "cyclic" in config:
             if not config["cyclic"]: