diff --git a/doc/api/esmvalcore.regridding_schemes.rst b/doc/api/esmvalcore.regridding_schemes.rst index 960b8c4b7d..c802c59e12 100644 --- a/doc/api/esmvalcore.regridding_schemes.rst +++ b/doc/api/esmvalcore.regridding_schemes.rst @@ -15,9 +15,9 @@ Example: .. code:: python - from esmvalcore.preprocessor.regrid_schemes import ESMPyAreaWeighted + from esmvalcore.preprocessor.regrid_schemes import IrisESMFRegrid - regridded_cube = cube.regrid(target_grid, ESMPyAreaWeighted()) + regridded_cube = cube.regrid(target_grid, IrisESMFRegrid(method="conservative")) .. automodule:: esmvalcore.preprocessor.regrid_schemes :no-show-inheritance: diff --git a/environment.yml b/environment.yml index 00a251a4ba..ae382ce831 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,6 @@ dependencies: - dask-jobqueue - distributed - esgf-pyclient >=0.3.1 - - esmpy - esmvaltool-sample-data - filelock - fiona diff --git a/esmvalcore/preprocessor/_mapping.py b/esmvalcore/preprocessor/_mapping.py deleted file mode 100644 index 4008752933..0000000000 --- a/esmvalcore/preprocessor/_mapping.py +++ /dev/null @@ -1,252 +0,0 @@ -"""Provides mapping of a cube.""" - -import collections - -import iris -import numpy as np - - -def _is_single_item(testee): - """ - Check if testee is a single item. - - Return whether this is a single item, rather than an iterable. - We count string types as 'single', also. - """ - return isinstance(testee, str) or not isinstance( - testee, - collections.abc.Iterable, - ) - - -def _as_list_of_coords(cube, names_or_coords): - """Convert a name, coord, or list of names/coords to a list of coords.""" - # If not iterable, convert to list of a single item - if _is_single_item(names_or_coords): - names_or_coords = [names_or_coords] - coords = [] - for name_or_coord in names_or_coords: - if isinstance(name_or_coord, (iris.coords.Coord, str)): - coords.append(cube.coord(name_or_coord)) - else: - # Don't know how to handle this type - msg = ( - f"Don't know how to handle coordinate of type {type(name_or_coord)}. " - "Ensure all coordinates are of type str " - "or iris.coords.Coord." - ) - raise TypeError(msg) - return coords - - -def ref_to_dims_index_as_coordinate(cube, ref): - """Get dims for coord ref.""" - coord = _as_list_of_coords(cube, ref)[0] - dims = cube.coord_dims(coord) - if not dims: - msg = ( - "Requested an iterator over a coordinate ({}) " - "which does not describe a dimension." - ) - msg = msg.format(coord.name()) - raise ValueError(msg) - return dims - - -def ref_to_dims_index_as_index(cube, ref): - """Get dim for index ref.""" - try: - dim = int(ref) - except (ValueError, TypeError) as exc: - msg = f"{ref} Incompatible type {type(ref)} for slicing" - raise ValueError( - msg, - ) from exc - if dim < 0 or dim > cube.ndim: - msg = ( - f"Requested an iterator over a dimension ({dim}) " - "which does not exist." - ) - raise ValueError(msg) - return [dim] - - -def ref_to_dims_index(cube, ref_to_slice): - """ - Map a list of :class:`iris.coords.DimCoord` to a tuple of indices. - - This method finds the indices of the dimensions in a cube that collectively - correspond to the given list of :class:`iris.coords.DimCoord`. - - Parameters - ---------- - cube: :class:`iris.cube.Cube` - The cube to examine. - ref_to_slice: iterable of or single :class:`iris.coords.DimCoord` - Specification of the dimensions in terms of coordinates. - - Returns - ------- - tuple: - A tuple of indices corresponding to the given dimensions. - """ - # Required to handle a mix between types - if _is_single_item(ref_to_slice): - ref_to_slice = [ref_to_slice] - dim_to_slice = [] - dim_to_slice_set = set() - for ref in ref_to_slice: - try: - dims = ref_to_dims_index_as_coordinate(cube, ref) - except TypeError: - dims = ref_to_dims_index_as_index(cube, ref) - for dim in dims: - if dim not in dim_to_slice_set: - dim_to_slice.append(dim) - dim_to_slice_set.add(dim) - return dim_to_slice - - -def get_associated_coords(cube, dimensions): - """ - Return all coords containing any of the given dimensions. - - Return all coords, dimensional and auxiliary, that contain any of - the given dimensions. - """ - dims = [] - dim_set = set() - for dim in dimensions: - if dim not in dim_set: - dims.append(dim) - dim_set.add(dim) - dim_coords = [] - for i in dims: - coords = cube.coords(contains_dimension=i, dim_coords=True) - if coords: - dim_coords.append(coords[0]) - aux_coords = [] - for i in dims: - coords = cube.coords(contains_dimension=i, dim_coords=False) - aux_coords.extend(coords) - return dim_coords, aux_coords - - -def get_empty_data(shape, dtype=np.float32): - """ - Create an empty data object of the given shape. - - Creates an empty data object of the given shape, potentially of the lazy - kind from biggus or dask, depending on the used iris version. - """ - data = np.empty(shape, dtype=dtype) - mask = np.empty(shape, dtype=bool) - return np.ma.masked_array(data, mask) - - -def get_slice_spec(cube, ref_to_slice): - """ - Turn a slice reference into a specification for the slice. - - Turns a slice reference into a specification comprised of the shape as well - as the relevant dimensional and auxiliary coordinates. - """ - slice_dims = ref_to_dims_index(cube, ref_to_slice) - slice_shape = tuple(cube.shape[d] for d in slice_dims) - dim_coords, aux_coords = get_associated_coords(cube, slice_dims) - return slice_shape, dim_coords, aux_coords - - -def index_iterator(dims_to_slice, shape): - """ - Return iterator for subsets of multidimensional objects. - - An iterator over a multidimensional object, giving both source and - destination indices. - """ - dst_slices = (slice(None, None),) * len(dims_to_slice) - dims = [1 if n in dims_to_slice else i for n, i in enumerate(shape)] - for index_tuple in np.ndindex(*dims): - src_ind = tuple( - slice(None, None) if n in dims_to_slice else i - for n, i in enumerate(index_tuple) - ) - dst_ind = ( - tuple( - i for n, i in enumerate(index_tuple) if n not in dims_to_slice - ) - + dst_slices - ) - yield src_ind, dst_ind - - -def get_slice_coords(cube): - """Return ordered set of unique coordinates.""" - slice_coords = [] - slice_set = set() - for i in range(cube.ndim): - coords = cube.coords(contains_dimension=i) - for coord in coords: - if coord not in slice_set: - slice_coords.append(coord) - slice_set.add(coord) - return slice_coords - - -def map_slices(src, func, src_rep, dst_rep): - """ - Map slices of a cube, replacing them with different slices. - - This method is similar to the standard cube collapsed and aggregated_by - methods, however, where they completely remove the mapped dimensions, this - method allows for their replacement with other dimensions. - The new dimensions are specified with a destination representant and will - be the last dimensions of the resulting cube, even if the removed - dimensions are can be any of the source cubes dimensions. - - Parameters - ---------- - src: :class:`iris.cube.Cube` - Source cube to be mapped. - func: callable - Callable that takes a single cube and returns a single numpy array. - src_rep: :class:`iris.cube.Cube` - Source representant that specifies the dimensions to be removed from - the source cube. - dst_rep: :class:`iris.cube.Cube` - Destination representant that specifies the shape of the new - dimensions. - - Returns - ------- - :class:`iris.cube.Cube`: - New cube that has the shape of the source cube with the removed - dimensions replaced with the destination dimensions. - All coordinates that span any of the removed dimensions are removed; - :class:`iris.coords.DimCoord` for the new dimensions are taken from - `dst_rep`. - """ - ref_to_slice = get_slice_coords(src_rep) - src_slice_dims = ref_to_dims_index(src, ref_to_slice) - src_keep_dims = list(set(range(src.ndim)) - set(src_slice_dims)) - src_keep_spec = get_slice_spec(src, src_keep_dims) - res_shape = src_keep_spec[0] + dst_rep.shape - dim_coords = src_keep_spec[1] + dst_rep.coords(dim_coords=True) - dim_coords_and_dims = [(c, i) for i, c in enumerate(dim_coords)] - aux_coords_and_dims = [(c, src.coord_dims(c)) for c in src_keep_spec[2]] - aux_coords_and_dims += [(c, src.coord_dims(c)) for c in dst_rep.aux_coords] - dst = iris.cube.Cube( - data=get_empty_data(res_shape, dtype=src.dtype), - standard_name=src.standard_name, - long_name=src.long_name, - var_name=src.var_name, - units=src.units, - attributes=src.attributes, - cell_methods=src.cell_methods, - dim_coords_and_dims=dim_coords_and_dims, - aux_coords_and_dims=aux_coords_and_dims, - ) - for src_ind, dst_ind in index_iterator(src_slice_dims, src.shape): - res = func(src[src_ind]) - dst.data[dst_ind] = res - return dst diff --git a/esmvalcore/preprocessor/_regrid_esmpy.py b/esmvalcore/preprocessor/_regrid_esmpy.py deleted file mode 100644 index 8adf003976..0000000000 --- a/esmvalcore/preprocessor/_regrid_esmpy.py +++ /dev/null @@ -1,538 +0,0 @@ -"""Provides regridding for irregular grids.""" - -try: - import esmpy -except ImportError as exc: - # Prior to v8.4.0, `esmpy`` could be imported as `ESMF`. - try: - import ESMF as esmpy # noqa: N811 - except ImportError: - raise exc from None -import warnings - -import iris -import numpy as np -from iris.cube import Cube - -from esmvalcore.exceptions import ESMValCoreDeprecationWarning - -from ._mapping import get_empty_data, map_slices, ref_to_dims_index - -ESMF_MANAGER = esmpy.Manager(debug=False) - -ESMF_LON, ESMF_LAT = 0, 1 - -ESMF_REGRID_METHODS = { - "linear": esmpy.RegridMethod.BILINEAR, - "area_weighted": esmpy.RegridMethod.CONSERVE, - "nearest": esmpy.RegridMethod.NEAREST_STOD, -} - -MASK_REGRIDDING_MASK_VALUE = { - esmpy.RegridMethod.BILINEAR: np.array([1]), - esmpy.RegridMethod.CONSERVE: np.array([1]), - esmpy.RegridMethod.NEAREST_STOD: np.array([]), -} - - -class ESMPyRegridder: - """General ESMPy regridder. - - Does not support lazy regridding nor weights caching. - - .. deprecated:: 2.12.0 - This regridder has been deprecated and is scheduled for removal in - version 2.14.0. Please use - :class:`~esmvalcore.preprocessor.regrid_schemes.IrisESMFRegrid` to - create an :doc:`esmf_regrid:index` regridder instead. - - Parameters - ---------- - src_cube: - Cube defining the source grid. - tgt_cube: - Cube defining the target grid. - method: - Regridding algorithm. Must be one of `linear`, `area_weighted`, - `nearest`. - mask_threshold: - Threshold used to regrid mask of input cube. - - """ - - def __init__( - self, - src_cube: Cube, - tgt_cube: Cube, - method: str = "linear", - mask_threshold: float = 0.99, - ): - """Initialize class instance.""" - # These regridders are not lazy, so load source and target data once. - src_cube.data # noqa: B018 pylint: disable=pointless-statement - tgt_cube.data # noqa: B018 pylint: disable=pointless-statement - self.src_cube = src_cube - self.tgt_cube = tgt_cube - self.method = method - self.mask_threshold = mask_threshold - - def __call__(self, cube: Cube) -> Cube: - """Perform regridding. - - Parameters - ---------- - cube: - Cube to be regridded. - - Returns - ------- - Cube - Regridded cube. - - """ - # These regridders are not lazy, so load source data once. - cube.data # noqa: B018 pylint: disable=pointless-statement - src_rep, dst_rep = get_grid_representants(cube, self.tgt_cube) - regridder = build_regridder( - src_rep, - dst_rep, - self.method, - mask_threshold=self.mask_threshold, - ) - return map_slices(cube, regridder, src_rep, dst_rep) - - -class _ESMPyScheme: - """General irregular regridding scheme. - - This class can be used in :meth:`iris.cube.Cube.regrid`. - - Note - ---- - See `ESMPy `__ for more details on - this. - - Parameters - ---------- - mask_threshold: - Threshold used to regrid mask of source cube. - - """ - - _METHOD = "" - - def __init__(self, mask_threshold: float = 0.99): - """Initialize class instance.""" - msg = ( - "The `esmvalcore.preprocessor.regrid_schemes." - f"{self.__class__.__name__}' regridding scheme has been " - "deprecated in ESMValCore version 2.12.0 and is scheduled for " - "removal in version 2.14.0. Please use " - "`esmvalcore.preprocessor.regrid_schemes.IrisESMFRegrid` " - "instead." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning, stacklevel=2) - self.mask_threshold = mask_threshold - - def __repr__(self) -> str: - """Return string representation of class.""" - return ( - f"{self.__class__.__name__}(mask_threshold={self.mask_threshold})" - ) - - def regridder(self, src_cube: Cube, tgt_cube: Cube) -> ESMPyRegridder: - """Get regridder. - - Parameters - ---------- - src_cube: - Cube defining the source grid. - tgt_cube: - Cube defining the target grid. - - Returns - ------- - ESMPyRegridder - Regridder instance. - - """ - return ESMPyRegridder( - src_cube, - tgt_cube, - method=self._METHOD, - mask_threshold=self.mask_threshold, - ) - - -class ESMPyAreaWeighted(_ESMPyScheme): - """ESMPy area-weighted regridding scheme. - - This class can be used in :meth:`iris.cube.Cube.regrid`. - - Does not support lazy regridding. - - .. deprecated:: 2.12.0 - This scheme has been deprecated and is scheduled for removal in version - 2.14.0. Please use - :class:`~esmvalcore.preprocessor.regrid_schemes.IrisESMFRegrid` - instead. - - """ - - _METHOD = "area_weighted" - - -class ESMPyLinear(_ESMPyScheme): - """ESMPy bilinear regridding scheme. - - This class can be used in :meth:`iris.cube.Cube.regrid`. - - Does not support lazy regridding. - - .. deprecated:: 2.12.0 - This scheme has been deprecated and is scheduled for removal in version - 2.14.0. Please use - :class:`~esmvalcore.preprocessor.regrid_schemes.IrisESMFRegrid` - instead. - - """ - - _METHOD = "linear" - - -class ESMPyNearest(_ESMPyScheme): - """ESMPy nearest-neighbor regridding scheme. - - This class can be used in :meth:`iris.cube.Cube.regrid`. - - Does not support lazy regridding. - - .. deprecated:: 2.12.0 - This scheme has been deprecated and is scheduled for removal in version - 2.14.0. Please use - :class:`~esmvalcore.preprocessor.regrid_schemes.IrisESMFRegrid` - instead. - - """ - - _METHOD = "nearest" - - -def cf_2d_bounds_to_esmpy_corners(bounds, circular): - """Convert cf style 2d bounds to normal (esmpy style) corners.""" - no_lat_points, no_lon_points = bounds.shape[:2] - no_lat_bounds = no_lat_points + 1 - no_lon_bounds = no_lon_points if circular else no_lon_points + 1 - esmpy_corners = np.empty((no_lon_bounds, no_lat_bounds)) - esmpy_corners[:no_lon_points, :no_lat_points] = bounds[:, :, 0].T - esmpy_corners[:no_lon_points, no_lat_points:] = bounds[-1:, :, 3].T - esmpy_corners[no_lon_points:, :no_lat_points] = bounds[:, -1:, 1].T - esmpy_corners[no_lon_points:, no_lat_points:] = bounds[-1:, -1:, 2].T - return esmpy_corners - - -def coords_iris_to_esmpy(lat, lon, circular): - """Build ESMF compatible coordinate information from iris coords.""" - dim = lat.ndim - if lon.ndim != dim: - msg = "Different dimensions in latitude({}) and longitude({}) coords." - raise ValueError(msg.format(lat.ndim, lon.ndim)) - if dim == 1: - for coord in [lat, lon]: - if not coord.has_bounds(): - coord.guess_bounds() - esmpy_lat, esmpy_lon = np.meshgrid(lat.points, lon.points) - lat_corners = np.concatenate([lat.bounds[:, 0], lat.bounds[-1:, 1]]) - if circular: - lon_corners = lon.bounds[:, 0] - else: - lon_corners = np.concatenate( - [lon.bounds[:, 0], lon.bounds[-1:, 1]], - ) - esmpy_lat_corners, esmpy_lon_corners = np.meshgrid( - lat_corners, - lon_corners, - ) - elif dim == 2: - esmpy_lat, esmpy_lon = lat.points.T.copy(), lon.points.T.copy() - esmpy_lat_corners = cf_2d_bounds_to_esmpy_corners(lat.bounds, circular) - esmpy_lon_corners = cf_2d_bounds_to_esmpy_corners(lon.bounds, circular) - else: - msg = f"Coord dimension is {dim}. Expected 1 or 2." - raise NotImplementedError( - msg, - ) - return esmpy_lat, esmpy_lon, esmpy_lat_corners, esmpy_lon_corners - - -def get_grid( - esmpy_lat, - esmpy_lon, - esmpy_lat_corners, - esmpy_lon_corners, - circular, -): - """Build EMSF grid from given coordinate information.""" - num_peri_dims = 1 if circular else 0 - - grid = esmpy.Grid( - np.vstack(esmpy_lat.shape), - num_peri_dims=num_peri_dims, - staggerloc=[esmpy.StaggerLoc.CENTER], - ) - grid.get_coords(ESMF_LON)[...] = esmpy_lon - grid.get_coords(ESMF_LAT)[...] = esmpy_lat - grid.add_coords([esmpy.StaggerLoc.CORNER]) - grid_lon_corners = grid.get_coords( - ESMF_LON, - staggerloc=esmpy.StaggerLoc.CORNER, - ) - grid_lat_corners = grid.get_coords( - ESMF_LAT, - staggerloc=esmpy.StaggerLoc.CORNER, - ) - grid_lon_corners[...] = esmpy_lon_corners - grid_lat_corners[...] = esmpy_lat_corners - grid.add_item(esmpy.GridItem.MASK, esmpy.StaggerLoc.CENTER) - return grid - - -def is_lon_circular(lon): - """Determine if longitudes are circular.""" - if isinstance(lon, iris.coords.DimCoord): - circular = lon.circular - elif isinstance(lon, iris.coords.AuxCoord): - if lon.ndim == 1: - seam = lon.bounds[-1, 1] - lon.bounds[0, 0] - elif lon.ndim == 2: - seam = lon.bounds[1:-1, -1, (1, 2)] - lon.bounds[1:-1, 0, (0, 3)] - else: - msg = ( - "AuxCoord longitude is higher dimensional than 2d. Giving up." - ) - raise NotImplementedError( - msg, - ) - circular = np.all(abs(seam) % 360.0 < 1.0e-3) - else: - msg = "longitude is neither DimCoord nor AuxCoord. Giving up." - raise TypeError(msg) - return circular - - -def cube_to_empty_field(cube): - """Build an empty ESMF field from a cube.""" - lat = cube.coord("latitude") - lon = cube.coord("longitude") - circular = is_lon_circular(lon) - esmpy_coords = coords_iris_to_esmpy(lat, lon, circular) - grid = get_grid(*esmpy_coords, circular=circular) - return esmpy.Field( - grid, - name=cube.long_name, - staggerloc=esmpy.StaggerLoc.CENTER, - ) - - -def get_representant(cube, ref_to_slice): - """Get a representative slice from a cube.""" - slice_dims = ref_to_dims_index(cube, ref_to_slice) - rep_ind = [0] * cube.ndim - for dim in slice_dims: - rep_ind[dim] = slice(None, None) - rep_ind = tuple(rep_ind) - return cube[rep_ind] - - -def regrid_mask_2d(src_data, regridding_arguments, mask_threshold): - """Regrid the mask from the source field to the destination grid.""" - src_field = regridding_arguments["srcfield"] - dst_field = regridding_arguments["dstfield"] - regrid_method = regridding_arguments["regrid_method"] - original_src_mask = np.ma.getmaskarray(src_data) - src_field.data[...] = ~original_src_mask.T - src_mask = src_field.grid.get_item( - esmpy.GridItem.MASK, - esmpy.StaggerLoc.CENTER, - ) - src_mask[...] = original_src_mask.T - center_mask = dst_field.grid.get_item( - esmpy.GridItem.MASK, - esmpy.StaggerLoc.CENTER, - ) - center_mask[...] = 0 - mask_regridder = esmpy.Regrid( - src_mask_values=MASK_REGRIDDING_MASK_VALUE[regrid_method], - dst_mask_values=np.array([]), - **regridding_arguments, - ) - regr_field = mask_regridder(src_field, dst_field) - dst_mask = mask_threshold > regr_field.data[...].T - center_mask[...] = dst_mask.T - if not dst_mask.any(): - dst_mask = np.ma.nomask - return dst_mask - - -def build_regridder_2d(src_rep, dst_rep, regrid_method, mask_threshold): - """Build regridder for 2d regridding.""" - dst_field = cube_to_empty_field(dst_rep) - src_field = cube_to_empty_field(src_rep) - regridding_arguments = { - "srcfield": src_field, - "dstfield": dst_field, - "regrid_method": regrid_method, - "unmapped_action": esmpy.UnmappedAction.IGNORE, - "ignore_degenerate": True, - } - dst_mask = regrid_mask_2d( - src_rep.data, - regridding_arguments, - mask_threshold, - ) - field_regridder = esmpy.Regrid( - src_mask_values=np.array([1]), - dst_mask_values=np.array([1]), - **regridding_arguments, - ) - - def regridder(src): - """Regrid 2d for irregular grids.""" - res = get_empty_data(dst_rep.shape, src.dtype) - data = src.data - if np.ma.is_masked(data): - data = data.data - src_field.data[...] = data.T - regr_field = field_regridder(src_field, dst_field) - res.data[...] = regr_field.data[...].T - res.mask[...] = dst_mask - return res - - return regridder - - -def build_regridder_3d(src_rep, dst_rep, regrid_method, mask_threshold): - # The necessary refactoring will be done for the full 3d regridding. - """Build regridder for 2.5d regridding.""" - esmf_regridders = [] - no_levels = src_rep.shape[0] - for level in range(no_levels): - esmf_regridders.append( - build_regridder_2d( - src_rep[level], - dst_rep[level], - regrid_method, - mask_threshold, - ), - ) - - def regridder(src): - """Regrid 2.5d for irregular grids.""" - res = get_empty_data(dst_rep.shape, src.dtype) - for i, esmf_regridder in enumerate(esmf_regridders): - res[i, ...] = esmf_regridder(src[i]) - return res - - return regridder - - -def build_regridder(src_rep, dst_rep, method, mask_threshold=0.99): - """Build regridders from representants.""" - regrid_method = ESMF_REGRID_METHODS[method] - if src_rep.ndim == 2: - regridder = build_regridder_2d( - src_rep, - dst_rep, - regrid_method, - mask_threshold, - ) - elif src_rep.ndim == 3: - regridder = build_regridder_3d( - src_rep, - dst_rep, - regrid_method, - mask_threshold, - ) - return regridder - - -def get_grid_representant(cube, horizontal_only=False): - """Extract the spatial grid from a cube.""" - horizontal_slice = ["latitude", "longitude"] - ref_to_slice = horizontal_slice - if not horizontal_only: - try: - cube_z_coord = cube.coord(axis="Z") - n_zdims = len(cube.coord_dims(cube_z_coord)) - if n_zdims == 0: - # scalar z coordinate, go on with 2d regridding - pass - elif n_zdims == 1: - ref_to_slice = [cube_z_coord, *horizontal_slice] - else: - msg = "Cube has multidimensional Z coordinate." - raise ValueError(msg) - except iris.exceptions.CoordinateNotFoundError: - # no z coordinate, go on with 2d regridding - pass - return get_representant(cube, ref_to_slice) - - -def get_grid_representants(src, dst): - """ - Construct cubes representing the source and destination grid. - - This method constructs two new cubes that representant the grids, - i.e. the spatial dimensions of the given cubes. - - Parameters - ---------- - src: :class:`iris.cube.Cube` - Cube to be regridded. Typically a time series of 2d or 3d slices. - dst: :class:`iris.cube.Cube` - Cube defining the destination grid. Usually just a 2d or 3d cube. - - Returns - ------- - tuple of :class:`iris.cube.Cube`: - A tuple containing two cubes, representing the source grid and the - destination grid, respectively. - """ - src_rep = get_grid_representant(src) - dst_horiz_rep = get_grid_representant(dst, horizontal_only=True) - if src_rep.ndim == 3: - dst_shape = (src_rep.shape[0],) - dim_coords = [src_rep.coord(dimensions=[0], dim_coords=True)] - else: - dst_shape = () - dim_coords = [] - dst_shape += dst_horiz_rep.shape - dim_coords += dst_horiz_rep.coords(dim_coords=True) - dim_coords_and_dims = [(c, i) for i, c in enumerate(dim_coords)] - aux_coords_and_dims = [] - for coord in dst_horiz_rep.aux_coords: - dims = dst_horiz_rep.coord_dims(coord) - if not dims: - continue - if src_rep.ndim == 3: - dims = [dim + 1 for dim in dims] - aux_coords_and_dims.append((coord, dims)) - - # Add scalar dimensions of source cube to target - for scalar_coord in src.coords(dimensions=()): - aux_coords_and_dims.append((scalar_coord, ())) - - dst_rep = iris.cube.Cube( - data=get_empty_data(dst_shape, src.dtype), - standard_name=src.standard_name, - long_name=src.long_name, - var_name=src.var_name, - units=src.units, - attributes=src.attributes, - cell_methods=src.cell_methods, - dim_coords_and_dims=dim_coords_and_dims, - aux_coords_and_dims=aux_coords_and_dims, - ) - return src_rep, dst_rep diff --git a/esmvalcore/preprocessor/_regrid_iris_esmf_regrid.py b/esmvalcore/preprocessor/_regrid_iris_esmf_regrid.py index f443982051..4437a09cbc 100644 --- a/esmvalcore/preprocessor/_regrid_iris_esmf_regrid.py +++ b/esmvalcore/preprocessor/_regrid_iris_esmf_regrid.py @@ -34,6 +34,11 @@ class IrisESMFRegrid: """:doc:`esmf_regrid:index` based regridding scheme. + This regridding scheme is a thin wrapper around the regridders provided by + :mod:`esmf_regrid` with improved handling of masks. This allows using + these regridders from the :ref:`regrid preprocessor ` + function. + Supports lazy regridding. Parameters @@ -131,9 +136,7 @@ def __init__( # noqa: PLR0913 "`method` should be one of 'bilinear', 'conservative', or " "'nearest'" ) - raise ValueError( - msg, - ) + raise ValueError(msg) if use_src_mask is None: use_src_mask = method != "nearest" @@ -154,9 +157,7 @@ def __init__( # noqa: PLR0913 "`mdol` can only be specified when `method='bilinear'` " "or `method='conservative'`" ) - raise TypeError( - msg, - ) + raise TypeError(msg) else: self.kwargs["mdtol"] = mdtol if method == "conservative": @@ -167,17 +168,13 @@ def __init__( # noqa: PLR0913 "`src_resolution` can only be specified when " "`method='conservative'`" ) - raise TypeError( - msg, - ) + raise TypeError(msg) elif tgt_resolution is not None: msg = ( "`tgt_resolution` can only be specified when " "`method='conservative'`" ) - raise TypeError( - msg, - ) + raise TypeError(msg) def __repr__(self) -> str: """Return string representation of class.""" diff --git a/esmvalcore/preprocessor/regrid_schemes.py b/esmvalcore/preprocessor/regrid_schemes.py index 965dfad2a7..01247ebc42 100644 --- a/esmvalcore/preprocessor/regrid_schemes.py +++ b/esmvalcore/preprocessor/regrid_schemes.py @@ -5,12 +5,6 @@ import logging from typing import TYPE_CHECKING -from esmvalcore.preprocessor._regrid_esmpy import ( - ESMPyAreaWeighted, - ESMPyLinear, - ESMPyNearest, - ESMPyRegridder, -) from esmvalcore.preprocessor._regrid_iris_esmf_regrid import IrisESMFRegrid from esmvalcore.preprocessor._regrid_unstructured import ( UnstructuredLinear, @@ -26,10 +20,6 @@ logger = logging.getLogger(__name__) __all__ = [ - "ESMPyAreaWeighted", - "ESMPyLinear", - "ESMPyNearest", - "ESMPyRegridder", "IrisESMFRegrid", "GenericFuncScheme", "GenericRegridder", diff --git a/pyproject.toml b/pyproject.toml index bd853ba50d..0f9b12a331 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ dependencies = [ "dask-jobqueue", "esgf-pyclient>=0.3.1", "esmf-regrid>=0.11.0", - "esmpy", # not on PyPI "filelock", "fiona", "fire", diff --git a/tests/unit/preprocessor/_mapping/__init__.py b/tests/unit/preprocessor/_mapping/__init__.py deleted file mode 100644 index 50b3449b35..0000000000 --- a/tests/unit/preprocessor/_mapping/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Unit tests for the :mod:`esmvalcore.preprocessor._mapping` module.""" diff --git a/tests/unit/preprocessor/_mapping/test_mapping.py b/tests/unit/preprocessor/_mapping/test_mapping.py deleted file mode 100644 index 07f1ed1e93..0000000000 --- a/tests/unit/preprocessor/_mapping/test_mapping.py +++ /dev/null @@ -1,289 +0,0 @@ -"""Unit tests for the esmvalcore.preprocessor._mapping module.""" - -from unittest import mock - -import cf_units -import iris -import numpy as np - -import tests -from esmvalcore.preprocessor._mapping import ( - get_empty_data, - map_slices, - ref_to_dims_index, -) - - -class TestHelpers(tests.Test): - """Unit tests for all helper methods.""" - - def setUp(self): - """Set up basic fixtures.""" - self.coord_system = mock.Mock(return_value=None) - self.scalar_coord = mock.sentinel.scalar_coord - self.scalar_coord.name = lambda: "scalar_coord" - self.coord = mock.sentinel.coord - self.coords = mock.Mock(return_value=[self.scalar_coord, self.coord]) - - def coord(name_or_coord): - """Return coord for mock cube.""" - if name_or_coord == "coord": - return self.coord - if name_or_coord == "scalar_coord": - return self.scalar_coord - msg = "" - raise iris.exceptions.CoordinateNotFoundError(msg) - - def coord_dims(coord): - """Return associated dims for coord in mock cube.""" - if coord == self.coord: - return [0] - if coord == self.scalar_coord: - return [] - msg = "" - raise iris.exceptions.CoordinateNotFoundError(msg) - - self.cube = mock.Mock( - spec=iris.cube.Cube, - dtype=np.float32, - coord_system=self.coord_system, - coords=self.coords, - coord=coord, - coord_dims=coord_dims, - ndim=4, - ) - - def test_get_empty_data(self): - """Test creation of empty data.""" - shape = (3, 3) - data = get_empty_data(shape) - self.assertIsInstance(data, np.ma.MaskedArray) - self.assertEqual(data.shape, shape) - - def test_ref_to_dims_index__int(self): - """Test ref_to_dims_index with valid integer.""" - dims = ref_to_dims_index(self.cube, 0) - self.assertEqual([0], dims) - - def test_ref_to_dims_index__invalid_int(self): - """Test ref_to_dims_index with invalid integer.""" - self.assertRaises(ValueError, ref_to_dims_index, self.cube, -1) - self.assertRaises(ValueError, ref_to_dims_index, self.cube, 100) - - def test_ref_to_dims_index__scalar_coord(self): - """Test ref_to_dims_index with scalar coordinate.""" - self.assertRaises( - ValueError, - ref_to_dims_index, - self.cube, - "scalar_coord", - ) - - def test_ref_to_dims_index__valid_coordinate_name(self): - """Test ref_to_dims_index with valid coordinate name.""" - dims = ref_to_dims_index(self.cube, "coord") - self.assertEqual([0], dims) - - def test_ref_to_dims_index__invalid_coordinate_name(self): - """Test ref_to_dims_index with invalid coordinate name.""" - self.assertRaises( - iris.exceptions.CoordinateNotFoundError, - ref_to_dims_index, - self.cube, - "test", - ) - - def test_ref_to_dims_index__invalid_type(self): - """Test ref_to_dims_index with invalid argument.""" - self.assertRaises( - ValueError, - ref_to_dims_index, - self.cube, - mock.sentinel.something, - ) - - -class Test(tests.Test): - """Unit tests for the main mapping method.""" - - # pylint: disable=too-many-instance-attributes - - def setup_coordinates(self): - """Set up coordinates for mock cube.""" - self.time = mock.Mock( - spec=iris.coords.DimCoord, - standard_name="time", - long_name="time", - shape=(3,), - ) - self.z = mock.Mock( - spec=iris.coords.DimCoord, - standard_name="height", - long_name="height", - shape=(4,), - ) - self.src_latitude = mock.Mock( - spec=iris.coords.DimCoord, - standard_name="latitude", - long_name="latitude", - shape=(5,), - points=np.array([1.1, 2.2, 3.3, 4.4, 5.5]), - ) - self.src_longitude = mock.Mock( - spec=iris.coords.DimCoord, - standard_name="longitude", - long_name="longitude", - shape=(6,), - points=np.array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6]), - ) - self.dst_latitude = mock.Mock( - spec=iris.coords.DimCoord, - standard_name="latitude", - long_name="latitude", - shape=(2,), - points=np.array([1.1, 2.2]), - ) - self.dst_longitude = mock.Mock( - spec=iris.coords.DimCoord, - standard_name="longitude", - long_name="longitude", - shape=(2,), - points=np.array([1.1, 2.2]), - ) - - def setUp( # noqa: C901 - self, - ): - """Set up fixtures for mapping test.""" - self.coord_system = mock.Mock(return_value=None) - self.scalar_coord = mock.sentinel.scalar_coord - self.scalar_coord.name = lambda: "scalar_coord" - self.setup_coordinates() - - def src_coord(name_or_coord): - """Return coord for mock source cube.""" - if name_or_coord in ["latitude", self.src_latitude]: - return self.src_latitude - if name_or_coord in ["longitude", self.src_longitude]: - return self.src_longitude - if name_or_coord == "scalar_coord": - return self.scalar_coord - msg = "" - raise iris.exceptions.CoordinateNotFoundError(msg) - - def coord_dims(coord): - """Return coord dim for mock cubes.""" - if coord in [self.time, self.dst_latitude]: - return [0] - if coord in [self.z, self.dst_longitude]: - return [1] - if coord in [self.src_latitude]: - return [2] - if coord in [self.src_longitude]: - return [3] - if coord == self.scalar_coord: - return [] - msg = "" - raise iris.exceptions.CoordinateNotFoundError(msg) - - def src_coords(*args, **kwargs): - """Return selected coords for source cube.""" - # pylint: disable=unused-argument - # Here, args is ignored. - dim_coords_list = [ - self.time, - self.z, - self.src_latitude, - self.src_longitude, - ] - contains_dimension = kwargs.get("contains_dimension") - dim_coords = kwargs.get("dim_coords") - if contains_dimension is not None: - if dim_coords: - return [dim_coords_list[contains_dimension]] - return [] - if dim_coords: - return dim_coords_list - return [self.scalar_coord, *dim_coords_list] - - def src_repr_coords(*args, **kwargs): - """Return selected coords for source representant cube.""" - # pylint: disable=unused-argument - # Here, args is ignored. - dim_coords = [self.src_latitude, self.src_longitude] - if kwargs.get("dim_coords", False): - return dim_coords - if "contains_dimension" in kwargs: - return dim_coords - return [self.scalar_coord, *dim_coords] - - def dst_repr_coords(*args, **kwargs): - """Return selected coords for destination representant cube.""" - # pylint: disable=unused-argument - # Here, args is ignored. - dim_coords = [self.dst_latitude, self.dst_longitude] - if kwargs.get("dim_coords", False): - return dim_coords - return [self.scalar_coord, *dim_coords] - - self.src_cube = mock.Mock( - spec=iris.cube.Cube, - dtype=np.float32, - coord_system=self.coord_system, - coords=src_coords, - coord=src_coord, - coord_dims=coord_dims, - ndim=4, - shape=(3, 4, 5, 6), - standard_name="sea_surface_temperature", - long_name="Sea surface temperature", - var_name="tos", - units=cf_units.Unit("K"), - attributes={}, - cell_methods={}, - aux_coords=[], - __getitem__=lambda a, b: mock.sentinel.src_data, - ) - self.src_repr = mock.Mock( - spec=iris.cube.Cube, - dtype=np.float32, - coords=src_repr_coords, - ndim=2, - aux_coords=[], - ) - self.dst_repr = mock.Mock( - spec=iris.cube.Cube, - dtype=np.float32, - coords=dst_repr_coords, - shape=(2, 2), - aux_coords=[], - ) - - @mock.patch("esmvalcore.preprocessor._mapping.get_empty_data") - @mock.patch("iris.cube.Cube") - def test_map_slices(self, mock_cube, mock_get_empty_data): - """Test map_slices.""" - mock_get_empty_data.return_value = mock.sentinel.empty_data - mock_cube.aux_coords = [] - dst = map_slices( - self.src_cube, - lambda s: np.ones((2, 2)), - self.src_repr, - self.dst_repr, - ) - self.assertEqual(dst, mock_cube.return_value) - dim_coords = self.src_cube.coords(dim_coords=True)[ - :2 - ] + self.dst_repr.coords(dim_coords=True) - dim_coords_and_dims = [(c, i) for i, c in enumerate(dim_coords)] - mock_cube.assert_called_once_with( - data=mock.sentinel.empty_data, - standard_name=self.src_cube.standard_name, - long_name=self.src_cube.long_name, - var_name=self.src_cube.var_name, - units=self.src_cube.units, - attributes=self.src_cube.attributes, - cell_methods=self.src_cube.cell_methods, - dim_coords_and_dims=dim_coords_and_dims, - aux_coords_and_dims=[], - ) diff --git a/tests/unit/preprocessor/_regrid_esmpy/__init__.py b/tests/unit/preprocessor/_regrid_esmpy/__init__.py deleted file mode 100644 index 22f93540f9..0000000000 --- a/tests/unit/preprocessor/_regrid_esmpy/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Unit tests for the :mod:`esmvalcore.preprocessor._regrid_esmpy` module.""" diff --git a/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py b/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py deleted file mode 100644 index cc78c8c339..0000000000 --- a/tests/unit/preprocessor/_regrid_esmpy/test_regrid_esmpy.py +++ /dev/null @@ -1,917 +0,0 @@ -"""Unit tests for the esmvalcore.preprocessor._regrid_esmpy module.""" - -from unittest import mock - -import cf_units -import iris -import numpy as np -import pytest -from iris.exceptions import CoordinateNotFoundError - -import tests -from esmvalcore.preprocessor._regrid_esmpy import ( - ESMPyAreaWeighted, - ESMPyLinear, - ESMPyNearest, - build_regridder, - build_regridder_2d, - coords_iris_to_esmpy, - cube_to_empty_field, - get_grid, - get_grid_representant, - get_grid_representants, - get_representant, - is_lon_circular, -) - - -def identity(*args, **kwargs): - """Return args, acting as identity for mocking functions.""" - # pylint: disable=unused-argument - # Here, kwargs will be ignored. - if len(args) == 1: - return args[0] - return args - - -def mock_cube_to_empty_field(cube): - """Return associated field for mock cube.""" - return cube.field - - -class MockGrid(mock.MagicMock): - """Mock ESMF grid.""" - - get_coords = mock.Mock(return_value=mock.MagicMock()) - add_coords = mock.Mock() - add_item = mock.Mock() - get_item = mock.Mock(return_value=mock.MagicMock()) - - -class MockGridItem(mock.Mock): - """Mock ESMF enum for grid items.""" - - MASK = mock.sentinel.gi_mask - - -class MockRegridMethod(mock.Mock): - """Mock ESMF enum for regridding methods.""" - - BILINEAR = mock.sentinel.rm_bilinear - CONSERVE = mock.sentinel.rm_conserve - NEAREST_STOD = mock.sentinel.rm_nearest_stod - - -class MockStaggerLoc(mock.Mock): - """Mock ESMF enum for stagger locations.""" - - CENTER = mock.sentinel.sl_center - CORNER = mock.sentinel.sl_corner - - -class MockUnmappedAction(mock.Mock): - """Mock ESMF enum for unmapped actions.""" - - IGNORE = mock.sentinel.ua_ignore - - -ESMF_REGRID_METHODS = { - "linear": MockRegridMethod.BILINEAR, - "area_weighted": MockRegridMethod.CONSERVE, - "nearest": MockRegridMethod.NEAREST_STOD, -} - -MASK_REGRIDDING_MASK_VALUE = { - mock.sentinel.rm_bilinear: np.array([1]), - mock.sentinel.rm_conserve: np.array([1]), - mock.sentinel.rm_nearest_stod: np.array([]), -} - - -@mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.MASK_REGRIDDING_MASK_VALUE", - MASK_REGRIDDING_MASK_VALUE, -) -@mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.ESMF_REGRID_METHODS", - ESMF_REGRID_METHODS, -) -@mock.patch("esmvalcore.preprocessor._regrid_esmpy.esmpy.Manager", mock.Mock) -@mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.esmpy.GridItem", - MockGridItem, -) -@mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.esmpy.RegridMethod", - MockRegridMethod, -) -@mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.esmpy.StaggerLoc", - MockStaggerLoc, -) -@mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.esmpy.UnmappedAction", - MockUnmappedAction, -) -class TestHelpers(tests.Test): - """Unit tests for helper functions.""" - - # pylint: disable=too-many-instance-attributes, too-many-public-methods - def setUp(self): # noqa: PLR0915 - """Set up fixtures.""" - # pylint: disable=too-many-locals - lat_1d_pre_bounds = np.linspace(-90, 90, 5) - lat_1d_bounds = np.stack( - [lat_1d_pre_bounds[:-1], lat_1d_pre_bounds[1:]], - axis=1, - ) - lat_1d_points = lat_1d_bounds.mean(axis=1) - lon_1d_pre_bounds = np.linspace(0, 360, 5) - lon_1d_bounds = np.stack( - [lon_1d_pre_bounds[:-1], lon_1d_pre_bounds[1:]], - axis=1, - ) - lon_1d_points = lon_1d_bounds.mean(axis=1) - lon_2d_points, lat_2d_points = np.meshgrid( - lon_1d_points, - lat_1d_points, - ) - (lon_2d_pre_bounds, lat_2d_pre_bounds) = np.meshgrid( - lon_1d_pre_bounds, - lat_1d_pre_bounds, - ) - lat_2d_bounds = np.stack( - [ - lat_2d_pre_bounds[:-1, :-1], - lat_2d_pre_bounds[:-1, 1:], - lat_2d_pre_bounds[1:, 1:], - lat_2d_pre_bounds[1:, :-1], - ], - axis=2, - ) - lon_2d_bounds = np.stack( - [ - lon_2d_pre_bounds[:-1, :-1], - lon_2d_pre_bounds[:-1, 1:], - lon_2d_pre_bounds[1:, 1:], - lon_2d_pre_bounds[1:, :-1], - ], - axis=2, - ) - self.lat_1d = mock.Mock( - iris.coords.DimCoord, - standard_name="latitude", - long_name="latitude", - ndim=1, - points=lat_1d_points, - bounds=lat_1d_bounds, - has_bounds=mock.Mock(return_value=True), - ) - self.lat_1d_no_bounds = mock.Mock( - iris.coords.DimCoord, - standard_name="latitude", - ndim=1, - points=lat_1d_points, - has_bounds=mock.Mock(return_value=False), - bounds=lat_1d_bounds, - guess_bounds=mock.Mock(), - ) - self.lon_1d = mock.Mock( - iris.coords.DimCoord, - standard_name="longitude", - long_name="longitude", - ndim=1, - points=lon_1d_points, - bounds=lon_1d_bounds, - has_bounds=mock.Mock(return_value=True), - circular=True, - ) - self.lon_1d_aux = mock.Mock( - iris.coords.AuxCoord, - standard_name="longitude", - long_name="longitude", - ndim=1, - shape=lon_1d_points.shape, - points=lon_1d_points, - bounds=lon_1d_bounds, - has_bounds=mock.Mock(return_value=True), - ) - self.lat_2d = mock.Mock( - iris.coords.AuxCoord, - standard_name="latitude", - long_name="latitude", - ndim=2, - points=lat_2d_points, - bounds=lat_2d_bounds, - has_bounds=mock.Mock(return_value=True), - ) - self.lon_2d = mock.Mock( - iris.coords.AuxCoord, - standard_name="longitude", - long_name="longitude", - ndim=2, - points=lon_2d_points, - bounds=lon_2d_bounds, - has_bounds=mock.Mock(return_value=True), - ) - self.lon_2d_non_circular = mock.Mock( - iris.coords.AuxCoord, - standard_name="longitude", - ndim=2, - points=lon_2d_points[:, 1:-1], - bounds=lon_2d_bounds[:, 1:-1], - has_bounds=mock.Mock(return_value=True), - ) - self.lat_3d = mock.Mock( - iris.coords.AuxCoord, - standard_name="latitude", - long_name="latitude", - ndim=3, - ) - self.lon_3d = mock.Mock( - iris.coords.AuxCoord, - standard_name="longitude", - long_name="longitude", - ndim=3, - ) - depth_pre_bounds = np.linspace(0, 5000, 5) - depth_bounds = np.stack( - [depth_pre_bounds[:-1], depth_pre_bounds[1:]], - axis=1, - ) - depth_points = depth_bounds.mean(axis=1) - self.depth = mock.Mock( - iris.coords.DimCoord, - standard_name="depth", - long_name="depth", - ndim=1, - shape=depth_points.shape, - points=depth_points, - bounds=depth_bounds, - has_bounds=mock.Mock(return_value=True), - ) - self.scalar_coord = mock.Mock( - iris.coords.AuxCoord, - long_name="scalar_coord", - ndim=1, - shape=(1,), - ) - data_shape = lon_2d_points.shape - raw_data = np.arange(np.prod(data_shape)).reshape(data_shape) - mask = np.zeros(data_shape) - mask[: data_shape[0] // 2] = True - self.data = np.ma.masked_array(raw_data, mask) - self.data_3d = np.repeat( - self.data[..., np.newaxis], - depth_points.shape[0], - axis=-1, - ) - self.expected_esmpy_lat = np.array( - [ - [-67.5, -22.5, 22.5, 67.5], - [-67.5, -22.5, 22.5, 67.5], - [-67.5, -22.5, 22.5, 67.5], - [-67.5, -22.5, 22.5, 67.5], - ], - ) - self.expected_esmpy_lon = np.array( - [ - [45.0, 45.0, 45.0, 45.0], - [135.0, 135.0, 135.0, 135.0], - [225.0, 225.0, 225.0, 225.0], - [315.0, 315.0, 315.0, 315.0], - ], - ) - self.expected_esmpy_lat_corners = np.array( - [ - [-90.0, -45.0, 0.0, 45.0, 90.0], - [-90.0, -45.0, 0.0, 45.0, 90.0], - [-90.0, -45.0, 0.0, 45.0, 90.0], - [-90.0, -45.0, 0.0, 45.0, 90.0], - [-90.0, -45.0, 0.0, 45.0, 90.0], - ], - ) - self.expected_esmpy_lon_corners = np.array( - [ - [0.0, 0.0, 0.0, 0.0, 0.0], - [90.0, 90.0, 90.0, 90.0, 90.0], - [180.0, 180.0, 180.0, 180.0, 180.0], - [270.0, 270.0, 270.0, 270.0, 270.0], - [360.0, 360.0, 360.0, 360.0, 360.0], - ], - ) - self.coords = { - "latitude": self.lat_2d, - "longitude": self.lon_2d, - "depth": self.depth, - "scalar_coord": self.scalar_coord, - } - self.coord_dims = { - "latitude": (0, 1), - "longitude": (0, 1), - self.lat_2d: (0, 1), - self.lon_2d: (0, 1), - "scalar_coord": (), - } - - def coord(name=None, axis=None): - """Return selected coordinate for mock cube.""" - if axis == "Z": - raise CoordinateNotFoundError - return self.coords[name] - - def coords(dim_coords=None, dimensions=None): - """Return coordinates for mock cube.""" - if dim_coords: - return [] - if dimensions == (): - return [self.scalar_coord] - return list(self.coords.values()) - - self.cube = mock.Mock( - spec=iris.cube.Cube, - dtype=np.float32, - long_name="longname", - ndim=2, - shape=self.data.shape, - data=self.data, - coord=coord, - coord_dims=lambda name: self.coord_dims[name], - coords=coords, - ) - self.cube.__getitem__ = mock.Mock(return_value=self.cube) - self.cube.aux_coords = [] - self.unmasked_cube = mock.Mock( - spec=iris.cube.Cube, - dtype=np.float32, - long_name="longname", - ) - self.coord_dims_3d = { - "latitude": (1, 2), - "longitude": (1, 2), - self.lat_2d: (1, 2), - self.lon_2d: (1, 2), - "depth": (0,), - self.depth: (0,), - } - - def coord_3d(name=None, dimensions=None, dim_coords=None, axis=None): - """Return coord for 3d mock cube.""" - # pylint: disable=unused-argument - if axis == "Z" or dimensions == [0]: - return self.coords["depth"] - return self.coords[name] - - def coords_3d(dimensions=None): - """Return coordinates for mock cube.""" - if dimensions == (): - return [] - return [self.lat_2d, self.lon_2d, self.depth] - - self.cube_3d = mock.Mock( - spec=iris.cube.Cube, - dtype=np.float32, - standard_name=None, - long_name="longname", - var_name="ln", - units=cf_units.Unit("1"), - attributes={}, - cell_methods=[], - ndim=3, - shape=self.data_3d.shape, - data=self.data_3d, - coord=coord_3d, - coord_dims=lambda name: self.coord_dims_3d[name], - coords=coords_3d, - ) - self.cube.__getitem__ = mock.Mock(return_value=self.cube) - - def test_coords_iris_to_esmpy_mismatched_dimensions(self): - """Test coord conversion with mismatched dimensions.""" - self.assertRaises( - ValueError, - coords_iris_to_esmpy, - self.lat_1d, - self.lon_2d, - True, - ) - - def test_coords_iris_to_esmpy_invalid_dimensions(self): - """Test coord conversion with invalid dimensions.""" - self.assertRaises( - NotImplementedError, - coords_iris_to_esmpy, - self.lat_3d, - self.lon_3d, - True, - ) - - def test_coords_iris_to_esmpy_call_guess_bounds(self): - """Test coord conversion with missing bounds.""" - coords_iris_to_esmpy(self.lat_1d_no_bounds, self.lon_1d, True) - self.lat_1d_no_bounds.guess_bounds.assert_called_once() - - def test_coords_iris_to_esmpy_1d_circular(self): - """Test coord conversion with 1d coords and circular longitudes.""" - (esmpy_lat, esmpy_lon, esmpy_lat_corners, esmpy_lon_corners) = ( - coords_iris_to_esmpy(self.lat_1d, self.lon_1d, True) - ) - self.assert_array_equal(esmpy_lat, self.expected_esmpy_lat) - self.assert_array_equal(esmpy_lon, self.expected_esmpy_lon) - self.assert_array_equal( - esmpy_lat_corners, - self.expected_esmpy_lat_corners[:-1], - ) - self.assert_array_equal( - esmpy_lon_corners, - self.expected_esmpy_lon_corners[:-1], - ) - - def test_coords_iris_to_esmpy_1d_non_circular(self): - """Test coord conversion with 1d coords and non circular longitudes.""" - (esmpy_lat, esmpy_lon, esmpy_lat_corners, esmpy_lon_corners) = ( - coords_iris_to_esmpy(self.lat_1d, self.lon_1d, False) - ) - self.assert_array_equal(esmpy_lat, self.expected_esmpy_lat) - self.assert_array_equal(esmpy_lon, self.expected_esmpy_lon) - self.assert_array_equal( - esmpy_lat_corners, - self.expected_esmpy_lat_corners, - ) - self.assert_array_equal( - esmpy_lon_corners, - self.expected_esmpy_lon_corners, - ) - - def test_coords_iris_to_esmpy_2d_circular(self): - """Test coord conversion with 2d coords and circular longitudes.""" - (esmpy_lat, esmpy_lon, esmpy_lat_corners, esmpy_lon_corners) = ( - coords_iris_to_esmpy(self.lat_2d, self.lon_2d, True) - ) - self.assert_array_equal(esmpy_lat, self.expected_esmpy_lat) - self.assert_array_equal(esmpy_lon, self.expected_esmpy_lon) - self.assert_array_equal( - esmpy_lat_corners, - self.expected_esmpy_lat_corners[:-1], - ) - self.assert_array_equal( - esmpy_lon_corners, - self.expected_esmpy_lon_corners[:-1], - ) - - def test_coords_iris_to_esmpy_2d_non_circular(self): - """Test coord conversion with 2d coords and non circular longitudes.""" - (esmpy_lat, esmpy_lon, esmpy_lat_corners, esmpy_lon_corners) = ( - coords_iris_to_esmpy(self.lat_2d, self.lon_2d, False) - ) - self.assert_array_equal(esmpy_lat, self.expected_esmpy_lat) - self.assert_array_equal(esmpy_lon, self.expected_esmpy_lon) - self.assert_array_equal( - esmpy_lat_corners, - self.expected_esmpy_lat_corners, - ) - self.assert_array_equal( - esmpy_lon_corners, - self.expected_esmpy_lon_corners, - ) - - def test_get_grid_circular(self): - """Test building of ESMF grid from iris cube circular longitude.""" - expected_get_coords_calls = [ - mock.call(0), - mock.call(1), - mock.call(0, staggerloc=mock.sentinel.sl_corner), - mock.call(1, staggerloc=mock.sentinel.sl_corner), - ] - with mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.esmpy.Grid", - MockGrid, - ) as mg: - mg.get_coords.reset_mock() - mg.add_coords.reset_mock() - mg.add_item.reset_mock() - get_grid( - self.expected_esmpy_lat, - self.expected_esmpy_lon, - self.expected_esmpy_lat_corners[:-1], - self.expected_esmpy_lon_corners[:-1], - True, - ) - mg.get_coords.assert_has_calls(expected_get_coords_calls) - mg.add_coords.assert_called_once_with([mock.sentinel.sl_corner]) - mg.add_item.assert_called_once_with( - mock.sentinel.gi_mask, - mock.sentinel.sl_center, - ) - - def test_get_grid_non_circular(self): - """Test building of ESMF grid from iris cube non circular longitude.""" - expected_get_coords_calls = [ - mock.call(0), - mock.call(1), - mock.call(0, staggerloc=mock.sentinel.sl_corner), - mock.call(1, staggerloc=mock.sentinel.sl_corner), - ] - with mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.esmpy.Grid", - MockGrid, - ) as mg: - mg.get_coords.reset_mock() - mg.add_coords.reset_mock() - mg.add_item.reset_mock() - get_grid( - self.expected_esmpy_lat, - self.expected_esmpy_lon, - self.expected_esmpy_lat_corners, - self.expected_esmpy_lon_corners, - False, - ) - mg.get_coords.assert_has_calls(expected_get_coords_calls) - mg.add_coords.assert_called_once_with([mock.sentinel.sl_corner]) - mg.add_item.assert_called_once_with( - mock.sentinel.gi_mask, - mock.sentinel.sl_center, - ) - - def test_is_lon_circular_dim_coords_true(self): - """Test detection of circular longitudes 1d dim coords.""" - is_circ = is_lon_circular(self.lon_1d) - self.assertTrue(is_circ) - - def test_is_lon_circular_dim_coords_false(self): - """Test detection of non circular longitudes 1d dim coords.""" - self.lon_1d.circular = False - is_circ = is_lon_circular(self.lon_1d) - self.assertFalse(is_circ) - - def test_is_lon_circular_1d_aux_coords(self): - """Test detection of circular longitudes 1d aux coords.""" - is_circ = is_lon_circular(self.lon_1d_aux) - self.assertTrue(is_circ) - - def test_is_lon_circular_invalid_dimension(self): - """Test detection of circular longitudes, invalid coordinates.""" - self.assertRaises(NotImplementedError, is_lon_circular, self.lon_3d) - - def test_is_lon_circular_invalid_argument(self): - """Test detection of circular longitudes, invalid argument.""" - self.assertRaises(TypeError, is_lon_circular, None) - - def test_is_lon_circular_2d_aux_coords(self): - """Test detection of circular longitudes 2d aux coords.""" - is_circ = is_lon_circular(self.lon_2d) - self.assertTrue(is_circ) - - def test_is_lon_circular_2d_aux_coords_non_circ(self): - """Test detection of non circular longitudes 2d aux coords.""" - is_circ = is_lon_circular(self.lon_2d_non_circular) - self.assertFalse(is_circ) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.esmpy.Grid", MockGrid) - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.esmpy.Field") - def test_cube_to_empty_field(self, mock_field): - """Test building of empty field from iris cube.""" - field = cube_to_empty_field(self.cube) - self.assertEqual(mock_field.return_value, field) - mock_field.assert_called_once() - ckwargs = mock_field.call_args[1] - self.assertEqual("longname", ckwargs["name"]) - self.assertEqual(mock.sentinel.sl_center, ckwargs["staggerloc"]) - - def test_get_representant(self): - """Test extraction of horizontal representant from iris cube.""" - horizontal_slice = ["latitude", "longitude"] - get_representant(self.cube, horizontal_slice) - self.cube.__getitem__.assert_called_once_with( - (slice(None, None, None), slice(None, None, None)), - ) - - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.cube_to_empty_field", - mock_cube_to_empty_field, - ) - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.esmpy.Regrid") - def test_build_regridder_2d_masked_data(self, mock_regrid): - """Test building of 2d regridder for masked data.""" - mock_regrid.return_value = mock.Mock( - return_value=mock.Mock(data=self.data.T), - ) - regrid_method = mock.sentinel.rm_bilinear - src_rep = mock.MagicMock(data=self.data) - dst_rep = mock.MagicMock() - src_rep.field = mock.MagicMock(data=self.data.copy()) - dst_rep.field = mock.MagicMock() - build_regridder_2d(src_rep, dst_rep, regrid_method, 0.99) - expected_calls = [ - mock.call( - src_mask_values=np.array([]), - dst_mask_values=np.array([]), - srcfield=src_rep.field, - dstfield=dst_rep.field, - unmapped_action=mock.sentinel.ua_ignore, - ignore_degenerate=True, - regrid_method=regrid_method, - ), - mock.call( - src_mask_values=np.array([1]), - dst_mask_values=np.array([1]), - regrid_method=regrid_method, - srcfield=src_rep.field, - dstfield=dst_rep.field, - unmapped_action=mock.sentinel.ua_ignore, - ignore_degenerate=True, - ), - ] - kwargs = mock_regrid.call_args_list[0][-1] - expected_kwargs = expected_calls[0][-1] - self.assertEqual(expected_kwargs.keys(), kwargs.keys()) - array_keys = {"src_mask_values", "dst_mask_values"} - for key in kwargs: - if key in array_keys: - self.assertTrue((expected_kwargs[key] == kwargs[key]).all()) - else: - self.assertEqual(expected_kwargs[key], kwargs[key]) - self.assertTrue(mock_regrid.call_args_list[1] == expected_calls[1]) - - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.cube_to_empty_field", - mock_cube_to_empty_field, - ) - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.esmpy.Regrid") - def test_regridder_2d_unmasked_data(self, mock_regrid): - """Test regridder for unmasked 2d data.""" - field_regridder = mock.Mock(return_value=mock.Mock(data=self.data.T)) - mock_regrid.return_value = field_regridder - regrid_method = mock.sentinel.rm_bilinear - src_rep = mock.MagicMock(data=self.data, dtype=np.float32) - dst_rep = mock.MagicMock(shape=(4, 4)) - regridder = build_regridder_2d(src_rep, dst_rep, regrid_method, 0.99) - field_regridder.reset_mock() - regridder(src_rep) - field_regridder.assert_called_once_with(src_rep.field, dst_rep.field) - - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.cube_to_empty_field", - mock_cube_to_empty_field, - ) - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.esmpy.Regrid") - def test_regridder_2d_masked_data(self, mock_regrid): - """Test regridder for masked 2d data.""" - field_regridder = mock.Mock(return_value=mock.Mock(data=self.data.T)) - mock_regrid.return_value = field_regridder - regrid_method = mock.sentinel.rm_bilinear - src_rep = mock.MagicMock(data=self.data) - dst_rep = mock.MagicMock(shape=(4, 4)) - regridder = build_regridder_2d(src_rep, dst_rep, regrid_method, 0.99) - field_regridder.reset_mock() - regridder(self.cube) - field_regridder.assert_called_once_with(src_rep.field, dst_rep.field) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.build_regridder_3d") - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.build_regridder_2d") - def test_build_regridder_2(self, mock_regridder_2d, mock_regridder_3d): - """Test build regridder for 2d data.""" - # pylint: disable=no-self-use - src_rep = mock.Mock(ndim=2) - dst_rep = mock.Mock(ndim=2) - build_regridder(src_rep, dst_rep, "nearest") - mock_regridder_2d.assert_called_once_with( - src_rep, - dst_rep, - mock.sentinel.rm_nearest_stod, - 0.99, - ) - mock_regridder_3d.assert_not_called() - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.build_regridder_3d") - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.build_regridder_2d") - def test_build_regridder_3(self, mock_regridder_2d, mock_regridder_3d): - """Test build regridder for 3d data.""" - # pylint: disable=no-self-use - src_rep = mock.Mock(ndim=3) - dst_rep = mock.Mock(ndim=3) - build_regridder(src_rep, dst_rep, "nearest") - mock_regridder_3d.assert_called_once_with( - src_rep, - dst_rep, - mock.sentinel.rm_nearest_stod, - 0.99, - ) - mock_regridder_2d.assert_not_called() - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.get_representant") - def test_get_grid_representant_2d(self, mock_get_representant): - """Test extraction of 2d grid representant from 2 spatial d cube.""" - mock_get_representant.return_value = mock.sentinel.ret - ret = get_grid_representant(self.cube) - self.assertEqual(mock.sentinel.ret, ret) - mock_get_representant.assert_called_once_with( - self.cube, - ["latitude", "longitude"], - ) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.get_representant") - def test_get_grid_representant_2d_horiz_only(self, mock_get_representant): - """Test extraction of forced 2d grid representant from 2d cube.""" - mock_get_representant.return_value = mock.sentinel.ret - ret = get_grid_representant(self.cube, True) - self.assertEqual(mock.sentinel.ret, ret) - mock_get_representant.assert_called_once_with( - self.cube, - ["latitude", "longitude"], - ) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.get_representant") - def test_get_grid_representant_3d(self, mock_get_representant): - """Test extraction of 3d grid representant from 3 spatial d cube.""" - mock_get_representant.return_value = mock.sentinel.ret - ret = get_grid_representant(self.cube_3d) - self.assertEqual(mock.sentinel.ret, ret) - mock_get_representant.assert_called_once_with( - self.cube_3d, - [self.depth, "latitude", "longitude"], - ) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.get_representant") - def test_get_grid_representant_3d_horiz_only(self, mock_get_representant): - """Test extraction of 2d grid representant from 3 spatial d cube.""" - mock_get_representant.return_value = mock.sentinel.ret - ret = get_grid_representant(self.cube_3d, True) - self.assertEqual(mock.sentinel.ret, ret) - mock_get_representant.assert_called_once_with( - self.cube_3d, - ["latitude", "longitude"], - ) - - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.get_grid_representant", - mock.Mock(side_effect=identity), - ) - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.get_empty_data") - @mock.patch("iris.cube.Cube") - def test_get_grid_representants_3d_src( - self, - mock_cube, - mock_get_empty_data, - ): - """Test extraction of grid representants from 3 spatial d cube.""" - src = self.cube_3d - mock_get_empty_data.return_value = mock.sentinel.empty_data - src_rep = get_grid_representants(src, self.cube)[0] - self.assertEqual(src, src_rep) - mock_cube.aux_coords = [] - mock_cube.assert_called_once_with( - data=mock.sentinel.empty_data, - standard_name=src.standard_name, - long_name=src.long_name, - var_name=src.var_name, - units=src.units, - attributes=src.attributes, - cell_methods=src.cell_methods, - dim_coords_and_dims=[(self.depth, 0)], - aux_coords_and_dims=[], - ) - - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.get_grid_representant", - mock.Mock(side_effect=identity), - ) - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.get_empty_data") - @mock.patch("iris.cube.Cube") - def test_get_grid_representants_2d_src( - self, - mock_cube, - mock_get_empty_data, - ): - """Test extraction of grid representants from 2 spatial d cube.""" - src = self.cube - mock_cube.aux_coords = [] - mock_get_empty_data.return_value = mock.sentinel.empty_data - src_rep = get_grid_representants(src, self.cube)[0] - self.assertEqual(src, src_rep) - mock_cube.assert_called_once_with( - data=mock.sentinel.empty_data, - standard_name=src.standard_name, - long_name=src.long_name, - var_name=src.var_name, - units=src.units, - attributes=src.attributes, - cell_methods=src.cell_methods, - dim_coords_and_dims=[], - aux_coords_and_dims=[(self.scalar_coord, ())], - ) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.map_slices") - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.build_regridder") - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.get_grid_representants", - mock.Mock(side_effect=identity), - ) - def test_regrid_nearest(self, mock_build_regridder, mock_map_slices): - """Test full regrid method.""" - mock_build_regridder.return_value = mock.sentinel.regridder - mock_map_slices.return_value = mock.sentinel.regridded - regridder = ESMPyNearest().regridder(self.cube_3d, self.cube) - regridder(self.cube_3d) - mock_build_regridder.assert_called_once_with( - self.cube_3d, - self.cube, - "nearest", - mask_threshold=0.99, - ) - mock_map_slices.assert_called_once_with( - self.cube_3d, - mock.sentinel.regridder, - self.cube_3d, - self.cube, - ) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.map_slices") - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.build_regridder") - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.get_grid_representants", - mock.Mock(side_effect=identity), - ) - def test_regrid_linear(self, mock_build_regridder, mock_map_slices): - """Test full regrid method.""" - mock_build_regridder.return_value = mock.sentinel.regridder - mock_map_slices.return_value = mock.sentinel.regridded - regridder = ESMPyLinear().regridder(self.cube_3d, self.cube) - regridder(self.cube_3d) - mock_build_regridder.assert_called_once_with( - self.cube_3d, - self.cube, - "linear", - mask_threshold=0.99, - ) - mock_map_slices.assert_called_once_with( - self.cube_3d, - mock.sentinel.regridder, - self.cube_3d, - self.cube, - ) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.map_slices") - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.build_regridder") - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.get_grid_representants", - mock.Mock(side_effect=identity), - ) - def test_regrid_area_weighted(self, mock_build_regridder, mock_map_slices): - """Test full regrid method.""" - mock_build_regridder.return_value = mock.sentinel.regridder - mock_map_slices.return_value = mock.sentinel.regridded - regridder = ESMPyAreaWeighted().regridder(self.cube_3d, self.cube) - regridder(self.cube_3d) - mock_build_regridder.assert_called_once_with( - self.cube_3d, - self.cube, - "area_weighted", - mask_threshold=0.99, - ) - mock_map_slices.assert_called_once_with( - self.cube_3d, - mock.sentinel.regridder, - self.cube_3d, - self.cube, - ) - - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.map_slices") - @mock.patch("esmvalcore.preprocessor._regrid_esmpy.build_regridder") - @mock.patch( - "esmvalcore.preprocessor._regrid_esmpy.get_grid_representants", - mock.Mock(side_effect=identity), - ) - def test_data_realized_once(self, mock_build_regridder, mock_map_slices): - """Test that the regridder realizes the data only once.""" - src_cube = mock.MagicMock() - src_data = mock.PropertyMock() - type(src_cube).data = src_data - tgt_cube1 = mock.MagicMock() - tgt_data1 = mock.PropertyMock() - type(tgt_cube1).data = tgt_data1 - # Check that constructing the regridder realizes the source and - # target data. - regridder = ESMPyAreaWeighted().regridder(src_cube, tgt_cube1) - src_data.assert_called_with() - tgt_data1.assert_called_with() - tgt_cube2 = mock.MagicMock() - tgt_data2 = mock.PropertyMock() - # Check that calling the regridder with another cube also realizes - # target data. - type(tgt_cube2).data = tgt_data2 - regridder(tgt_cube2) - tgt_data2.assert_called_with() - - -@pytest.mark.parametrize( - ("scheme", "output"), - [ - (ESMPyAreaWeighted(), "ESMPyAreaWeighted(mask_threshold=0.99)"), - (ESMPyLinear(), "ESMPyLinear(mask_threshold=0.99)"), - (ESMPyNearest(), "ESMPyNearest(mask_threshold=0.99)"), - ], -) -def test_scheme_repr(scheme, output): - """Test ``_ESMPyScheme.__repr__``.""" - assert scheme.__repr__() == output