Skip to content

Commit 4b9e156

Browse files
davidhasselldavidhassell
committed
dev
1 parent c964060 commit 4b9e156

File tree

5 files changed

+61
-186
lines changed

5 files changed

+61
-186
lines changed

‎cf/data/array/h5netcdfarray.py‎

Lines changed: 1 addition & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,11 @@
11
import cfdm
22

33
from ...mixin_container import Container
4-
5-
# from .locks import netcdf_lock
6-
from .mixin import ( # , IndexMixin
7-
ActiveStorageMixin,
8-
ArrayMixin,
9-
FileArrayMixin,
10-
)
4+
from .mixin import ActiveStorageMixin, ArrayMixin, FileArrayMixin
115

126

137
class H5netcdfArray(
148
ActiveStorageMixin,
15-
# IndexMixin,
169
FileArrayMixin,
1710
ArrayMixin,
1811
Container,
@@ -28,59 +21,3 @@ class H5netcdfArray(
2821
.. versionadded:: NEXTVERSION
2922
3023
"""
31-
32-
# def __dask_tokenize__(self):
33-
# """Return a value fully representative of the object.
34-
35-
# .. versionadded:: NEXTVERSION
36-
37-
# """
38-
# return super().__dask_tokenize__() + (self.get_mask(),)
39-
40-
# @property
41-
# def _lock(self):
42-
# """Set the lock for use in `dask.array.from_array`.
43-
44-
# Returns a lock object because concurrent reads are not
45-
# currently supported by the HDF5 library. The lock object will
46-
# be the same for all `NetCDF4Array` and `H5netcdfArray`
47-
# instances, regardless of the dataset they access, which means
48-
# that access to all netCDF and HDF files coordinates around the
49-
# same lock.
50-
51-
# .. versionadded:: NEXTVERSION
52-
53-
# """
54-
# return netcdf_lock
55-
56-
# def _get_array(self, index=None):
57-
# """Returns a subspace of the dataset variable.
58-
59-
# .. versionadded:: NEXTVERSION
60-
61-
# .. seealso:: `__array__`, `index`
62-
63-
# :Parameters:
64-
65-
# {{index: `tuple` or `None`, optional}}
66-
67-
# :Returns:
68-
69-
# `numpy.ndarray`
70-
# The subspace.
71-
72-
# """
73-
# if index is None:
74-
# index = self.index()
75-
76-
# # We need to lock because the netCDF file is about to be accessed.
77-
# self._lock.acquire()
78-
79-
# # It's cfdm.H5netcdfArray.__getitem__ that we want to
80-
# # call here, but we use 'Container' in super because
81-
# # that comes immediately before cfdm.H5netcdfArray in
82-
# # the method resolution order.
83-
# array = super(Container, self).__getitem__(index)
84-
85-
# self._lock.release()
86-
# return array

‎cf/data/array/locks.py‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎cf/data/array/netcdf4array.py‎

Lines changed: 0 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -19,62 +19,3 @@ class NetCDF4Array(
1919
method. See `cf.data.collapse.Collapse` for details.
2020
2121
"""
22-
23-
# def __dask_tokenize__(self):
24-
# """Return a value fully representative of the object.
25-
#
26-
# .. versionadded:: 3.15.0
27-
#
28-
# """
29-
# return super().__dask_tokenize__() + (self.get_mask(),)
30-
31-
32-
#
33-
# @property
34-
# def _lock(self):
35-
# """Set the lock for use in `dask.array.from_array`.
36-
#
37-
# Returns a lock object because concurrent reads are not
38-
# currently supported by the netCDF and HDF libraries. The lock
39-
# object will be the same for all `NetCDF4Array` and
40-
# `H5netcdfArray` instances, regardless of the dataset they
41-
# access, which means that access to all netCDF and HDF files
42-
# coordinates around the same lock.
43-
#
44-
# .. versionadded:: 3.14.0
45-
#
46-
# """
47-
# return netcdf_lock
48-
#
49-
# def _get_array(self, index=None):
50-
# """Returns a subspace of the dataset variable.
51-
#
52-
# .. versionadded:: NEXTVERSION
53-
#
54-
# .. seealso:: `__array__`, `index`
55-
#
56-
# :Parameters:
57-
#
58-
# {{index: `tuple` or `None`, optional}}
59-
#
60-
# :Returns:
61-
#
62-
# `numpy.ndarray`
63-
# The subspace.
64-
#
65-
# """
66-
# if index is None:
67-
# index = self.index()
68-
#
69-
# # Note: We need to lock because the netCDF file is about to be
70-
# # accessed.
71-
# self._lock.acquire()
72-
#
73-
# # Note: It's cfdm.NetCDFArray.__getitem__ that we want to call
74-
# # here, but we use 'Container' in super because that
75-
# # comes immediately before cfdm.NetCDFArray in the
76-
# # method resolution order.
77-
# array = super(Container, self).__getitem__(index)
78-
#
79-
# self._lock.release()
80-
# return array

‎cf/regrid/regrid.py‎

Lines changed: 45 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -2462,10 +2462,10 @@ def create_esmpy_weights(
24622462
# Write the weights to a netCDF file (copying the
24632463
# dimension and variable names and structure of a weights
24642464
# file created by ESMF).
2465+
from cfdm.data.locks import netcdf_lock
24652466
from netCDF4 import Dataset
24662467

24672468
from .. import __version__
2468-
from ..data.array.locks import netcdf_lock
24692469

24702470
if (
24712471
max(dst_esmpy_field.data.size, src_esmpy_field.data.size)
@@ -2491,48 +2491,51 @@ def create_esmpy_weights(
24912491
if src_grid.ln_z:
24922492
regrid_method += f", ln {src_grid.method} in vertical"
24932493

2494-
netcdf_lock.acquire()
2495-
nc = Dataset(weights_file, "w", format="NETCDF4")
2494+
with netcdf_lock:
2495+
nc = Dataset(weights_file, "w", format="NETCDF4")
24962496

2497-
nc.title = (
2498-
f"Regridding weights from source {src_grid.type} "
2499-
f"with shape {src_shape} to destination "
2500-
f"{dst_grid.type} with shape {dst_shape}"
2501-
)
2502-
nc.source = f"cf v{__version__}, esmpy v{esmpy.__version__}"
2503-
nc.history = f"Created at {datetime.now()}"
2504-
nc.regrid_method = regrid_method
2505-
nc.ESMF_unmapped_action = r.unmapped_action
2506-
nc.ESMF_ignore_degenerate = int(r.ignore_degenerate)
2507-
2508-
nc.createDimension("n_s", weights.size)
2509-
nc.createDimension("src_grid_rank", src_esmpy_grid.rank)
2510-
nc.createDimension("dst_grid_rank", dst_esmpy_grid.rank)
2511-
2512-
v = nc.createVariable("src_grid_dims", i_dtype, ("src_grid_rank",))
2513-
v.long_name = "Source grid shape"
2514-
v[...] = src_shape
2515-
2516-
v = nc.createVariable("dst_grid_dims", i_dtype, ("dst_grid_rank",))
2517-
v.long_name = "Destination grid shape"
2518-
v[...] = dst_shape
2519-
2520-
v = nc.createVariable("S", weights.dtype, ("n_s",))
2521-
v.long_name = "Weights values"
2522-
v[...] = weights
2523-
2524-
v = nc.createVariable("row", i_dtype, ("n_s",), zlib=True)
2525-
v.long_name = "Destination/row indices"
2526-
v.start_index = start_index
2527-
v[...] = row
2528-
2529-
v = nc.createVariable("col", i_dtype, ("n_s",), zlib=True)
2530-
v.long_name = "Source/col indices"
2531-
v.start_index = start_index
2532-
v[...] = col
2533-
2534-
nc.close()
2535-
netcdf_lock.release()
2497+
nc.title = (
2498+
f"Regridding weights from source {src_grid.type} "
2499+
f"with shape {src_shape} to destination "
2500+
f"{dst_grid.type} with shape {dst_shape}"
2501+
)
2502+
nc.source = f"cf v{__version__}, esmpy v{esmpy.__version__}"
2503+
nc.history = f"Created at {datetime.now()}"
2504+
nc.regrid_method = regrid_method
2505+
nc.ESMF_unmapped_action = r.unmapped_action
2506+
nc.ESMF_ignore_degenerate = int(r.ignore_degenerate)
2507+
2508+
nc.createDimension("n_s", weights.size)
2509+
nc.createDimension("src_grid_rank", src_esmpy_grid.rank)
2510+
nc.createDimension("dst_grid_rank", dst_esmpy_grid.rank)
2511+
2512+
v = nc.createVariable(
2513+
"src_grid_dims", i_dtype, ("src_grid_rank",)
2514+
)
2515+
v.long_name = "Source grid shape"
2516+
v[...] = src_shape
2517+
2518+
v = nc.createVariable(
2519+
"dst_grid_dims", i_dtype, ("dst_grid_rank",)
2520+
)
2521+
v.long_name = "Destination grid shape"
2522+
v[...] = dst_shape
2523+
2524+
v = nc.createVariable("S", weights.dtype, ("n_s",))
2525+
v.long_name = "Weights values"
2526+
v[...] = weights
2527+
2528+
v = nc.createVariable("row", i_dtype, ("n_s",), zlib=True)
2529+
v.long_name = "Destination/row indices"
2530+
v.start_index = start_index
2531+
v[...] = row
2532+
2533+
v = nc.createVariable("col", i_dtype, ("n_s",), zlib=True)
2534+
v.long_name = "Source/col indices"
2535+
v.start_index = start_index
2536+
v[...] = col
2537+
2538+
nc.close()
25362539

25372540
if esmpy_regrid_operator is None:
25382541
# Destroy esmpy objects (the esmpy.Grid objects exist even if

‎cf/regrid/regridoperator.py‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -725,28 +725,26 @@ def tosparse(self):
725725
weights_file = self.weights_file
726726
if weights_file is not None:
727727
# Read the weights from the weights file
728+
from cfdm.data.locks import netcdf_lock
728729
from netCDF4 import Dataset
729730

730-
from ..data.array.locks import netcdf_lock
731+
with netcdf_lock:
732+
nc = Dataset(weights_file, "r")
733+
weights = nc.variables["S"][...]
734+
row = nc.variables["row"][...]
735+
col = nc.variables["col"][...]
731736

732-
netcdf_lock.acquire()
733-
nc = Dataset(weights_file, "r")
734-
weights = nc.variables["S"][...]
735-
row = nc.variables["row"][...]
736-
col = nc.variables["col"][...]
737+
try:
738+
col_start_index = nc.variables["col"].start_index
739+
except AttributeError:
740+
col_start_index = 1
737741

738-
try:
739-
col_start_index = nc.variables["col"].start_index
740-
except AttributeError:
741-
col_start_index = 1
742+
try:
743+
row_start_index = nc.variables["row"].start_index
744+
except AttributeError:
745+
row_start_index = 1
742746

743-
try:
744-
row_start_index = nc.variables["row"].start_index
745-
except AttributeError:
746-
row_start_index = 1
747-
748-
nc.close()
749-
netcdf_lock.release()
747+
nc.close()
750748
else:
751749
raise ValueError(
752750
"Conversion to sparse array format requires at least "

0 commit comments

Comments
 (0)