diff --git a/Changelog.rst b/Changelog.rst index 1bbaf8583d..40ef92fecc 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,10 +1,22 @@ Version NEXTVERSION ------------------- -**2025-12-??** - +**2026-01-??** + +* New function to control the creation of cached elements during data + display: `cf.display_data` + (https://github.com/NCAS-CMS/cf-python/issues/913) +* New methods: `cf.Data.get_cached_elements` + `cf.Data.cache_elements` + (https://github.com/NCAS-CMS/cf-python/issues/913) +* Set cached elements during `cf.Data.__init__` + (https://github.com/NCAS-CMS/cf-python/issues/913) +* Removed the `cf.constants.CONSTANTS` dictionary, replacing it + `cf.ConstantAccess.constants` + (https://github.com/NCAS-CMS/cf-python/issues/902) * Reduce the time taken to import `cf` (https://github.com/NCAS-CMS/cf-python/issues/902) +* Changed dependency: ``cfdm>=1.12.4.0, <1.12.5.0`` ---- diff --git a/cf/functions.py b/cf/functions.py index f63b037305..7fa8a3d08a 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -156,6 +156,7 @@ def configuration( tempdir=None, chunksize=None, log_level=None, + display_data=None, regrid_logging=None, relaxed_identities=None, bounds_combination_mode=None, @@ -177,6 +178,7 @@ def configuration( * `tempdir` * `chunksize` * `log_level` + * `display_data` * `regrid_logging` * `relaxed_identities` * `bounds_combination_mode` @@ -200,10 +202,10 @@ def configuration( .. versionadded:: 3.6.0 .. seealso:: `atol`, `rtol`, `tempdir`, `chunksize`, - `total_memory`, `log_level`, `regrid_logging`, - `relaxed_identities`, `bounds_combination_mode`, - `active_storage`, `active_storage_url`, - `active_storage_max_requests` + `total_memory`, `log_level`, `display_data`, + `regrid_logging`, `relaxed_identities`, + `bounds_combination_mode`, `active_storage`, + `active_storage_url`, `active_storage_max_requests` :Parameters: @@ -245,6 +247,12 @@ def configuration( * ``'DETAIL'`` (``3``); * ``'DEBUG'`` (``-1``). + display_data `bool` or `Constant`, optional + The new display data option. The default is to not change + the current behaviour. + + .. versionadded:: NEXTVERSION + regrid_logging: `bool` or `Constant`, optional The new value (either True to enable logging or False to disable it). The default is to not change the current @@ -303,6 +311,7 @@ def configuration( 'log_level': 'WARNING', 'bounds_combination_mode': 'AND', 'chunksize': 82873466.88000001, + 'display_data': True, 'active_storage': False, 'active_storage_url': None, 'active_storage_max_requests': 100} @@ -320,6 +329,7 @@ def configuration( 'log_level': 'WARNING', 'bounds_combination_mode': 'AND', 'chunksize': 75000000.0, + 'display_data': True, 'active_storage': False, 'active_storage_url': None, 'active_storage_max_requests': 100} @@ -347,6 +357,7 @@ def configuration( 'log_level': 'INFO', 'bounds_combination_mode': 'AND', 'chunksize': 75000000.0, + 'display_data': True, 'active_storage': False, 'active_storage_url': None} >>> with cf.configuration(atol=9, rtol=10): @@ -360,6 +371,7 @@ def configuration( 'log_level': 'INFO', 'bounds_combination_mode': 'AND', 'chunksize': 75000000.0, + 'display_data': True, 'active_storage': False, 'active_storage_url': None, 'active_storage_max_requests': 100} @@ -372,6 +384,7 @@ def configuration( 'log_level': 'INFO', 'bounds_combination_mode': 'AND', 'chunksize': 75000000.0, + 'display_data': True, 'active_storage': False, 'active_storage_url': None, 'active_storage_max_requests': 100} @@ -402,6 +415,7 @@ def configuration( new_tempdir=tempdir, new_chunksize=chunksize, new_log_level=log_level, + new_display_data=display_data, new_regrid_logging=regrid_logging, new_relaxed_identities=relaxed_identities, bounds_combination_mode=bounds_combination_mode, @@ -445,6 +459,7 @@ def _configuration(_Configuration, **kwargs): "new_tempdir": tempdir, "new_chunksize": chunksize, "new_log_level": log_level, + "new_display_data": display_data, "new_regrid_logging": regrid_logging, "new_relaxed_identities": relaxed_identities, "bounds_combination_mode": bounds_combination_mode, @@ -459,10 +474,6 @@ def _configuration(_Configuration, **kwargs): old = ConstantAccess.constants(copy=True) - # old = {name.lower(): val for name, val in CONSTANTS.items()} - # - # old.pop("total_memory", None) - # Filter out 'None' kwargs from configuration() defaults. Note that this # does not filter out '0' or 'True' values, which is important as the user # might be trying to set those, as opposed to None emerging as default. @@ -552,7 +563,6 @@ def FREE_MEMORY(): # Functions inherited from cfdm # -------------------------------------------------------------------- class ConstantAccess(cfdm.ConstantAccess): - _constants = {} _Constant = Constant def __docstring_substitutions__(self): @@ -576,6 +586,10 @@ class log_level(ConstantAccess, cfdm.log_level): _reset_log_emergence_level = _reset_log_emergence_level +class display_data(ConstantAccess, cfdm.display_data): + pass + + class regrid_logging(ConstantAccess): """Whether or not to enable `esmpy` regridding logging. diff --git a/cf/mixin/properties.py b/cf/mixin/properties.py index dd7d80fa86..326c286678 100644 --- a/cf/mixin/properties.py +++ b/cf/mixin/properties.py @@ -7,8 +7,6 @@ _DEPRECATION_ERROR_KWARGS, _DEPRECATION_ERROR_METHOD, ) -from ..functions import atol as cf_atol -from ..functions import rtol as cf_rtol from ..mixin_container import Container from ..query import Query from ..units import Units @@ -34,32 +32,6 @@ def __new__(cls, *args, **kwargs): instance._Data = Data return instance - # ---------------------------------------------------------------- - # Private attributes - # ---------------------------------------------------------------- - @property - def _atol(self): - """Return the tolerance on absolute differences between real - numbers, as returned by the `cf.atol` function. - - This is used by, for example, the `_equals` method. - - """ - return cf_atol().value - - @property - def _rtol(self): - """Return the tolerance on relative differences between real - numbers, as returned by the `cf.rtol` function. - - This is used by, for example, the `_equals` method. - - """ - return cf_rtol().value - - # ---------------------------------------------------------------- - # Private methods - # ---------------------------------------------------------------- def _matching_values(self, value0, value1, units=False, basic=False): """Whether two values match. @@ -100,9 +72,6 @@ def _matching_values(self, value0, value1, units=False, basic=False): return self._equals(value1, value0, basic=basic) - # ---------------------------------------------------------------- - # Attributes - # ---------------------------------------------------------------- @property def id(self): """An identity for the {{class}} object. @@ -150,9 +119,6 @@ def id(self): f"{self.__class__.__name__} doesn't have attribute 'id'" ) - # ---------------------------------------------------------------- - # CF properties - # ---------------------------------------------------------------- @property def calendar(self): """The calendar CF property. @@ -554,9 +520,6 @@ def valid_range(self, value): def valid_range(self): self.del_property("valid_range", default=AttributeError()) - # ---------------------------------------------------------------- - # Methods - # ---------------------------------------------------------------- def get_property(self, prop, default=ValueError()): """Get a CF property. diff --git a/cf/mixin2/container.py b/cf/mixin2/container.py index c5f0081462..44397301f4 100644 --- a/cf/mixin2/container.py +++ b/cf/mixin2/container.py @@ -6,7 +6,6 @@ """ from ..docstring import _docstring_substitution_definitions -from ..functions import atol, rtol class Container: @@ -55,23 +54,3 @@ def __docstring_package_depth__(self): """ return 0 - - @property - def _atol(self): - """Internal alias for `{{package}}.atol`. - - An alias is necessary to avoid a name clash with the keyword - argument of identical name (`atol`) in calling functions. - - """ - return atol().value - - @property - def _rtol(self): - """Internal alias for `{{package}}.rtol`. - - An alias is necessary to avoid a name clash with the keyword - argument of identical name (`rtol`) in calling functions. - - """ - return rtol().value diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 2300a62e47..21bdff254d 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -1165,7 +1165,7 @@ def test_Data_concatenate(self): str(d) str(e) f = cf.Data.concatenate([d, e], axis=1) - cached = f._get_cached_elements() + cached = f.get_cached_elements() self.assertEqual(cached[0], d.first_element()) self.assertEqual(cached[-1], e.last_element()) @@ -1205,7 +1205,7 @@ def test_Data_concatenate(self): repr(e) f = cf.Data.concatenate([d, e], axis=0) self.assertEqual( - f._get_cached_elements(), + f.get_cached_elements(), {0: d.first_element(), -1: e.last_element()}, ) @@ -1709,6 +1709,12 @@ def test_Data_array(self): d = cf.Data([["2000-12-3 12:00"]], "days since 2000-12-01", dt=True) self.assertEqual(d.array, 2.5) + # Cached values + d = cf.Data([1, 2]) + d._del_cached_elements() + d.array + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 2}) + def test_Data_binary_mask(self): """Test the `binary_mask` Data property.""" d = cf.Data([[0, 1, 2, 3.0]], "m") @@ -3221,6 +3227,12 @@ def test_Data_compute(self): d = cf.Data([["2000-12-3 12:00"]], "days since 2000-12-01", dt=True) self.assertEqual(d.compute(), 2.5) + # Cached values + d = cf.Data([1, 2]) + d._del_cached_elements() + d.compute() + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 2}) + def test_Data_persist(self): """Test Data.persist.""" d = cf.Data(9, "km") @@ -4376,9 +4388,8 @@ def test_Data_Units(self): # Adjusted cached values d = cf.Data([1000, 2000, 3000], "m") - repr(d) d.Units = cf.Units("km") - self.assertEqual(d._get_cached_elements(), {0: 1.0, 1: 2.0, -1: 3.0}) + self.assertEqual(d.get_cached_elements(), {0: 1.0, 1: 2.0, -1: 3.0}) def test_Data_get_data(self): """Test the `get_data` Data method.""" @@ -4450,53 +4461,24 @@ def test_Data__str__(self): """Test `Data.__str__`""" elements0 = (0, -1, 1) for array in ([1], [1, 2], [1, 2, 3]): - elements = elements0[: len(array)] - d = cf.Data(array) - cache = d._get_cached_elements() - for element in elements: - self.assertNotIn(element, cache) - - self.assertEqual(str(d), str(array)) - cache = d._get_cached_elements() - for element in elements: - self.assertIn(element, cache) - d[0] = 1 - cache = d._get_cached_elements() - for element in elements: - self.assertNotIn(element, cache) - self.assertEqual(str(d), str(array)) - cache = d._get_cached_elements() - for element in elements: - self.assertIn(element, cache) - d += 0 - cache = d._get_cached_elements() - for element in elements: - self.assertNotIn(element, cache) - self.assertEqual(str(d), str(array)) - cache = d._get_cached_elements() - for element in elements: - self.assertIn(element, cache) # Test when size > 3, i.e. second element is not there. d = cf.Data([1, 2, 3, 4]) - cache = d._get_cached_elements() - for element in elements0: - self.assertNotIn(element, cache) self.assertEqual(str(d), "[1, ..., 4]") - cache = d._get_cached_elements() + cache = d.get_cached_elements() self.assertNotIn(1, cache) for element in elements0[:2]: self.assertIn(element, cache) d[0] = 1 for element in elements0: - self.assertNotIn(element, d._get_cached_elements()) + self.assertNotIn(element, d.get_cached_elements()) def test_Data_cull_graph(self): """Test Data.cull_graph.""" @@ -4703,6 +4685,135 @@ def test_Data_to_units(self): with self.assertRaises(ValueError): e.to_units("degC") + def test_Data_cache_elements(self): + """Test setting of cached elements.""" + d = cf.Data(1) + self.assertEqual(d.get_cached_elements(), {0: 1, -1: 1}) + self.assertIsNone(d._del_cached_elements()) + self.assertFalse(d.get_cached_elements()) + + # Test via __init__, which calls `cache_elements` + for array in (np.ma.masked, True, "x"): + d = cf.Data(array) + for i in range(2): + self.assertEqual( + d.get_cached_elements(), {0: array, -1: array} + ) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in ( + 1, + 1.0, + np.array(1), + np.array([1]), + [1], + (1,), + ): + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, -1: 1}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in (np.array([1, 2]), [1, 2]): + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 2}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in (np.array([1, 2, 3]), (1, 2, 3)): + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 3}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in (np.array([1, 2, 3, 4]), (1, 2, 3, 4)): + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, -1: 4}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in (np.array([[1, 2]]), [[1, 2]]): + d = cf.Data(array) + for i in range(2): + self.assertEqual( + d.get_cached_elements(), {0: 1, 1: 2, -2: 1, -1: 2} + ) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in ( + np.array([[1, 2], [7, 8]]), + ([1, 2], [7, 8]), + np.array([[1, 2], [3, 4], [7, 8]]), + [[1, 2], [3, 4], [5, 6], [7, 8]], + ): + d = cf.Data(array) + for i in range(2): + self.assertEqual( + d.get_cached_elements(), {0: 1, 1: 2, -2: 7, -1: 8} + ) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + # Sparse array + from scipy.sparse import csr_array + + indptr = np.array([0, 2, 3, 6]) + indices = np.array([0, 2, 2, 0, 1, 2]) + data = np.array([1, 2, 3, 4, 5, 6]) + array = csr_array((data, indices, indptr), shape=(3, 3)) + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, -1: 6}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + # Check set_cached_elements + for array in ([1, 2, 3], [[1, 2, 3]]): + d = cf.Data(array) + d._del_cached_elements() + d.cache_elements() + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 3}) + + # Check that __str__ sets missing cached elements + d = cf.Data([[1, 2, 3]]) + d._del_cached_elements() + str(d) + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 3}) + + # Interaction with `cf.display_data` + d = cf.Data([[1, 2, 3]]) + d._del_cached_elements() + with cf.display_data(False): + self.assertEqual(repr(d), "") + + with cf.display_data(True): + self.assertEqual(repr(d), "") + + with cf.display_data(False): + self.assertEqual(repr(d), "") + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_functions.py b/cf/test/test_functions.py index 1a26a1afdc..4ca7750e5f 100644 --- a/cf/test/test_functions.py +++ b/cf/test/test_functions.py @@ -55,7 +55,7 @@ def test_configuration(self): self.assertIsInstance(org, dict) # Check all keys that should be there are, with correct value type: - self.assertEqual(len(org), 11) # update expected len if add new key(s) + self.assertEqual(len(org), 12) # update expected len if add new key(s) # Types expected: self.assertIsInstance(org["atol"], float) @@ -70,6 +70,7 @@ def test_configuration(self): # Log level may be input as an int but always given as # equiv. string self.assertIsInstance(org["log_level"], str) + self.assertIsInstance(org["display_data"], bool) # Store some sensible values to reset items to for testing, ensuring: # 1) they are kept different to the defaults (i.e. org values); and diff --git a/docs/source/recipes/plot_08_recipe.py b/docs/source/recipes/plot_08_recipe.py index 63427f62a7..6045f51448 100644 --- a/docs/source/recipes/plot_08_recipe.py +++ b/docs/source/recipes/plot_08_recipe.py @@ -9,11 +9,10 @@ # 1. Import cf-python, cf-plot, numpy and scipy.stats: import cfplot as cfp -import cf - import numpy as np import scipy.stats as stats +import cf # %% # 2. Three functions are defined: diff --git a/docs/source/recipes/plot_12_recipe.py b/docs/source/recipes/plot_12_recipe.py index b09db0b29f..5304194b19 100644 --- a/docs/source/recipes/plot_12_recipe.py +++ b/docs/source/recipes/plot_12_recipe.py @@ -13,8 +13,8 @@ # %% # 1. Import cf-python, cf-plot and matplotlib.pyplot: -import matplotlib.pyplot as plt import cfplot as cfp +import matplotlib.pyplot as plt import cf diff --git a/docs/source/recipes/plot_13_recipe.py b/docs/source/recipes/plot_13_recipe.py index bf0398713e..9b658597d8 100644 --- a/docs/source/recipes/plot_13_recipe.py +++ b/docs/source/recipes/plot_13_recipe.py @@ -18,13 +18,11 @@ # in next steps. import cartopy.crs as ccrs -import matplotlib.patches as mpatches - import cfplot as cfp +import matplotlib.patches as mpatches import cf - # %% # 2. Read and select the SST by index and look at its contents: sst = cf.read("~/recipes/ERA5_monthly_averaged_SST.nc")[0] diff --git a/docs/source/recipes/plot_17_recipe.py b/docs/source/recipes/plot_17_recipe.py index c94769e2ba..a66c90b518 100644 --- a/docs/source/recipes/plot_17_recipe.py +++ b/docs/source/recipes/plot_17_recipe.py @@ -11,8 +11,8 @@ # %% # 1. Import cf-python and cf-plot: -import matplotlib.pyplot as plt import cfplot as cfp +import matplotlib.pyplot as plt import cf diff --git a/docs/source/recipes/plot_18_recipe.py b/docs/source/recipes/plot_18_recipe.py index f0eae36e35..3beb9d0db9 100644 --- a/docs/source/recipes/plot_18_recipe.py +++ b/docs/source/recipes/plot_18_recipe.py @@ -10,15 +10,15 @@ """ +import cfplot as cfp + # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt import scipy.stats.mstats as mstats -import cfplot as cfp import cf - # %% # 2. Read the data in and unpack the Fields from FieldLists using indexing. # In our example We are investigating the influence of the land height on diff --git a/docs/source/recipes/plot_19_recipe.py b/docs/source/recipes/plot_19_recipe.py index 02d493dc21..ceb9db1c5c 100644 --- a/docs/source/recipes/plot_19_recipe.py +++ b/docs/source/recipes/plot_19_recipe.py @@ -9,10 +9,11 @@ maxima. """ +import cfplot as cfp + # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt -import cfplot as cfp import cf diff --git a/docs/source/recipes/plot_22_recipe.py b/docs/source/recipes/plot_22_recipe.py index 377313c899..fe329cda9d 100644 --- a/docs/source/recipes/plot_22_recipe.py +++ b/docs/source/recipes/plot_22_recipe.py @@ -11,10 +11,11 @@ # %% # 1. Import cf-python, Dask.array, NumPy, and Matplotlib: -import cf import dask.array as da -import numpy as np import matplotlib.pyplot as plt +import numpy as np + +import cf # %% # 2. Read the field constructs and load the wind speed component fields: diff --git a/docs/source/recipes/plot_23_recipe.py b/docs/source/recipes/plot_23_recipe.py index 2499b0d875..29537803af 100644 --- a/docs/source/recipes/plot_23_recipe.py +++ b/docs/source/recipes/plot_23_recipe.py @@ -18,12 +18,12 @@ # sphinx_gallery_thumbnail_number = 2 # sphinx_gallery_end_ignore -import matplotlib.pyplot as plt import cfplot as cfp -import cf - -import numpy as np import dask.array as da +import matplotlib.pyplot as plt +import numpy as np + +import cf # %% # 2. Read example data field constructs, and set region for our plots: