From aca78e2ace6a4f521ecdde1b969de065f74ce8e8 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 31 Mar 2025 13:56:06 +0100 Subject: [PATCH 1/2] Fix bug that caused Data._axes to be incorrect after a call to cf.Field.collapse --- Changelog.rst | 3 +++ cf/data/utils.py | 16 +++++++++++++++- cf/test/test_Data.py | 27 +++++++++++++++++++++++++++ docs/source/recipes/plot_08_recipe.py | 3 +-- docs/source/recipes/plot_12_recipe.py | 2 +- docs/source/recipes/plot_13_recipe.py | 4 +--- docs/source/recipes/plot_17_recipe.py | 2 +- docs/source/recipes/plot_18_recipe.py | 4 ++-- docs/source/recipes/plot_19_recipe.py | 3 ++- 9 files changed, 53 insertions(+), 11 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 67108990b5..f0d5f95692 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -13,6 +13,9 @@ version 3.17.0 ``key`` (https://github.com/NCAS-CMS/cf-python/issues/802) * New keyword parameter to `cf.histogram`: ``density`` (https://github.com/NCAS-CMS/cf-python/issues/794) +* Fix bug that caused `Data._axes` to be incorrect after a call to + `cf.Field.collapse` + (https://github.com/NCAS-CMS/cf-python/issues/857) * Changed dependency: ``Python>=3.9.0`` * Changed dependency: ``numpy>=2.0.0`` * Changed dependency: ``cfdm>=1.12.0.0, <1.12.1.0`` diff --git a/cf/data/utils.py b/cf/data/utils.py index c1b1a63920..63b2a40e88 100644 --- a/cf/data/utils.py +++ b/cf/data/utils.py @@ -402,8 +402,14 @@ def collapse( weights, axis)``. """ + original_size = d.size + if axis is None: + axis = range(d.ndim) + else: + axis = d._parse_axes(axis) + kwargs = { - "axis": axis, + "axis": tuple(axis), "keepdims": keepdims, "split_every": split_every, "mtol": mtol, @@ -424,6 +430,14 @@ def collapse( dx = func(dx, **kwargs) d._set_dask(dx) + if not keepdims: + # Remove collapsed axis names + d._axes = [a for i, a in enumerate(d._axes) if i not in axis] + + if d.size != original_size: + # Remove the out-dated HDF5 chunking strategy + d.nc_clear_hdf5_chunksizes() + return d, weights diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index b25342afce..6692737993 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4657,6 +4657,33 @@ def test_Data_is_masked(self): self.assertTrue(d[0].is_masked) self.assertFalse(d[1].is_masked) + def test_Data_collapse_axes_hdf_chunks(self): + """Test that _axes and hdf_chunks are updated after a collapse.""" + d = cf.Data([[1, 2, 3, 4]]) + chunks = d.shape + d.nc_set_hdf5_chunksizes(chunks) + e = d.mean(axes=1) + self.assertEqual(d._axes, ("dim0", "dim1")) + self.assertEqual(d.nc_hdf5_chunksizes(), chunks) + + e = d.mean(axes=1) + self.assertNotEqual(e.size, d.size) + self.assertEqual(e._axes, d._axes) + self.assertEqual(e.nc_hdf5_chunksizes(), None) + + e = d.mean(axes=1, squeeze=True) + self.assertEqual(e._axes, d._axes[:1]) + self.assertEqual(e.nc_hdf5_chunksizes(), None) + + e = d.mean(axes=0) + self.assertEqual(e.size, d.size) + self.assertEqual(e._axes, d._axes) + self.assertEqual(e.nc_hdf5_chunksizes(), chunks) + + e = d.mean(axes=0, squeeze=True) + self.assertEqual(e._axes, d._axes[1:]) + self.assertEqual(e.nc_hdf5_chunksizes(), chunks) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/docs/source/recipes/plot_08_recipe.py b/docs/source/recipes/plot_08_recipe.py index 63427f62a7..6045f51448 100644 --- a/docs/source/recipes/plot_08_recipe.py +++ b/docs/source/recipes/plot_08_recipe.py @@ -9,11 +9,10 @@ # 1. Import cf-python, cf-plot, numpy and scipy.stats: import cfplot as cfp -import cf - import numpy as np import scipy.stats as stats +import cf # %% # 2. Three functions are defined: diff --git a/docs/source/recipes/plot_12_recipe.py b/docs/source/recipes/plot_12_recipe.py index b09db0b29f..5304194b19 100644 --- a/docs/source/recipes/plot_12_recipe.py +++ b/docs/source/recipes/plot_12_recipe.py @@ -13,8 +13,8 @@ # %% # 1. Import cf-python, cf-plot and matplotlib.pyplot: -import matplotlib.pyplot as plt import cfplot as cfp +import matplotlib.pyplot as plt import cf diff --git a/docs/source/recipes/plot_13_recipe.py b/docs/source/recipes/plot_13_recipe.py index bf0398713e..9b658597d8 100644 --- a/docs/source/recipes/plot_13_recipe.py +++ b/docs/source/recipes/plot_13_recipe.py @@ -18,13 +18,11 @@ # in next steps. import cartopy.crs as ccrs -import matplotlib.patches as mpatches - import cfplot as cfp +import matplotlib.patches as mpatches import cf - # %% # 2. Read and select the SST by index and look at its contents: sst = cf.read("~/recipes/ERA5_monthly_averaged_SST.nc")[0] diff --git a/docs/source/recipes/plot_17_recipe.py b/docs/source/recipes/plot_17_recipe.py index bc6c8fba14..656f7c8717 100644 --- a/docs/source/recipes/plot_17_recipe.py +++ b/docs/source/recipes/plot_17_recipe.py @@ -11,8 +11,8 @@ # %% # 1. Import cf-python and cf-plot: -import matplotlib.pyplot as plt import cfplot as cfp +import matplotlib.pyplot as plt import cf diff --git a/docs/source/recipes/plot_18_recipe.py b/docs/source/recipes/plot_18_recipe.py index 2e7d97d7d4..4744e868af 100644 --- a/docs/source/recipes/plot_18_recipe.py +++ b/docs/source/recipes/plot_18_recipe.py @@ -10,15 +10,15 @@ """ +import cfplot as cfp + # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt import scipy.stats.mstats as mstats -import cfplot as cfp import cf - # %% # 2. Read the data in and unpack the Fields from FieldLists using indexing. # In our example We are investigating the influence of the land height on diff --git a/docs/source/recipes/plot_19_recipe.py b/docs/source/recipes/plot_19_recipe.py index 6cd37dbc7a..0e28eb44d1 100644 --- a/docs/source/recipes/plot_19_recipe.py +++ b/docs/source/recipes/plot_19_recipe.py @@ -9,10 +9,11 @@ maxima. """ +import cfplot as cfp + # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt -import cfplot as cfp import cf From c6504e00083dc891f13d0658b2e9b510a70f7e8a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 31 Mar 2025 14:02:47 +0100 Subject: [PATCH 2/2] tidy --- docs/source/recipes/plot_08_recipe.py | 3 ++- docs/source/recipes/plot_12_recipe.py | 2 +- docs/source/recipes/plot_13_recipe.py | 4 +++- docs/source/recipes/plot_17_recipe.py | 2 +- docs/source/recipes/plot_18_recipe.py | 4 ++-- docs/source/recipes/plot_19_recipe.py | 3 +-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/source/recipes/plot_08_recipe.py b/docs/source/recipes/plot_08_recipe.py index 6045f51448..63427f62a7 100644 --- a/docs/source/recipes/plot_08_recipe.py +++ b/docs/source/recipes/plot_08_recipe.py @@ -9,10 +9,11 @@ # 1. Import cf-python, cf-plot, numpy and scipy.stats: import cfplot as cfp +import cf + import numpy as np import scipy.stats as stats -import cf # %% # 2. Three functions are defined: diff --git a/docs/source/recipes/plot_12_recipe.py b/docs/source/recipes/plot_12_recipe.py index 5304194b19..b09db0b29f 100644 --- a/docs/source/recipes/plot_12_recipe.py +++ b/docs/source/recipes/plot_12_recipe.py @@ -13,8 +13,8 @@ # %% # 1. Import cf-python, cf-plot and matplotlib.pyplot: -import cfplot as cfp import matplotlib.pyplot as plt +import cfplot as cfp import cf diff --git a/docs/source/recipes/plot_13_recipe.py b/docs/source/recipes/plot_13_recipe.py index 9b658597d8..bf0398713e 100644 --- a/docs/source/recipes/plot_13_recipe.py +++ b/docs/source/recipes/plot_13_recipe.py @@ -18,11 +18,13 @@ # in next steps. import cartopy.crs as ccrs -import cfplot as cfp import matplotlib.patches as mpatches +import cfplot as cfp + import cf + # %% # 2. Read and select the SST by index and look at its contents: sst = cf.read("~/recipes/ERA5_monthly_averaged_SST.nc")[0] diff --git a/docs/source/recipes/plot_17_recipe.py b/docs/source/recipes/plot_17_recipe.py index 656f7c8717..bc6c8fba14 100644 --- a/docs/source/recipes/plot_17_recipe.py +++ b/docs/source/recipes/plot_17_recipe.py @@ -11,8 +11,8 @@ # %% # 1. Import cf-python and cf-plot: -import cfplot as cfp import matplotlib.pyplot as plt +import cfplot as cfp import cf diff --git a/docs/source/recipes/plot_18_recipe.py b/docs/source/recipes/plot_18_recipe.py index 4744e868af..2e7d97d7d4 100644 --- a/docs/source/recipes/plot_18_recipe.py +++ b/docs/source/recipes/plot_18_recipe.py @@ -10,15 +10,15 @@ """ -import cfplot as cfp - # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt import scipy.stats.mstats as mstats +import cfplot as cfp import cf + # %% # 2. Read the data in and unpack the Fields from FieldLists using indexing. # In our example We are investigating the influence of the land height on diff --git a/docs/source/recipes/plot_19_recipe.py b/docs/source/recipes/plot_19_recipe.py index 0e28eb44d1..6cd37dbc7a 100644 --- a/docs/source/recipes/plot_19_recipe.py +++ b/docs/source/recipes/plot_19_recipe.py @@ -9,11 +9,10 @@ maxima. """ -import cfplot as cfp - # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt +import cfplot as cfp import cf