dev

Author: davidhassell

Changelog.rst

@@ -9,6 +9,9 @@ version NEXT
   to regrid the vertical axis in logarithmic coordinates to
   `cf.Field.regrids` and `cf.Field.regridc`
   (https://github.com/NCAS-CMS/cf-python/issues/715)
+* Reduce output CFA netCDF file size by setting the HDF5 chunksizes of
+  CFA variables to be no larger than required
+  (https://github.com/NCAS-CMS/cf-python/issues/739)
 * Fix misleading error message when it is not possible to create area
   weights requested from `cf.Field.collapse`
   (https://github.com/NCAS-CMS/cf-python/issues/731)

cf/read_write/netcdf/netcdfwrite.py

@@ -430,7 +430,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
 
             `None`
 
-        """        
+        """
         g = self.write_vars
 
         ndim = data.ndim
@@ -485,12 +485,12 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data ,#cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             location_ncdimensions,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
-        
+
         # File
         term = "file"
         if substitutions:
@@ -506,7 +506,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             fragment_ncdimensions,
             attributes=attributes,
@@ -527,7 +527,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #         cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             dimensions,
         )
@@ -547,7 +547,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             dimensions,
         )
@@ -819,7 +819,7 @@ def _cfa_write_non_standard_terms(
             data = type(data)(dx)
             self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
             term_ncvar = self._cfa_write_term_variable(
-                data=data, #type(data)(dx),
+                data=data,
                 ncvar=aggregated_data.get(term, f"cfa_{term}"),
                 ncdimensions=fragment_ncdimensions,
             )
@@ -893,10 +893,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         from os.path import abspath, join, relpath
         from pathlib import PurePath
         from urllib.parse import urlparse
-        import time # TODO
-        print (f"\n{cfvar!r}") # TODO
-        start = time.time() # TODO
-        
+
         g = self.write_vars
 
         # Define the CFA file susbstitutions, giving precedence over
@@ -911,15 +908,12 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # Size of the trailing dimension
         n_trailing = 0
 
-        start1 = time.time() # TODO
         aggregation_file = []
         aggregation_address = []
         aggregation_format = []
-        nnn = 0
         for indices in data.chunk_indices():
-            nnn += 1
             file_details = self._cfa_get_file_details(data[indices])
-       
+
             if len(file_details) != 1:
                 if file_details:
                     raise ValueError(
@@ -964,9 +958,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
             aggregation_file.append(tuple(filenames2))
             aggregation_address.append(addresses)
             aggregation_format.append(formats)
-        print ('len(data.chunk_indices()) =',nnn)
-        print (f"loop 1: {time.time() - start1:.3}")
-       
+
         # Pad each value of the aggregation instruction arrays so that
         # it has 'n_trailing' elements
         a_shape = data.numblocks
@@ -980,6 +972,8 @@ def _cfa_aggregation_instructions(self, data, cfvar):
             ):
                 n = n_trailing - len(filenames)
                 if n:
+                    # This chunk has fewer fragment files than some
+                    # others, so some padding is required.
                     pad = ("",) * n
                     aggregation_file[i] = filenames + pad
                     aggregation_format[i] = formats + pad
@@ -1023,7 +1017,6 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # Return Data objects
         # ------------------------------------------------------------
         data = type(data)
-        print (f"_cfa_aggregation_instructions: {time.time() - start:.3}")
         return {
             "location": data(aggregation_location),
             "file": data(aggregation_file),
@@ -1074,13 +1067,12 @@ def _cfa_get_file_details(self, data):
         {(('/home/file.pp',), (34556,), ('um',))}
 
         """
-        out = set()
+        out = []
+        append = out.append
         for a in data.todict().values():
             try:
-                out.update(
-                    ((a.get_filenames(), a.get_addresses(), a.get_formats()),)
-                )
+                append((a.get_filenames(), a.get_addresses(), a.get_formats()))
             except AttributeError:
                 pass
 
-        return out
+        return set(out)