Merge PR #395 (Remove xr.Dataset.load() commands to improve performance)

This merge brings PR #395 (Remove xr.Dataset.load() commands to improve
performance, by @yantosca) into the GCPy 1.7.0 development stream.

PR #395 removes the .load() commands that are appended to the end of
xarray Dataset objects.  This will load the entire Dataset into memory
but is not necessary, as xarray can handle the memory management of
large files "under the hood".

Signed-off-by: Bob Yantosca <yantosca@seas.harvard.edu>

Author: yantosca

CHANGELOG.md

@@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Removed
 - Removed `PdfMerger()` from `compare_single_level` and `compare_zonal_mean`, it has been removed in pypdf >= 5.0.0
+- Removed `.load()` statements from xarray Datasets to improve performance
 
 ## [1.6.2] - 2025-06-12
 ### Added

gcpy/append_grid_corners.py

@@ -46,7 +46,6 @@
     })
 
     # Write to netCDF
-    ds.load()
     ds.close()
     ds.to_netcdf(args.filein)
 

gcpy/benchmark/modules/benchmark_funcs.py

@@ -1054,8 +1054,8 @@ def make_benchmark_conc_plots(
     reader = util.dataset_reader(time_mean, verbose=verbose)
 
     # Open datasets
-    refds = reader(ref, drop_variables=SKIP_THESE_VARS).load()
-    devds = reader(dev, drop_variables=SKIP_THESE_VARS).load()
+    refds = reader(ref, drop_variables=SKIP_THESE_VARS)
+    devds = reader(dev, drop_variables=SKIP_THESE_VARS)
 
     # Rename SpeciesConc_ to SpeciesConcVV_ for consistency with new
     # naming introduced in GEOS-Chem 14.1.0
@@ -1080,9 +1080,9 @@ def make_benchmark_conc_plots(
     refmetds = None
     devmetds = None
     if refmet:
-        refmetds = reader(refmet, drop_variables=SKIP_THESE_VARS).load()
+        refmetds = reader(refmet, drop_variables=SKIP_THESE_VARS)
     if devmet:
-        devmetds = reader(devmet, drop_variables=SKIP_THESE_VARS).load()
+        devmetds = reader(devmet, drop_variables=SKIP_THESE_VARS)
 
     # Determine if doing diff-of-diffs
     diff_of_diffs = False
@@ -1093,8 +1093,8 @@ def make_benchmark_conc_plots(
     # Open second datasets if passed as arguments (used for diff of diffs)
     # Regrid to same horz grid resolution if two refs or two devs do not match.
     if diff_of_diffs:
-        second_refds = reader(second_ref, drop_variables=SKIP_THESE_VARS).load()
-        second_devds = reader(second_dev, drop_variables=SKIP_THESE_VARS).load()
+        second_refds = reader(second_ref, drop_variables=SKIP_THESE_VARS)
+        second_devds = reader(second_dev, drop_variables=SKIP_THESE_VARS)
 
         print('\nPrinting second_refds (dev of ref for diff-of-diffs)\n')
         print(second_refds)
@@ -5695,7 +5695,7 @@ def create_benchmark_summary_table(
                 is_gchp=ref_gchp
             ),
             drop_variables=skip_vars
-        ).load()
+        )
 
         # Get Dev data
         devdata = reader(
@@ -5706,7 +5706,7 @@ def create_benchmark_summary_table(
                 is_gchp=dev_gchp
             ),
             drop_variables=skip_vars
-        ).load()
+        )
 
         # Make sure that Ref and Dev datasets have the same variables.
         # Variables that are in Ref but not in Dev will be added to Dev
@@ -5962,7 +5962,7 @@ def create_benchmark_sanity_check_table(
             dset = reader(
                 file_name,
                 drop_variables=skip_vars
-            ).load()
+            )
 
             # Determine which variables are all zeroes or NaN
             all_zeros_or_nans = []

gcpy/benchmark/modules/benchmark_mass_cons_table.py

@@ -44,7 +44,7 @@ def get_area(
     # Otherwise read the data from the supplied area_path)
     reader = dataset_reader(multi_files=False, verbose=False)
     return get_area_from_dataset(
-        reader(area_path, drop_variables=SKIP_THESE_VARS).load()
+        reader(area_path, drop_variables=SKIP_THESE_VARS)
     )
 
 
@@ -371,11 +371,11 @@ def make_benchmark_mass_conservation_table(
             ref_data = reader(
                 ref_files[t_idx],
                 drop_variables=SKIP_THESE_VARS
-            ).load()
+            )
             dev_data = reader(
                 dev_files[t_idx],
                 drop_variables=SKIP_THESE_VARS
-            ).load()
+            )
             ref_area = get_area(ref_areapath, ref_data)
             dev_area = get_area(dev_areapath, dev_data)
             ref_delta_prs = get_delta_pressure(ref_data)

gcpy/benchmark/modules/benchmark_models_vs_obs.py

@@ -208,7 +208,7 @@ def read_model_data(
 
     # Read data and rename SpeciesConc_ to SpeciesConcVV_, if necessary
     # (needed for backwards compatibility with older versions.)
-    dataset = reader(filepaths,drop_variables=SKIP_THESE_VARS).load()
+    dataset = reader(filepaths,drop_variables=SKIP_THESE_VARS)
     dataset = rename_speciesconc_to_speciesconcvv(dataset)
 
     # Create a DataArray object and convert to ppbv (if necessary)

gcpy/examples/diagnostics/compare_diags.py

@@ -91,7 +91,7 @@ def read_data(config):
         refdata = reader(
             ref_file,
             drop_variables=SKIP_THESE_VARS
-        ).load()
+        )
     except FileNotFoundError as exc:
         msg = "Error reading " + ref_file
         raise FileNotFoundError(msg) from exc
@@ -101,7 +101,7 @@ def read_data(config):
         devdata = reader(
             dev_file,
             drop_variables=SKIP_THESE_VARS
-        ).load()
+        )
     except FileNotFoundError as exc:
         msg = "Error reading " + dev_file
         raise FileNotFoundError(msg) from exc

gcpy/file_regrid.py

@@ -117,7 +117,7 @@ def file_regrid(
         filein,
         decode_cf=False,
         engine="netcdf4"
-    ).load()
+    )
     cs_res_in = get_cubed_sphere_res(dset)
 
     # Verbose printout of inputs

gcpy/plot/compare_single_level.py

@@ -202,8 +202,6 @@ def compare_single_level(
 
     # Prepare diff-of-diffs datasets if needed
     if diff_of_diffs:
-        refdata, devdata = refdata.load(), devdata.load()
-        second_ref, second_dev = second_ref.load(), second_dev.load()
 
 #        # If needed, use fake time dim in case dates are different
 #        # in datasets.  This needs more work for case of single versus

gcpy/plot/compare_zonal_mean.py

@@ -209,8 +209,6 @@ def compare_zonal_mean(
 
     # Prepare diff-of-diffs datasets if needed
     if diff_of_diffs:
-        refdata, devdata = refdata.load(), devdata.load()
-        second_ref, second_dev = second_ref.load(), second_dev.load()
 
 #        # If needed, use fake time dim in case dates are different in datasets.
 #        # This needs more work for case of single versus multiple times.