Removed xr.Dataset.load commands to improve performance

append_grid_corners.py
file_regrid.py
plot/compare_single_level.py
plot/compare_zonal_mean.py
benchmark/modules/benchmark_funcs.py
benchmark/modules/benchmark_mass_cons_table.py
benchmark/modules/benchmark_models_vs_obs.py
examples/diagnostics/compare_diags.py
- Removed .load() commands for xarray Dataset objects.  We can let
  xarray decide when to load data into memory.  This should speed up
  data I/O significantly.

CHANGELOG.md
- Updated accordingly

Signed-off-by: Bob Yantosca <yantosca@seas.harvard.edu>

Author: yantosca

CHANGELOG.md

@@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Removed
 - Removed `PdfMerger()` from `compare_single_level` and `compare_zonal_mean`, it has been removed in pypdf >= 5.0.0
+- Removed `.load()` statements from xarray Datasets to improve performance
 
 ## [1.6.2] - 2025-06-12
 ### Added

gcpy/append_grid_corners.py

@@ -46,7 +46,6 @@
     })
 
     # Write to netCDF
-    ds.load()
     ds.close()
     ds.to_netcdf(args.filein)
 

gcpy/benchmark/modules/benchmark_funcs.py

@@ -1054,8 +1054,8 @@ def make_benchmark_conc_plots(
     reader = util.dataset_reader(time_mean, verbose=verbose)
 
     # Open datasets
-    refds = reader(ref, drop_variables=SKIP_THESE_VARS).load()
-    devds = reader(dev, drop_variables=SKIP_THESE_VARS).load()
+    refds = reader(ref, drop_variables=SKIP_THESE_VARS)
+    devds = reader(dev, drop_variables=SKIP_THESE_VARS)
 
     # Rename SpeciesConc_ to SpeciesConcVV_ for consistency with new
     # naming introduced in GEOS-Chem 14.1.0
@@ -1080,9 +1080,9 @@ def make_benchmark_conc_plots(
     refmetds = None
     devmetds = None
     if refmet:
-        refmetds = reader(refmet, drop_variables=SKIP_THESE_VARS).load()
+        refmetds = reader(refmet, drop_variables=SKIP_THESE_VARS)
     if devmet:
-        devmetds = reader(devmet, drop_variables=SKIP_THESE_VARS).load()
+        devmetds = reader(devmet, drop_variables=SKIP_THESE_VARS)
 
     # Determine if doing diff-of-diffs
     diff_of_diffs = False
@@ -1093,8 +1093,8 @@ def make_benchmark_conc_plots(
     # Open second datasets if passed as arguments (used for diff of diffs)
     # Regrid to same horz grid resolution if two refs or two devs do not match.
     if diff_of_diffs:
-        second_refds = reader(second_ref, drop_variables=SKIP_THESE_VARS).load()
-        second_devds = reader(second_dev, drop_variables=SKIP_THESE_VARS).load()
+        second_refds = reader(second_ref, drop_variables=SKIP_THESE_VARS)
+        second_devds = reader(second_dev, drop_variables=SKIP_THESE_VARS)
 
         print('\nPrinting second_refds (dev of ref for diff-of-diffs)\n')
         print(second_refds)
@@ -5695,7 +5695,7 @@ def create_benchmark_summary_table(
                 is_gchp=ref_gchp
             ),
             drop_variables=skip_vars
-        ).load()
+        )
 
         # Get Dev data
         devdata = reader(
@@ -5706,7 +5706,7 @@ def create_benchmark_summary_table(
                 is_gchp=dev_gchp
             ),
             drop_variables=skip_vars
-        ).load()
+        )
 
         # Make sure that Ref and Dev datasets have the same variables.
         # Variables that are in Ref but not in Dev will be added to Dev
@@ -5962,7 +5962,7 @@ def create_benchmark_sanity_check_table(
             dset = reader(
                 file_name,
                 drop_variables=skip_vars
-            ).load()
+            )
 
             # Determine which variables are all zeroes or NaN
             all_zeros_or_nans = []

gcpy/benchmark/modules/benchmark_mass_cons_table.py

@@ -44,7 +44,7 @@ def get_area(
     # Otherwise read the data from the supplied area_path)
     reader = dataset_reader(multi_files=False, verbose=False)
     return get_area_from_dataset(
-        reader(area_path, drop_variables=SKIP_THESE_VARS).load()
+        reader(area_path, drop_variables=SKIP_THESE_VARS)
     )
 
 
@@ -371,11 +371,11 @@ def make_benchmark_mass_conservation_table(
             ref_data = reader(
                 ref_files[t_idx],
                 drop_variables=SKIP_THESE_VARS
-            ).load()
+            )
             dev_data = reader(
                 dev_files[t_idx],
                 drop_variables=SKIP_THESE_VARS
-            ).load()
+            )
             ref_area = get_area(ref_areapath, ref_data)
             dev_area = get_area(dev_areapath, dev_data)
             ref_delta_prs = get_delta_pressure(ref_data)

gcpy/benchmark/modules/benchmark_models_vs_obs.py

@@ -208,7 +208,7 @@ def read_model_data(
 
     # Read data and rename SpeciesConc_ to SpeciesConcVV_, if necessary
     # (needed for backwards compatibility with older versions.)
-    dataset = reader(filepaths,drop_variables=SKIP_THESE_VARS).load()
+    dataset = reader(filepaths,drop_variables=SKIP_THESE_VARS)
     dataset = rename_speciesconc_to_speciesconcvv(dataset)
 
     # Create a DataArray object and convert to ppbv (if necessary)

gcpy/examples/diagnostics/compare_diags.py

@@ -91,7 +91,7 @@ def read_data(config):
         refdata = reader(
             ref_file,
             drop_variables=SKIP_THESE_VARS
-        ).load()
+        )
     except FileNotFoundError as exc:
         msg = "Error reading " + ref_file
         raise FileNotFoundError(msg) from exc
@@ -101,7 +101,7 @@ def read_data(config):
         devdata = reader(
             dev_file,
             drop_variables=SKIP_THESE_VARS
-        ).load()
+        )
     except FileNotFoundError as exc:
         msg = "Error reading " + dev_file
         raise FileNotFoundError(msg) from exc

gcpy/file_regrid.py

@@ -117,7 +117,7 @@ def file_regrid(
         filein,
         decode_cf=False,
         engine="netcdf4"
-    ).load()
+    )
     cs_res_in = get_cubed_sphere_res(dset)
 
     # Verbose printout of inputs

gcpy/plot/compare_single_level.py

@@ -202,8 +202,6 @@ def compare_single_level(
 
     # Prepare diff-of-diffs datasets if needed
     if diff_of_diffs:
-        refdata, devdata = refdata.load(), devdata.load()
-        second_ref, second_dev = second_ref.load(), second_dev.load()
 
 #        # If needed, use fake time dim in case dates are different
 #        # in datasets.  This needs more work for case of single versus

gcpy/plot/compare_zonal_mean.py

@@ -209,8 +209,6 @@ def compare_zonal_mean(
 
     # Prepare diff-of-diffs datasets if needed
     if diff_of_diffs:
-        refdata, devdata = refdata.load(), devdata.load()
-        second_ref, second_dev = second_ref.load(), second_dev.load()
 
 #        # If needed, use fake time dim in case dates are different in datasets.
 #        # This needs more work for case of single versus multiple times.