Add LIVVkit fixes and test setup

Author: forsyth2

docs/source/post.v3.livvkit.cfg examples/post.v3.livvkit.cfgdocs/source/post.v3.livvkit.cfg renamed to examples/post.v3.livvkit.cfg

@@ -1,10 +1,3 @@
-# This is an example zppy v3.1.0 configuration file for Perlmutter.
-# zppy v3.1.0 is included in E3SM Unified 1.12.
-#
-# NOTE: This configuration generates 8 tasks/jobs when fully enabled.
-# Users should only enable the tasks and time periods they need based on their specific
-# analysis requirements to reduce computational load and queue time.
-
 [default]
 # Change these two paths to include your username instead
 # You can also change these to match your preferred directory structure
@@ -13,102 +6,71 @@ www = /global/cfs/cdirs/e3sm/www/mek/zppy_out
 # Other paths
 input = /pscratch/sd/m/mek/e3sm_scratch/pm-cpu/archive/v2.1.r05.BGWCYCL20TR-steve_2005_2014
 
-# zppy auto-determines and sources latest e3sm-unified on each supported machine with environment_commands commented out.
-#environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"
-# For testing Unified rc:
-#environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
-environment_commands = "source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh"
-
-# Other parameters:
+account = e3sm
 campaign = "water_cycle"
 case = v2.1.r05.BGWCYCL20TR-steve_2005_2014
-account = e3sm
 dry_run = False
+environment_commands = "source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh"
 
 [climo]
 active = True
+climo_jobs = 12
+frequency = "monthly"
+input_files = "elm.h0"
+input_subdir = lnd/hist
+nodes = 1
 walltime = "0:10:00"
 years = "2005:2014:10"
-nodes = 1
 
   [[ land_monthly_climo ]]
-  frequency = "monthly"
-  climo_jobs = 12
-  input_files = "elm.h0"
-  input_subdir = lnd/hist
   mapping_file = /global/cfs/cdirs/e3sm/zender/maps/map_r05_to_cmip6_180x360_traave.20240901.nc
   vars = "" # Setting this as "" will tell zppy to use ALL variables
 
   [[ land_monthly_climo_native ]]
-  frequency = "monthly"
-  climo_jobs = 12
-  input_files = "elm.h0"
-  input_subdir = lnd/hist
   vars = ""
 
   [[ land_monthly_climo_racmo_gis ]]
-  frequency = "monthly"
-  climo_jobs = 12
   climo_subsection = "racmo_gis"
   grid = "racmo_gis"
-  input_files = "elm.h0"
-  input_subdir = lnd/hist
   mapping_file = /global/cfs/cdirs/e3sm/zender/maps/map_r05_to_racmo_gis_566x438_traave.20240801.nc
   vars = ""
 
   [[ land_monthly_climo_racmo_ais ]]
-  frequency = "monthly"
-  climo_jobs = 12
-  input_files = "elm.h0"
   climo_subsection = "racmo_ais"
   grid = "racmo_ais"
-  input_subdir = lnd/hist
   mapping_file = /global/cfs/cdirs/e3sm/livvkit/maps/map_r05_to_racmo_ais_591x726.nc
   vars = ""
 
   [[ land_monthly_climo_merra2 ]]
-  frequency = "monthly"
-  climo_jobs = 12
-  input_files = "elm.h0"
   climo_subsection = "merra2"
   grid = "merra2"
-  input_subdir = lnd/hist
   mapping_file = /global/cfs/cdirs/e3sm/livvkit/maps/map_r05_to_merra2_traave.20250124.nc
   vars = ""
 
   [[ land_monthly_climo_era5 ]]
-  frequency = "monthly"
-  climo_jobs = 12
-  input_files = "elm.h0"
   climo_subsection = "era5"
   grid = "era5"
-  input_subdir = lnd/hist
   mapping_file = /global/cfs/cdirs/e3sm/livvkit/maps/map_r05_to_era5_s2n_721x1440.traave.20240124.nc
   vars = ""
 
-
-[livvkit]
-active = True
-nodes = 1
-walltime = "0:30:00"
-# qos = "debug"
-icesheets = "gis,ais"
-sets = "cmb,smb,energy_racmo,energy_era5,energy_merra2,energy_ceres"
-cfg = "inclusions/livvkit/livvkit_r05.jinja"
-years = "2005:2014:10"
-
 [ts]
-# 2 DEPRECATED PARAMETERS in zppy v3.0.0 (the new e3sm_to_cmip task renders these obsolete):
-# e3sm_to_cmip_environment_commands
-# ts_fmt (e.g., `ts_fmt = "cmip"`)
 active = True
+frequency = "monthly"
+input_files = "elm.h0"
+input_subdir = "lnd/hist"
 walltime = "00:05:00"
 years = "2005:2014:10",
 
   [[ land_monthly ]]
-  # This subtask is a dependency for the e3sm_to_cmip task's land_monthly subtask.
-  frequency = "monthly"
-  input_files = "elm.h0"
-  input_subdir = "lnd/hist"
-  # Variables:
   vars = "EFLX_LH_TOT,FIRA,FLDS,FSA,FSDS,FSRND,FSRVD,FSDSND,FSDSVD,FSH,QICE,QICE_MELT,QRUNOFF,QSNOFRZ,QSNOMELT,QSOIL,RAIN,SNOW,TSA"
+
+[livvkit]
+active = True
+cfg = "inclusions/livvkit/livvkit_r05.jinja"
+# Once LIVVkit is included in E3SM Unified, this won't be necessary:
+environment_commands = "source /global/cfs/projectdirs/e3sm/livvkit/envs/load_e3sm_unified_pm-cpu_lex_test.sh"
+icesheets = "gis,ais"
+nodes = 1
+sets = "cmb,smb,energy_racmo,energy_era5,energy_merra2,energy_ceres"
+walltime = "0:30:00"
+years = "2005:2014:10"

pyproject.toml

@@ -56,6 +56,7 @@ zppy = [
   "**/*.ini",
   "**/*.sh",
   "**/*.json",
+  "**/*.jinja",
 ]
 
 [tool.black]

setup.py

@@ -18,7 +18,7 @@ def package_files(directory, prefixes, extensions):
 
 
 data_files = package_files(
-    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json"]
+    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json", "jinja"]
 )
 
 setup(

tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg

@@ -17,10 +17,10 @@ years = "1985:1989:2",
 
 [climo]
 active = True
+frequency = "monthly"
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   vars = ""
@@ -34,12 +34,72 @@ walltime = "00:30:00"
 
   [[ land_monthly_climo ]]
   active = True
-  frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
   mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
   vars = ""
 
+  [[ land_monthly_180x360_traave ]]
+   active = True
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = "/lcrc/group/e3sm/ac.forsyth2/maps/map_r05_to_cmip6_180x360_traave.20240901.nc"
+  vars = ""
+  years = "1985:1994:10",
+
+  [[ land_monthly_climo_native ]]
+  active = True
+  climo_jobs = 12
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = ""
+  vars = ""
+  years = "1985:1994:10"
+
+  [[ land_monthly_climo_racmo_gis ]]
+  active = True
+  climo_jobs = 12
+  climo_subsection = "racmo_gis"
+  grid = "racmo_gis"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = /lcrc/group/e3sm/ac.forsyth2/maps/map_r05_to_racmo_gis_566x438_traave.20240801.nc
+  vars = ""
+  years = "1985:1994:10"
+
+  [[ land_monthly_climo_racmo_ais ]]
+  active = True
+  climo_jobs = 12
+  input_files = "elm.h0"
+  climo_subsection = "racmo_ais"
+  grid = "racmo_ais"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = /lcrc/group/e3sm/public_html/diagnostics/maps/map_r05_to_racmo_ais_591x726.nc
+  vars = ""
+  years = "1985:1994:10"
+
+  [[ land_monthly_climo_merra2 ]]
+  active = True
+  climo_jobs = 12
+  input_files = "elm.h0"
+  climo_subsection = "merra2"
+  grid = "merra2"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = /lcrc/group/e3sm/public_html/diagnostics/maps/map_r05_to_merra2_traave.20250124.nc
+  vars = ""
+  years = "1985:1994:10"
+
+  [[ land_monthly_climo_era5 ]]
+  active = True
+  climo_jobs = 12
+  input_files = "elm.h0"
+  climo_subsection = "era5"
+  grid = "era5"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = /lcrc/group/e3sm/public_html/diagnostics/maps/map_r05_to_era5_s2n_721x1440.traave.20240124.nc
+  vars = ""
+  years = "1985:1994:10"
+
 [ts]
 active = True
 walltime = "00:30:00"
@@ -92,6 +152,25 @@ walltime = "00:30:00"
   mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
+  [[ land_monthly_energy ]]
+  active = True
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = ""
+  vars = "EFLX_LH_TOT,FIRA,FLDS,FSA,FSDS,FSRND,FSRVD,FSDSND,FSDSVD,FSH,TSA"
+  years = "1985:1994:10"
+
+  [[ land_monthly_smb ]]
+  # The data necessary for this is not found in the data input path.
+  active = False
+  frequency = "monthly"
+  input_files = "elm.h0"
+  input_subdir = "archive/lnd/hist"
+  mapping_file = ""
+  vars = "QICE,QICE_MELT,QRUNOFF,QSNOFRZ,QSNOMELT,QSOIL,RAIN,SNOW"
+  years = "1985:1994:10"
+
 [e3sm_to_cmip]
 active = True
 frequency = "monthly"
@@ -312,6 +391,20 @@ ts_num_years = 2
 walltime = "2:00:00"
 years = "1985:1989:4"
 
+[livvkit]
+active = True
+climo_subsections = "land_monthly_180x360_traave","land_monthly_climo_native","land_monthly_climo_racmo_gis","land_monthly_climo_racmo_ais","land_monthly_climo_merra2","land_monthly_climo_era5"
+cfg = "inclusions/livvkit/livvkit_r05.jinja"
+debug = True
+environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+icesheets = "gis,ais"
+nodes = 1
+sets = "energy_racmo,energy_era5,energy_merra2,energy_ceres"
+ts_land_subsection = "land_monthly_energy"
+walltime = "0:30:00"
+years = "1985:1994:10"
+
+
 [pcmdi_diags]
 active = True
 environment_commands_secondary = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"