diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 9fa2ce6f2c3..308a8585f20 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -46,6 +46,7 @@ body: - Gaea-C6 - Orion - Hercules + - Jet - Cloud validations: required: true diff --git a/dev/jobs/JGFS_ATMOS_VERIFICATION b/dev/jobs/JGFS_ATMOS_VERIFICATION index 261e180b0ae..15986bffa9d 100755 --- a/dev/jobs/JGFS_ATMOS_VERIFICATION +++ b/dev/jobs/JGFS_ATMOS_VERIFICATION @@ -37,20 +37,18 @@ for grid in '1p00'; do declare -rx "${prod_dir}"="${ROTDIR}/${RUN}.${PDY}/${cyc}/products/atmos/grib2/${grid}" done +# TODO: If none of these are on, why are we running this job? if [[ "${RUN_GRID2GRID_STEP1}" == "YES" || "${RUN_GRID2OBS_STEP1}" == "YES" || "${RUN_PRECIP_STEP1}" == "YES" ]]; then # Override the -e in VERIF_GLOBALSH's shebang and un-export SHELLOPTS # TODO: clean up the verif-global script so it does not raise false-positive errors source "${USHglobal}/unset_strict.sh" export -n SHELLOPTS - "${VERIF_GLOBALSH}" + bash -x "${VERIF_GLOBALSH}" err=$? source "${USHglobal}/set_strict.sh" if [[ ${err} -ne 0 ]]; then exit "${err}" fi -else - export err=1 - err_exit "FATAL ERROR: No verification steps are enabled. Please check your configuration." fi if [[ ${KEEPDATA:-"NO"} == "NO" ]]; then diff --git a/dev/parm/config/gcafs/config.resources b/dev/parm/config/gcafs/config.resources index e19209d99fa..3bd2ed049c8 100644 --- a/dev/parm/config/gcafs/config.resources +++ b/dev/parm/config/gcafs/config.resources @@ -63,6 +63,43 @@ case ${machine} in max_tasks_per_node=80 mem_node_max="500GB" ;; + "JET") + case ${PARTITION_BATCH} in + "xjet") + max_tasks_per_node=24 + mem_node_max="61GB" + ;; + "vjet") + max_tasks_per_node=16 + mem_node_max="61GB" + ;; + "sjet") + max_tasks_per_node=16 + mem_node_max="29GB" + ;; + "kjet") + max_tasks_per_node=40 + mem_node_max="88GB" + ;; + *) + echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" + exit 3 + esac + ;; + "S4") + case ${PARTITION_BATCH} in + "s4") max_tasks_per_node=32 + mem_node_max="168GB" + ;; + "ivy") + max_tasks_per_node=20 + mem_node_max="128GB" + ;; + *) + echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" + exit 3 + esac + ;; "AWSPW") export PARTITION_BATCH="compute" npe_node_max=48 diff --git a/dev/parm/config/gfs/config.base.j2 b/dev/parm/config/gfs/config.base.j2 index b8c45d08756..c0c7c44a0c5 100644 --- a/dev/parm/config/gfs/config.base.j2 +++ b/dev/parm/config/gfs/config.base.j2 @@ -488,12 +488,6 @@ export DO_CA="YES" export DO_METP="{{ DO_METP }}" # Run METPLUS jobs - set METPLUS settings in config.metp export DO_FIT2OBS="YES" # Run fit to observations package -# METplus verification is not enabled on Hera or Orion -if [[ "${machine}" == "hera" || "${machine}" == "orion" ]] && [[ ${DO_METP} == "YES" ]]; then - echo "WARNING: METplus verification not supported on ${machine}. Setting DO_METP=NO" - export DO_METP="NO" -fi - #--online archive of netcdf files for fit2obs verification export FHMAX_FITS=132 if [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]]; then @@ -525,6 +519,12 @@ else fi fi +# TODO: Enable METplus on Ursa when verif-global has been upgraded to spack-stack 1.9.x+ +# TODO: Clean this up by allowing DO_METP to be system-specific and/or allow overriding that parameter +if [[ "${machine}" == "URSA" || "${machine}" == "GAEAC6" || "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then + export DO_METP=NO +fi + if [[ "${DOENKFONLY_ATM:-NO}" == "YES" ]] ; then export RECENTER_ENKF="NO" # Turn off recentering ensemble analysis export DO_VERFOZN="NO" # Ozone data assimilation monitoring diff --git a/dev/parm/config/gfs/config.metp b/dev/parm/config/gfs/config.metp index 7e2e57b2f63..b15622eaf77 100644 --- a/dev/parm/config/gfs/config.metp +++ b/dev/parm/config/gfs/config.metp @@ -24,8 +24,8 @@ export VERIF_GLOBALSH=${HOMEverif_global}/ush/run_verif_global_in_global_workflo ## INPUT DATA SETTINGS export model=${PSLOT} export model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init?fmt=%Y%m%d%H}.grib2" -export model_hpss_dir=${ATARDIR} -export model_dir=${ARCDIR} +export model_hpss_dir=${ATARDIR}/.. +export model_dir=${ARCDIR}/.. export get_data_from_hpss="NO" export hpss_walltime="10" ## OUTPUT SETTINGS @@ -45,19 +45,22 @@ export g2g1_anom_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" export g2g1_anom_fhr_min=${FHMIN_GFS} export g2g1_anom_fhr_max=${FHMAX_GFS} export g2g1_anom_grid="G002" -export g2g1_anom_gather_by="VALID" +export g2g1_anom_gather_by="VSDB" export g2g1_pres_truth_name="self_anl" export g2g1_pres_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" export g2g1_pres_fhr_min=${FHMIN_GFS} export g2g1_pres_fhr_max=${FHMAX_GFS} export g2g1_pres_grid="G002" -export g2g1_pres_gather_by="VALID" +export g2g1_pres_gather_by="VSDB" export g2g1_sfc_truth_name="self_f00" export g2g1_sfc_truth_file_format="pgbf00.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" export g2g1_sfc_fhr_min=${FHMIN_GFS} export g2g1_sfc_fhr_max=${FHMAX_GFS} export g2g1_sfc_grid="G002" -export g2g1_sfc_gather_by="VALID" +export g2g1_sfc_gather_by="VSDB" +export g2g1_mv_database_name="mv_${PSLOT}_grid2grid_metplus" +export g2g1_mv_database_group="NOAA NCEP" +export g2g1_mv_database_desc="Grid-to-grid METplus data for global workflow experiment ${PSLOT}" # GRID-TO-OBS STEP 1: gfsmetpg2o1 export g2o1_type_list="upper_air conus_sfc" export g2o1_upper_air_msg_type_list="ADPUPA" @@ -65,20 +68,23 @@ export g2o1_upper_air_vhr_list="00 06 12 18" export g2o1_upper_air_fhr_min=${FHMIN_GFS} export g2o1_upper_air_fhr_max="240" export g2o1_upper_air_grid="G003" -export g2o1_upper_air_gather_by="VALID" +export g2o1_upper_air_gather_by="VSDB" export g2o1_conus_sfc_msg_type_list="ONLYSF ADPUPA" export g2o1_conus_sfc_vhr_list="00 03 06 09 12 15 18 21" export g2o1_conus_sfc_fhr_min=${FHMIN_GFS} export g2o1_conus_sfc_fhr_max="240" export g2o1_conus_sfc_grid="G104" -export g2o1_conus_sfc_gather_by="VALID" +export g2o1_conus_sfc_gather_by="VSDB" export g2o1_polar_sfc_msg_type_list="IABP" export g2o1_polar_sfc_vhr_list="00 03 06 09 12 15 18 21" export g2o1_polar_sfc_fhr_min=${FHMIN_GFS} export g2o1_polar_sfc_fhr_max="240" export g2o1_polar_sfc_grid="G219" -export g2o1_polar_sfc_gather_by="VALID" +export g2o1_polar_sfc_gather_by="VSDB" export g2o1_prepbufr_data_run_hpss="NO" +export g2o1_mv_database_name="mv_${PSLOT}_grid2obs_metplus" +export g2o1_mv_database_group="NOAA NCEP" +export g2o1_mv_database_desc="Grid-to-obs METplus data for global workflow experiment ${PSLOT}" # PRECIP STEP 1: gfsmetppcp1 export precip1_type_list="ccpa_accum24hr" export precip1_ccpa_accum24hr_model_bucket="06" @@ -87,7 +93,10 @@ export precip1_ccpa_accum24hr_model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init export precip1_ccpa_accum24hr_fhr_min=${FHMIN_GFS} export precip1_ccpa_accum24hr_fhr_max="180" export precip1_ccpa_accum24hr_grid="G211" -export precip1_ccpa_accum24hr_gather_by="VALID" +export precip1_ccpa_accum24hr_gather_by="VSDB" export precip1_obs_data_run_hpss="NO" +export precip1_mv_database_name="mv_${PSLOT}_precip_metplus" +export precip1_mv_database_group="NOAA NCEP" +export precip1_mv_database_desc="Precip METplus data for global workflow experiment ${PSLOT}" echo "END: config.metp" diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources index 6ccf9b18a2b..38fec5b7dbc 100644 --- a/dev/parm/config/gfs/config.resources +++ b/dev/parm/config/gfs/config.resources @@ -1110,7 +1110,6 @@ case ${step} in ntasks=1 tasks_per_node=1 memory="80G" - prepost=True ;; "echgres") diff --git a/dev/ush/gw_setup.sh b/dev/ush/gw_setup.sh index e3b4a408f01..0dcdf2a35d7 100755 --- a/dev/ush/gw_setup.sh +++ b/dev/ush/gw_setup.sh @@ -10,7 +10,7 @@ # Determine if HOMEglobal is already set unset_homegfs=NO -if [[ -z "${HOMEglobal:-}" ]]; then +if [[ -z "${HOMEglobal+x}" ]]; then script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" HOMEglobal=$(cd "${script_dir}" && git rev-parse --show-toplevel) export HOMEglobal diff --git a/dev/ush/load_modules.sh b/dev/ush/load_modules.sh index 3b0be684be9..2f134f57aec 100755 --- a/dev/ush/load_modules.sh +++ b/dev/ush/load_modules.sh @@ -152,34 +152,13 @@ case "${MODULE_TYPE}" in export PYTHONPATH ;; - "verif") - # EMC_verif-global modules -- use that submodule's module files - if [[ "${MACHINE_ID}" == "wcoss2" ]]; then - source "${HOMEglobal}/sorc/verif-global.fd/versions/run.ver" - fi - module use "${HOMEglobal}/sorc/verif-global.fd/modulefiles" - module load "emc_verif_global_${MACHINE_ID}" - export err=$? - if [[ ${err} -ne 0 ]]; then - echo "FATAL ERROR: Failed to load emc_verif_global_${MACHINE_ID}" - exit 1 - fi - module list - - ;; - - "run" | "gsi" | "setup" | "upp") + "run" | "gsi" | "verif" | "setup" | "upp") # Test that the version file exists if [[ ! -f "${HOMEglobal}/versions/run.ver" ]]; then echo "FATAL ERROR: ${HOMEglobal}/versions/run.ver does not exist!" echo "HINT: Run link_workflow.sh first." - # Exit with 0 if loading setup modules (so the user's terminal doesn't close), else with 1 - if [[ "${MODULE_TYPE}" == "setup" ]]; then - exit 0 - else - exit 1 - fi + exit 1 fi # Load our modules: @@ -199,8 +178,8 @@ case "${MODULE_TYPE}" in mod_type="${MODULE_TYPE}" fi - # Source versions file (except for upp and verification) - if [[ "${mod_type}" != "upp" && "${mod_type}" != "verif" ]]; then + # Source versions file (except for upp) + if [[ "${mod_type}" != "upp" ]]; then source "${HOMEglobal}/versions/run.ver" fi diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index 44f58015a48..eb7d943d50d 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -17,7 +17,10 @@ The following system software requirements are the minimum for any new or existi +==============+=============+=======================================+ | Bash | 4.4.20 | | +--------------+-------------+---------------------------------------+ -| Python | 3.11.6 | | +| Python | * 3.8.6 | * 3.10.x is not supported by METplus | +| | * 3.10.13+ | verification software | +| | * 3.11.6+ | * 3.11.6 is packaged with spack-stack | +| | | * 3.9.x is untested | +--------------+-------------+---------------------------------------+ | Spack-Stack | 1.6.0 | * Available everywhere but WCOSS2 | +--------------+-------------+---------------------------------------+ @@ -103,7 +106,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - - - X - - X + - - X * - Hercules - 1 @@ -118,7 +121,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - - - X - - X + - - X * - Gaea C6 - 1 @@ -133,10 +136,10 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - - - X - - X + - - X * - Hera - - 2 + - 1 - X - X - X @@ -148,7 +151,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - X - - X - - + - X - X * - Orion - 2 diff --git a/modulefiles/gw_run.common.lua b/modulefiles/gw_run.common.lua index e8013f6922a..997466b5748 100644 --- a/modulefiles/gw_run.common.lua +++ b/modulefiles/gw_run.common.lua @@ -46,6 +46,9 @@ local common_modules = { "py-pandas", "py-python-dateutil", "py-xarray", + -- TODO: Reenable when MET/METplus and verif-global are at compatible versions + -- "met", + -- "metplus", } for _, name in pairs(common_modules) do diff --git a/modulefiles/gw_run.gaeac6.lua b/modulefiles/gw_run.gaeac6.lua index e5c8331a74a..82be4745e03 100644 --- a/modulefiles/gw_run.gaeac6.lua +++ b/modulefiles/gw_run.gaeac6.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Gaea C6 -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None" and mode() == "load") then +if (homegfssdir == "None") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.hera.lua b/modulefiles/gw_run.hera.lua index f8538f8df56..c151c62f5a8 100644 --- a/modulefiles/gw_run.hera.lua +++ b/modulefiles/gw_run.hera.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Hera -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None" and mode() == "load") then +if (homegfssdir == "None") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.hercules.lua b/modulefiles/gw_run.hercules.lua index 86b5e5bf625..2942e96822d 100644 --- a/modulefiles/gw_run.hercules.lua +++ b/modulefiles/gw_run.hercules.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Hercules -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None" and mode() == "load") then +if (homegfssdir == "None") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.noaacloud.lua b/modulefiles/gw_run.noaacloud.lua index 4f48650f2e3..2508d6de0f5 100644 --- a/modulefiles/gw_run.noaacloud.lua +++ b/modulefiles/gw_run.noaacloud.lua @@ -5,7 +5,7 @@ Load environment to run GFS on NOAA cloud -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None" and mode() == "load") then +if (homegfssdir == "None") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.orion.lua b/modulefiles/gw_run.orion.lua index 183dbce9917..9e4a32defc2 100644 --- a/modulefiles/gw_run.orion.lua +++ b/modulefiles/gw_run.orion.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Orion -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None" and mode() == "load") then +if (homegfssdir == "None") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.ursa.lua b/modulefiles/gw_run.ursa.lua index bbd83a92e3f..0fe3847f1a0 100644 --- a/modulefiles/gw_run.ursa.lua +++ b/modulefiles/gw_run.ursa.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Ursa -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None" and mode() == "load") then +if (homegfssdir == "None") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_verif.hera.lua b/modulefiles/gw_verif.hera.lua new file mode 100644 index 00000000000..ae567e30e35 --- /dev/null +++ b/modulefiles/gw_verif.hera.lua @@ -0,0 +1,15 @@ +help([[ +Load environment to run EMC_verif-global on Hera +]]) + +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-fms-2024.01/install/modulefiles/Core") +load("stack-intel") +load("stack-python") +load("stack-intel-oneapi-mpi") +load("met/9.1.3") +load("metplus/3.1.1") +load("grib-util") +load("prod_util") +load("wgrib2") + +whatis("Description: EMC_verif-global run environment") diff --git a/modulefiles/gw_verif.wcoss2.lua b/modulefiles/gw_verif.wcoss2.lua new file mode 100644 index 00000000000..700e2d4f8e0 --- /dev/null +++ b/modulefiles/gw_verif.wcoss2.lua @@ -0,0 +1,35 @@ +help([[ +Load environment to run verification on WCOSS2 +]]) + +load(pathJoin("PrgEnv-intel", (os.getenv("PrgEnv_intel_ver") or "None"))) +load(pathJoin("craype", (os.getenv("craype_ver") or "None"))) +load(pathJoin("intel", (os.getenv("intel_ver") or "None"))) +load(pathJoin("cray-mpich", (os.getenv("cray_mpich_ver") or "None"))) +load(pathJoin("cray-pals", (os.getenv("cray_pals_ver") or "None"))) +load(pathJoin("cfp", (os.getenv("cfp_ver") or "None"))) +setenv("USE_CFP","YES") + +-- Use Python 3.8.6 for verification +load(pathJoin("python", '3.8.6')) + +load(pathJoin("libjpeg", (os.getenv("libjpeg_ver") or "None"))) + +-- MET was build with these versions of HDF5 and netCDF +load(pathJoin("hdf5", '1.10.6')) +load(pathJoin("netcdf", '4.7.4')) + +-- The cray library path for C MPI libraries (needed by C-only programs using netCDF) +local cray_lib_path=os.getenv("CRAY_LD_LIBRARY_PATH") or "" +prepend_path("LD_LIBRARY_PATH", cray_lib_path) + +load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) +load(pathJoin("grib_util", (os.getenv("grib_util_ver") or "None"))) +load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) + +prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304") +setenv("HPC_OPT", "/apps/ops/para/libs") +load(pathJoin("met", (os.getenv("met_ver") or "None"))) +load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) + +whatis("Description: Verification environment") diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index 0f7aed29d3c..d7991eae9e7 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit 0f7aed29d3c84c19e0bddacc511926d074a2e721 +Subproject commit d7991eae9e73845b8feea11c9173eeec3c7e08f2