Revert "Re-enable and upgrade verif-global to use spack-stack 2.0.0, metplus 6, met 12"

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -46,6 +46,7 @@ body:
         - Gaea-C6
         - Orion
         - Hercules
+        - Jet
         - Cloud
     validations:
       required: true

dev/jobs/JGFS_ATMOS_VERIFICATION

@@ -37,20 +37,18 @@ for grid in '1p00'; do
     declare -rx "${prod_dir}"="${ROTDIR}/${RUN}.${PDY}/${cyc}/products/atmos/grib2/${grid}"
 done
 
+# TODO: If none of these are on, why are we running this job?
 if [[ "${RUN_GRID2GRID_STEP1}" == "YES" || "${RUN_GRID2OBS_STEP1}" == "YES" || "${RUN_PRECIP_STEP1}" == "YES" ]]; then
     # Override the -e in VERIF_GLOBALSH's shebang and un-export SHELLOPTS
     # TODO: clean up the verif-global script so it does not raise false-positive errors
     source "${USHglobal}/unset_strict.sh"
     export -n SHELLOPTS
-    "${VERIF_GLOBALSH}"
+    bash -x "${VERIF_GLOBALSH}"
     err=$?
     source "${USHglobal}/set_strict.sh"
     if [[ ${err} -ne 0 ]]; then
         exit "${err}"
     fi
-else
-    export err=1
-    err_exit "FATAL ERROR: No verification steps are enabled. Please check your configuration."
 fi
 
 if [[ ${KEEPDATA:-"NO"} == "NO" ]]; then

dev/parm/config/gcafs/config.resources

@@ -63,6 +63,43 @@ case ${machine} in
               max_tasks_per_node=80
               mem_node_max="500GB"
     ;;
+  "JET")
+    case ${PARTITION_BATCH} in
+      "xjet")
+              max_tasks_per_node=24
+              mem_node_max="61GB"
+        ;;
+      "vjet")
+              max_tasks_per_node=16
+              mem_node_max="61GB"
+        ;;
+      "sjet")
+              max_tasks_per_node=16
+              mem_node_max="29GB"
+        ;;
+      "kjet")
+              max_tasks_per_node=40
+              mem_node_max="88GB"
+        ;;
+      *)
+        echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}"
+        exit 3
+    esac
+    ;;
+  "S4")
+    case ${PARTITION_BATCH} in
+      "s4")   max_tasks_per_node=32
+              mem_node_max="168GB"
+        ;;
+      "ivy")
+              max_tasks_per_node=20
+              mem_node_max="128GB"
+        ;;
+      *)
+        echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}"
+        exit 3
+    esac
+    ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
     npe_node_max=48

dev/parm/config/gfs/config.base.j2

@@ -488,12 +488,6 @@ export DO_CA="YES"
 export DO_METP="{{ DO_METP }}"   # Run METPLUS jobs - set METPLUS settings in config.metp
 export DO_FIT2OBS="YES"      # Run fit to observations package
 
-# METplus verification is not enabled on Hera or Orion
-if [[ "${machine}" == "hera" || "${machine}" == "orion" ]] && [[ ${DO_METP} == "YES" ]]; then
-   echo "WARNING: METplus verification not supported on ${machine}. Setting DO_METP=NO"
-   export DO_METP="NO"
-fi
-
 #--online archive of netcdf files for fit2obs verification
 export FHMAX_FITS=132
 if [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]]; then
@@ -525,6 +519,12 @@ else
    fi
 fi
 
+# TODO: Enable METplus on Ursa when verif-global has been upgraded to spack-stack 1.9.x+
+# TODO: Clean this up by allowing DO_METP to be system-specific and/or allow overriding that parameter
+if [[ "${machine}" == "URSA" || "${machine}" == "GAEAC6" || "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then
+    export DO_METP=NO
+fi
+
 if [[ "${DOENKFONLY_ATM:-NO}" == "YES" ]] ; then
     export RECENTER_ENKF="NO"   # Turn off recentering ensemble analysis
     export DO_VERFOZN="NO"      # Ozone data assimilation monitoring

dev/parm/config/gfs/config.metp

@@ -24,8 +24,8 @@ export VERIF_GLOBALSH=${HOMEverif_global}/ush/run_verif_global_in_global_workflo
 ## INPUT DATA SETTINGS
 export model=${PSLOT}
 export model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init?fmt=%Y%m%d%H}.grib2"
-export model_hpss_dir=${ATARDIR}
-export model_dir=${ARCDIR}
+export model_hpss_dir=${ATARDIR}/..
+export model_dir=${ARCDIR}/..
 export get_data_from_hpss="NO"
 export hpss_walltime="10"
 ## OUTPUT SETTINGS
@@ -45,40 +45,46 @@ export g2g1_anom_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2"
 export g2g1_anom_fhr_min=${FHMIN_GFS}
 export g2g1_anom_fhr_max=${FHMAX_GFS}
 export g2g1_anom_grid="G002"
-export g2g1_anom_gather_by="VALID"
+export g2g1_anom_gather_by="VSDB"
 export g2g1_pres_truth_name="self_anl"
 export g2g1_pres_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2"
 export g2g1_pres_fhr_min=${FHMIN_GFS}
 export g2g1_pres_fhr_max=${FHMAX_GFS}
 export g2g1_pres_grid="G002"
-export g2g1_pres_gather_by="VALID"
+export g2g1_pres_gather_by="VSDB"
 export g2g1_sfc_truth_name="self_f00"
 export g2g1_sfc_truth_file_format="pgbf00.${RUN}.{valid?fmt=%Y%m%d%H}.grib2"
 export g2g1_sfc_fhr_min=${FHMIN_GFS}
 export g2g1_sfc_fhr_max=${FHMAX_GFS}
 export g2g1_sfc_grid="G002"
-export g2g1_sfc_gather_by="VALID"
+export g2g1_sfc_gather_by="VSDB"
+export g2g1_mv_database_name="mv_${PSLOT}_grid2grid_metplus"
+export g2g1_mv_database_group="NOAA NCEP"
+export g2g1_mv_database_desc="Grid-to-grid METplus data for global workflow experiment ${PSLOT}"
 # GRID-TO-OBS STEP 1: gfsmetpg2o1
 export g2o1_type_list="upper_air conus_sfc"
 export g2o1_upper_air_msg_type_list="ADPUPA"
 export g2o1_upper_air_vhr_list="00 06 12 18"
 export g2o1_upper_air_fhr_min=${FHMIN_GFS}
 export g2o1_upper_air_fhr_max="240"
 export g2o1_upper_air_grid="G003"
-export g2o1_upper_air_gather_by="VALID"
+export g2o1_upper_air_gather_by="VSDB"
 export g2o1_conus_sfc_msg_type_list="ONLYSF ADPUPA"
 export g2o1_conus_sfc_vhr_list="00 03 06 09 12 15 18 21"
 export g2o1_conus_sfc_fhr_min=${FHMIN_GFS}
 export g2o1_conus_sfc_fhr_max="240"
 export g2o1_conus_sfc_grid="G104"
-export g2o1_conus_sfc_gather_by="VALID"
+export g2o1_conus_sfc_gather_by="VSDB"
 export g2o1_polar_sfc_msg_type_list="IABP"
 export g2o1_polar_sfc_vhr_list="00 03 06 09 12 15 18 21"
 export g2o1_polar_sfc_fhr_min=${FHMIN_GFS}
 export g2o1_polar_sfc_fhr_max="240"
 export g2o1_polar_sfc_grid="G219"
-export g2o1_polar_sfc_gather_by="VALID"
+export g2o1_polar_sfc_gather_by="VSDB"
 export g2o1_prepbufr_data_run_hpss="NO"
+export g2o1_mv_database_name="mv_${PSLOT}_grid2obs_metplus"
+export g2o1_mv_database_group="NOAA NCEP"
+export g2o1_mv_database_desc="Grid-to-obs METplus data for global workflow experiment ${PSLOT}"
 # PRECIP STEP 1: gfsmetppcp1
 export precip1_type_list="ccpa_accum24hr"
 export precip1_ccpa_accum24hr_model_bucket="06"
@@ -87,7 +93,10 @@ export precip1_ccpa_accum24hr_model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init
 export precip1_ccpa_accum24hr_fhr_min=${FHMIN_GFS}
 export precip1_ccpa_accum24hr_fhr_max="180"
 export precip1_ccpa_accum24hr_grid="G211"
-export precip1_ccpa_accum24hr_gather_by="VALID"
+export precip1_ccpa_accum24hr_gather_by="VSDB"
 export precip1_obs_data_run_hpss="NO"
+export precip1_mv_database_name="mv_${PSLOT}_precip_metplus"
+export precip1_mv_database_group="NOAA NCEP"
+export precip1_mv_database_desc="Precip METplus data for global workflow experiment ${PSLOT}"
 
 echo "END: config.metp"

dev/parm/config/gfs/config.resources

@@ -1110,7 +1110,6 @@ case ${step} in
     ntasks=1
     tasks_per_node=1
     memory="80G"
-    prepost=True
     ;;
 
   "echgres")

dev/ush/gw_setup.sh

@@ -10,7 +10,7 @@
 
 # Determine if HOMEglobal is already set
 unset_homegfs=NO
-if [[ -z "${HOMEglobal:-}" ]]; then
+if [[ -z "${HOMEglobal+x}" ]]; then
     script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
     HOMEglobal=$(cd "${script_dir}" && git rev-parse --show-toplevel)
     export HOMEglobal

dev/ush/load_modules.sh

@@ -152,34 +152,13 @@ case "${MODULE_TYPE}" in
         export PYTHONPATH
         ;;
 
-    "verif")
-        # EMC_verif-global modules -- use that submodule's module files
-        if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
-            source "${HOMEglobal}/sorc/verif-global.fd/versions/run.ver"
-        fi
-        module use "${HOMEglobal}/sorc/verif-global.fd/modulefiles"
-        module load "emc_verif_global_${MACHINE_ID}"
-        export err=$?
-        if [[ ${err} -ne 0 ]]; then
-            echo "FATAL ERROR: Failed to load emc_verif_global_${MACHINE_ID}"
-            exit 1
-        fi
-        module list
-
-        ;;
-
-    "run" | "gsi" | "setup" | "upp")
+    "run" | "gsi" | "verif" | "setup" | "upp")
 
         # Test that the version file exists
         if [[ ! -f "${HOMEglobal}/versions/run.ver" ]]; then
             echo "FATAL ERROR: ${HOMEglobal}/versions/run.ver does not exist!"
             echo "HINT: Run link_workflow.sh first."
-            # Exit with 0 if loading setup modules (so the user's terminal doesn't close), else with 1
-            if [[ "${MODULE_TYPE}" == "setup" ]]; then
-                exit 0
-            else
-                exit 1
-            fi
+            exit 1
         fi
 
         # Load our modules:
@@ -199,8 +178,8 @@ case "${MODULE_TYPE}" in
             mod_type="${MODULE_TYPE}"
         fi
 
-        # Source versions file (except for upp and verification)
-        if [[ "${mod_type}" != "upp" && "${mod_type}" != "verif" ]]; then
+        # Source versions file (except for upp)
+        if [[ "${mod_type}" != "upp" ]]; then
             source "${HOMEglobal}/versions/run.ver"
         fi
 

docs/source/hpc.rst

@@ -17,7 +17,10 @@ The following system software requirements are the minimum for any new or existi
 +==============+=============+=======================================+
 | Bash         | 4.4.20      |                                       |
 +--------------+-------------+---------------------------------------+
-| Python       | 3.11.6      |                                       |
+| Python       | * 3.8.6     | * 3.10.x is not supported by METplus  |
+|              | * 3.10.13+  |   verification software               |
+|              | * 3.11.6+   | * 3.11.6 is packaged with spack-stack |
+|              |             | * 3.9.x is untested                   |
 +--------------+-------------+---------------------------------------+
 | Spack-Stack  | 1.6.0       | * Available everywhere but WCOSS2     |
 +--------------+-------------+---------------------------------------+
@@ -103,7 +106,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas
      -
      -
      - X
-     - X
+     -
      - X
    * - Hercules
      - 1
@@ -118,7 +121,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas
      -
      -
      - X
-     - X
+     -
      - X
    * - Gaea C6
      - 1
@@ -133,10 +136,10 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas
      -
      -
      - X
-     - X
+     -
      - X
    * - Hera
-     - 2
+     - 1
      - X
      - X
      - X
@@ -148,7 +151,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas
      - X
      -
      - X
-     -
+     - X
      - X
    * - Orion
      - 2

modulefiles/gw_run.common.lua

@@ -46,6 +46,9 @@ local common_modules = {
   "py-pandas",
   "py-python-dateutil",
   "py-xarray",
+  -- TODO: Reenable when MET/METplus and verif-global are at compatible versions
+  -- "met",
+  -- "metplus",
 }
 
 for _, name in pairs(common_modules) do

modulefiles/gw_run.gaeac6.lua

@@ -5,7 +5,7 @@ Load environment to run GFS on Gaea C6
 -- Test that HOMEglobal is set.
 -- If not, load_modules.sh was not sourced to load this module.
 local homegfssdir=os.getenv("HOMEglobal") or "None"
-if (homegfssdir == "None" and mode() == "load") then
+if (homegfssdir == "None") then
     LmodError("FATAL ERROR HOMEglobal variable is unset.\n" ..
               "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n")
 end

modulefiles/gw_run.hera.lua

@@ -5,7 +5,7 @@ Load environment to run GFS on Hera
 -- Test that HOMEglobal is set.
 -- If not, load_modules.sh was not sourced to load this module.
 local homegfssdir=os.getenv("HOMEglobal") or "None"
-if (homegfssdir == "None" and mode() == "load") then
+if (homegfssdir == "None") then
     LmodError("FATAL ERROR HOMEglobal variable is unset.\n" ..
               "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n")
 end

modulefiles/gw_run.hercules.lua

@@ -5,7 +5,7 @@ Load environment to run GFS on Hercules
 -- Test that HOMEglobal is set.
 -- If not, load_modules.sh was not sourced to load this module.
 local homegfssdir=os.getenv("HOMEglobal") or "None"
-if (homegfssdir == "None" and mode() == "load") then
+if (homegfssdir == "None") then
     LmodError("FATAL ERROR HOMEglobal variable is unset.\n" ..
               "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n")
 end

modulefiles/gw_run.noaacloud.lua

@@ -5,7 +5,7 @@ Load environment to run GFS on NOAA cloud
 -- Test that HOMEglobal is set.
 -- If not, load_modules.sh was not sourced to load this module.
 local homegfssdir=os.getenv("HOMEglobal") or "None"
-if (homegfssdir == "None" and mode() == "load") then
+if (homegfssdir == "None") then
     LmodError("FATAL ERROR HOMEglobal variable is unset.\n" ..
               "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n")
 end

modulefiles/gw_run.orion.lua

@@ -5,7 +5,7 @@ Load environment to run GFS on Orion
 -- Test that HOMEglobal is set.
 -- If not, load_modules.sh was not sourced to load this module.
 local homegfssdir=os.getenv("HOMEglobal") or "None"
-if (homegfssdir == "None" and mode() == "load") then
+if (homegfssdir == "None") then
     LmodError("FATAL ERROR HOMEglobal variable is unset.\n" ..
               "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n")
 end

modulefiles/gw_run.ursa.lua

@@ -5,7 +5,7 @@ Load environment to run GFS on Ursa
 -- Test that HOMEglobal is set.
 -- If not, load_modules.sh was not sourced to load this module.
 local homegfssdir=os.getenv("HOMEglobal") or "None"
-if (homegfssdir == "None" and mode() == "load") then
+if (homegfssdir == "None") then
     LmodError("FATAL ERROR HOMEglobal variable is unset.\n" ..
               "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n")
 end

modulefiles/gw_verif.hera.lua

@@ -0,0 +1,15 @@
+help([[
+Load environment to run EMC_verif-global on Hera
+]])
+
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-fms-2024.01/install/modulefiles/Core")
+load("stack-intel")
+load("stack-python")
+load("stack-intel-oneapi-mpi")
+load("met/9.1.3")
+load("metplus/3.1.1")
+load("grib-util")
+load("prod_util")
+load("wgrib2")
+
+whatis("Description: EMC_verif-global run environment")