Sync develop into main

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 
 # Anchors in case we need to override the defaults from the orb
-#baselibs_version: &baselibs_version v7.27.0
+#baselibs_version: &baselibs_version v7.33.0
 #bcs_version: &bcs_version v11.6.0
 
 orbs:

.github/workflows/spack-ci.yml

@@ -81,7 +81,6 @@ jobs:
       - name: Set default compiler and target
         shell: spack-bash {0}
         run: |
-          spack config add 'packages:all:compiler:[gcc@14.2.0]'
           spack config add 'packages:all:require:target=x86_64_v3'
 
       - name: Create Spack environment
@@ -107,6 +106,7 @@ jobs:
       - name: Install
         shell: spack-bash {0}
         run: |
+          spack clean -m
           spack -e spack-env install --add --no-check-signature --use-buildcache only \
             esmf gftl gftl-shared fargparse pflogger pfunit yafyaml ecbuild udunits openblas
 

.github/workflows/workflow.yml

@@ -21,7 +21,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, '0 diff trivial')"
     runs-on: ubuntu-24.04
     container:
-      image: gmao/ubuntu24-geos-env:v7.32.0-intelmpi_2021.13-ifort_2021.13
+      image: gmao/ubuntu24-geos-env:v7.33.0-intelmpi_2021.13-ifort_2021.13
       # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495
       # It seems like we might not need secrets on GitHub Actions which is good for forked
       # pull requests

CAP.rc.tmpl

@@ -1,6 +1,4 @@
-
-MAPLROOT_COMPNAME: GCM
-        ROOT_NAME: GCM
+ROOT_NAME: GCM
 ROOT_CF: AGCM.rc
 HIST_CF: HISTORY.rc
 

HISTORY.AGCM.rc.tmpl

@@ -335,6 +335,8 @@ PC@HIST_IMx@HIST_JM-DC.LM: @AGCM_LM
                                  'SWNDWTR'      , 'SURFACE'  , 'SWGNTWTR'    ,
                                  'SWNDICE'      , 'SURFACE'  , 'SWGNTICE'    ,
                                  'SNOWOCN'      , 'SURFACE'  , 'PRECSNOOCN'  ,
+                                 'ICEFOCN'      , 'SURFACE'  , 'PRECICEOCN'  ,
+                                 'SPTOTOCN'     , 'SURFACE'  , 'PRECSOLOCN'  ,
                                  'RAINOCN'      , 'SURFACE'  ,
                                  'TAUXW;TAUYW'  , 'SURFACE'  , 'TAUXWTR;TAUYWTR',
                                  'TAUXI;TAUYI'  , 'SURFACE'  , 'TAUXICE;TAUYICE',

HISTORY.AOGCM-MOM5.rc.tmpl

@@ -343,6 +343,8 @@ PC@HIST_IMx@HIST_JM-DC.LM: @AGCM_LM
                                  'SWNDWTR'      , 'SURFACE'  , 'SWGNTWTR'    ,
                                  'SWNDICE'      , 'SURFACE'  , 'SWGNTICE'    ,
                                  'SNOWOCN'      , 'SURFACE'  , 'PRECSNOOCN'  ,
+                                 'ICEFOCN'      , 'SURFACE'  , 'PRECICEOCN'  ,
+                                 'SPTOTOCN'     , 'SURFACE'  , 'PRECSOLOCN'  ,
                                  'RAINOCN'      , 'SURFACE'  ,
                                  'TAUXW;TAUYW'  , 'SURFACE'  , 'TAUXWTR;TAUYWTR',
                                  'TAUXI;TAUYI'  , 'SURFACE'  , 'TAUXICE;TAUYICE',
@@ -1657,6 +1659,8 @@ PC@HIST_IMx@HIST_JM-DC.LM: @AGCM_LM
                             'RAINOCN'     , 'OPENWATER',
                             'AO_RAIN'     , 'OPENWATER',
                             'SNOWOCN'     , 'OPENWATER',
+                            'ICEFOCN'     , 'OPENWATER',
+                            'SPTOTOCN'    , 'OPENWATER',
                             'AO_SNOW'     , 'OPENWATER',
                             'FRACI'       , 'SEAICETHERMO',
                             'FRZMLT'      , 'SEAICETHERMO',

HISTORY.AOGCM.rc.tmpl

@@ -340,6 +340,8 @@ PC@HIST_IMx@HIST_JM-DC.LM: @AGCM_LM
                                  'SWNDWTR'      , 'SURFACE'  , 'SWGNTWTR'    ,
                                  'SWNDICE'      , 'SURFACE'  , 'SWGNTICE'    ,
                                  'SNOWOCN'      , 'SURFACE'  , 'PRECSNOOCN'  ,
+                                 'ICEFOCN'      , 'SURFACE'  , 'PRECICEOCN'  ,
+                                 'SPTOTOCN'     , 'SURFACE'  , 'PRECSOLOCN'  ,
                                  'RAINOCN'      , 'SURFACE'  ,
                                  'TAUXW;TAUYW'  , 'SURFACE'  , 'TAUXWTR;TAUYWTR',
                                  'TAUXI;TAUYI'  , 'SURFACE'  , 'TAUXICE;TAUYICE',
@@ -1582,6 +1584,8 @@ PC@HIST_IMx@HIST_JM-DC.LM: @AGCM_LM
                             'RAINOCN'     , 'OPENWATER',
                             'AO_RAIN'     , 'OPENWATER',
                             'SNOWOCN'     , 'OPENWATER',
+                            'ICEFOCN'     , 'OPENWATER',
+                            'SPTOTOCN'    , 'OPENWATER',
                             'AO_SNOW'     , 'OPENWATER',
                             'FRACI'       , 'SEAICETHERMO',
                             'FRZMLT'      , 'SEAICETHERMO',

HISTORY.AOGCM_MITgcm.rc.tmpl

@@ -309,6 +309,8 @@ PC360x181-DC.GRIDNAME: PC360x181-DC
                                  'SWNDWTR'      , 'SURFACE'  , 'SWGNTWTR'    ,
                                  'SWNDICE'      , 'SURFACE'  , 'SWGNTICE'    ,
                                  'SNOWOCN'      , 'SURFACE'  , 'PRECSNOOCN'  ,
+                                 'ICEFOCN'      , 'SURFACE'  , 'PRECICEOCN'  ,
+                                 'SPTOTOCN'     , 'SURFACE'  , 'PRECSOLOCN'  ,
                                  'RAINOCN'      , 'SURFACE'  ,
                                  'TAUXW;TAUYW'  , 'SURFACE'  , 'TAUXWTR;TAUYWTR',
                                  'TAUXI;TAUYI'  , 'SURFACE'  , 'TAUXICE;TAUYICE',
@@ -1665,6 +1667,8 @@ PC360x181-DC.GRIDNAME: PC360x181-DC
                             'RAINOCN'     , 'OPENWATER',
                             'AO_RAIN'     , 'OPENWATER',
                             'SNOWOCN'     , 'OPENWATER',
+                            'ICEFOCN'     , 'OPENWATER',
+                            'SPTOTOCN'    , 'OPENWATER',
                             'AO_SNOW'     , 'OPENWATER',
                             'FRACI'       , 'SEAICETHERMO',
                             'FRZMLT'      , 'SEAICETHERMO',

HISTORY.rc.tmpl

@@ -130,6 +130,8 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
              'bkg.eta'
              'bkg.sfc'
              'cbkg.eta'
+             'bkg.lfo_inst'
+             'bkg.lfo_tavg'
              'vtx.mix'
 #            'vtx.prs'
 #            'prog.sfc+-'
@@ -139,9 +141,6 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
 #            'prog.eta+-'
 #            'traj.lcv'
 #            'ptrj.prs'
-# predictor forcing files for land da coupling 
-#            'inst1_2d_lfo_Nx+-' 
-#            'tavg1_2d_lfo_Nx+-' 
 # Forecast Files
 #            'inst3_3d_asm_Np+-'
 #            'inst3_3d_asm_Nv+-'
@@ -1039,6 +1038,8 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
                                   'SWNDWTR'      , 'SURFACE'  , 'SWGNTWTR'        ,
                                   'SWNDICE'      , 'SURFACE'  , 'SWGNTICE'        ,
                                   'SNOWOCN'      , 'SURFACE'  , 'PRECSNOOCN'      ,
+                                  'ICEFOCN'      , 'SURFACE'  , 'PRECICEOCN'      ,
+                                  'SPTOTOCN'     , 'SURFACE'  , 'PRECSOLOCN'      ,
                                   'RAINOCN'      , 'SURFACE'  ,
                                   'TAUXW;TAUYW'  , 'SURFACE'  , 'TAUXWTR;TAUYWTR' ,
                                   'TAUXI;TAUYI'  , 'SURFACE'  , 'TAUXICE;TAUYICE' ,
@@ -1097,6 +1098,8 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
                                   'SWNDWTR'      , 'SURFACE'  , 'SWGNTWTR'    ,
                                   'SWNDICE'      , 'SURFACE'  , 'SWGNTICE'    ,
                                   'SNOWOCN'      , 'SURFACE'  , 'PRECSNOOCN'  ,
+                                  'ICEFOCN'      , 'SURFACE'  , 'PRECICEOCN'  ,
+                                  'SPTOTOCN'     , 'SURFACE'  , 'PRECSOLOCN'  ,
                                   'RAINOCN'      , 'SURFACE'  ,
                                   'TAUXW;TAUYW'  , 'SURFACE'  , 'TAUXWTR;TAUYWTR' ,
                                   'TAUXI;TAUYI'  , 'SURFACE'  , 'TAUXICE;TAUYICE' ,
@@ -1288,6 +1291,47 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
                                   'GOCART::CO2'  , 'GOCART'   , 'CO2'  ,
                                    ::
 
+#######################################################################
+#                    Ldas bkg forcing 
+#######################################################################
+
+  bkg.lfo_tavg.format:   'CFIO' ,
+  bkg.lfo_tavg.descr:      '2d,1-Hourly,Time-Averaged,Single-Level,Land Surface Forcings' , 
+  bkg.lfo_tavg.template:   '%y4%m2%d2_%h2%n2z.>>>NCSUFFIX<<<' ,
+  bkg.lfo_tavg.mode:       'time-averaged' ,
+  bkg.lfo_tavg.frequency:  010000 ,
+  bkg.lfo_tavg.duration:   010000 ,
+  bkg.lfo_tavg.ref_date:   >>>IOBBKGD<<< ,
+  bkg.lfo_tavg.ref_time:   >>>IOBBKGT<<< ,
+  bkg.lfo_tavg.end_date:   >>>IOEBKGD<<< ,
+  bkg.lfo_tavg.end_time:   >>>IOEBKGT<<< ,
+  bkg.lfo_tavg.fields:     'SLRSF'   , 'SOLAR'    , 'SWGDN'    ,
+                                'LWS'     , 'IRRAD'    , 'LWGAB'    ,
+                                'PCU'     , 'SURFACE'  , 'PRECCU'   ,
+                                'PLS'     , 'SURFACE'  , 'PRECLS'   ,
+                                'SNO'     , 'SURFACE'  , 'PRECSNO'  ,
+                                'DFPAR'   , 'SOLAR'    , 'PARDF'    ,
+                                'DRPAR'   , 'SOLAR'    , 'PARDR'    ,
+                               ::
+
+  bkg.lfo_inst.format:     'CFIO' ,
+  bkg.lfo_inst.descr:      '2d,1-Hourly,Instantaneous,Single-Level,Land Surface Forcings' ,
+  bkg.lfo_inst.template:   '%y4%m2%d2_%h2%n2z.>>>NCSUFFIX<<<' ,
+  bkg.lfo_inst.mode:       'instantaneous' ,
+  bkg.lfo_inst.frequency:  010000 ,
+  bkg.lfo_inst.duration:   010000 ,
+  bkg.lfo_inst.ref_date:   >>>IOBBKGD<<< ,
+  bkg.lfo_inst.ref_time:   >>>IOBBKGT<<< ,
+  bkg.lfo_inst.end_date:   >>>IOEBKGD<<< ,
+  bkg.lfo_inst.end_time:   >>>IOEBKGT<<< ,
+  bkg.lfo_inst.fields:     'DZ'     , 'DYN'  , 'HLML'      ,
+                                'TA'     , 'DYN'  , 'TLML'      ,
+                                'QA'     , 'DYN'  , 'QLML'      ,
+                                'SPEED'  , 'DYN'  , 'SPEEDLML'  ,
+                                'PS'     , 'DYN'  ,
+                               ::
+
+
 #######################################################################
 #                     Aerosols/Constituents
 #######################################################################
@@ -1981,49 +2025,6 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
                                   'ZLE'    , 'DYN'      , 'hght'   ,
                                    ::
 
-#######################################################################
-#                predictor forcing files for land da coupling 
-#######################################################################
-
-  tavg1_2d_lfo_Nx+-.format:      'CFIO' ,
-  tavg1_2d_lfo_Nx+-.descr:       '2d,1-Hourly,Time-Averaged,Single-Level,Land Surface Forcings' ,
-# tavg1_2d_lfo_Nx+-.nbits:       10 ,
-  tavg1_2d_lfo_Nx+-.template:    '%y4%m2%d2_%h2%n2z.>>>NCSUFFIX<<<' ,
-  tavg1_2d_lfo_Nx+-.mode:        'time-averaged' ,
-  tavg1_2d_lfo_Nx+-.frequency:   010000 ,
-  tavg1_2d_lfo_Nx+-.duration:    010000 ,
-  tavg1_2d_lfo_Nx+-.ref_date:    >>>IOBBKGD<<< ,
-  tavg1_2d_lfo_Nx+-.ref_time:    >>>IOBBKGT<<< ,
-  tavg1_2d_lfo_Nx+-.end_date:    >>>IOEBKGD<<< ,
-  tavg1_2d_lfo_Nx+-.end_time:    >>>IOEBKGT<<< ,
-  tavg1_2d_lfo_Nx+-.fields:    'SLRSF'   , 'SOLAR'    , 'SWGDN'    ,
-                               'SWLAND'  , 'SURFACE'  ,
-                               'LWS'     , 'IRRAD'    , 'LWGAB'    ,
-                               'PCU'     , 'SURFACE'  , 'PRECRAINCU'   ,
-                               'PLS'     , 'SURFACE'  , 'PRECRAINLS'   ,
-                               'SNO'     , 'SURFACE'  , 'PRECSNO'  ,
-                               'DFPAR'   , 'SOLAR'    , 'PARDF'    ,
-                               'DRPAR'   , 'SOLAR'    , 'PARDR'    ,
-                               ::
-
-  inst1_2d_lfo_Nx+-.format:      'CFIO' ,
-  inst1_2d_lfo_Nx+-.descr:       '2d,1-Hourly,Instantaneous,Single-Level,Land Surface Forcings' ,
-# inst1_2d_lfo_Nx+-.nbits:       10 ,
-  inst1_2d_lfo_Nx+-.template:    '%y4%m2%d2_%h2%n2z.>>>NCSUFFIX<<<' ,
-  inst1_2d_lfo_Nx+-.mode:        'instantaneous' ,
-  inst1_2d_lfo_Nx+-.frequency:   010000 ,
-  inst1_2d_lfo_Nx+-.duration:    010000 ,
-  inst1_2d_lfo_Nx+-.ref_date:    >>>IOBBKGD<<< ,
-  inst1_2d_lfo_Nx+-.ref_time:    >>>IOBBKGT<<< ,
-  inst1_2d_lfo_Nx+-.end_date:    >>>IOEBKGD<<< ,
-  inst1_2d_lfo_Nx+-.end_time:    >>>IOEBKGT<<< ,
-  inst1_2d_lfo_Nx+-.fields:      'DZ'     , 'DYN'  , 'HLML'      ,
-                                 'TA'     , 'DYN'  , 'TLML'      ,
-                                 'QA'     , 'DYN'  , 'QLML'      ,
-                                 'SPEED'  , 'DYN'  , 'SPEEDLML'  ,
-                                 'PS'     , 'DYN'  ,
-                                 ::
-
 #######################################################################
 #                         Forecast Files
 #######################################################################

gcm_forecast.tmpl

@@ -766,7 +766,7 @@ else
 endif
 
 @SINGULARITY_BUILD @OCEAN_PRELOAD $RUN_CMD $TOTAL_PES $SINGULARITY_RUN $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
-@NATIVE_BUILD @OCEAN_PRELOAD $RUN_CMD $TOTAL_PES $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
+@NATIVE_BUILD @OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $TOTAL_PES $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
 
 if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null
 

gcm_regress.j

@@ -451,7 +451,7 @@ if( $RUN_STARTSTOP == TRUE ) then
 
    echo "=== Running test of duration ${test_duration_step1} with NX = $NX and NY = $NY starting at $nymd0 $nhms0 ==="
 
-   @OCEAN_PRELOAD $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
+   @OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
 
    set date = `cat cap_restart`
    set nymde1 = $date[1]
@@ -517,7 +517,7 @@ set NY = `grep "^ *NY": AGCM.rc | cut -d':' -f2`
 
 echo "=== Running test of duration ${test_duration_step2} with NX = $NX and NY = $NY starting at $nymd0 $nhms0 ==="
 
-@OCEAN_PRELOAD $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
+@OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
 
 set date = `cat cap_restart`
 set nymde2 = $date[1]
@@ -623,7 +623,7 @@ if ($RUN_STARTSTOP == TRUE) then
 
    echo "=== Running test of duration ${test_duration_step3} with NX = $NX and NY = $NY starting at $nymdb $nhmsb ==="
 
-   @OCEAN_PRELOAD $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
+   @OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
 
    set date = `cat cap_restart`
    set nymde3 = $date[1]
@@ -737,7 +737,7 @@ if ( $RUN_LAYOUT == TRUE) then
 
    echo "=== Running test of duration ${test_duration_step4} with NX = $test_NX and NY = $test_NY starting at $nymd0 $nhms0 ==="
 
-   @OCEAN_PRELOAD $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
+   @OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
 
    set date = `cat cap_restart`
    set nymde4 = $date[1]

gcm_run.j

@@ -37,6 +37,7 @@ setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${GEOSDIR}/lib
 # We only add BASEDIR to the @LD_LIBRARY_PATH_CMD if BASEDIR is defined (i.e., not running with Spack)
 if ( $?BASEDIR ) then
     setenv @LD_LIBRARY_PATH_CMD ${@LD_LIBRARY_PATH_CMD}:${BASEDIR}/${ARCH}/lib
+    setenv PATH ${PATH}:${BASEDIR}/${ARCH}/bin
 endif
 
 setenv RUN_CMD "@RUN_CMD"
@@ -416,9 +417,14 @@ chmod +x linkbcs
 @SINGULARITY_BUILD    setenv GEOSEXE $GEOSBIN/GEOSgcm.x
 @SINGULARITY_BUILD endif
 
-@NATIVE_BUILD echo "Copying $EXPDIR/GEOSgcm.x to $SCRDIR"
-@NATIVE_BUILD echo ""
-@NATIVE_BUILD /bin/cp $EXPDIR/GEOSgcm.x $SCRDIR/GEOSgcm.x
+@NATIVE_BUILD if (-e $EXPDIR/GEOSgcm.x) then
+@NATIVE_BUILD    echo "Copying $EXPDIR/GEOSgcm.x to $SCRDIR"
+@NATIVE_BUILD    echo ""
+@NATIVE_BUILD    /bin/cp $EXPDIR/GEOSgcm.x $SCRDIR/GEOSgcm.x
+@NATIVE_BUILD else
+@NATIVE_BUILD    echo "$EXPDIR/GEOSgcm.x not found. Please link or copy the executable to the experiment directory."
+@NATIVE_BUILD    exit 1
+@NATIVE_BUILD endif
 @NATIVE_BUILD setenv GEOSEXE $SCRDIR/GEOSgcm.x
 
 #######################################################################
@@ -836,6 +842,16 @@ endif
 #ln -sf $SSTDIR/dataoceanfile_MERRA2_SST.${OGCM_IM}x${OGCM_JM}.${yy}.data sst.data
 #ln -sf $SSTDIR/dataoceanfile_MERRA2_ICE.${OGCM_IM}x${OGCM_JM}.${yy}.data fraci.data
 
+@CICE6 #detect exisistence of certain fields in CICE6 restart
+@CICE6 ncdump -h INPUT/iced.nc | grep 'apnd' > /dev/null
+@CICE6 if( $status == 0 ) then
+@CICE6    echo 'pond state in restart, turn on restart flag if not already'
+@CICE6    sed -i -E 's/^[[:space:]]*restart_pond_lvl[[:space:]]*=[[:space:]]*\.false\./    restart_pond_lvl  = .true./' ice_in
+@CICE6 else
+@CICE6    echo 'pond state NOT in restart, turn off restart flag if already on'
+@CICE6    sed -i -E 's/^[[:space:]]*restart_pond_lvl[[:space:]]*=[[:space:]]*\.true\./    restart_pond_lvl  = .false./' ice_in
+@CICE6 endif
+
 #######################################################################
 #                Split Saltwater Restart if detected
 #######################################################################
@@ -1059,7 +1075,7 @@ else
 endif
 
 @SINGULARITY_BUILD @OCEAN_PRELOAD $RUN_CMD $TOTAL_PES $SINGULARITY_RUN $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
-@NATIVE_BUILD @OCEAN_PRELOAD $RUN_CMD $TOTAL_PES $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
+@NATIVE_BUILD @OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $TOTAL_PES $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
 
 if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null
 

gcm_setup

@@ -2157,6 +2157,9 @@ set RESTART_BY_OSERVER = NO
 
 /bin/rm -f $HOMDIR/SETENV.commands
 
+# NAS has a "several_tries" script but we need an empty
+# default
+set SEVERAL_TRIES = ''
 
 if( $MPI_STACK == openmpi ) then
 
@@ -2206,6 +2209,10 @@ EOF
 
 else if( $MPI_STACK == mpt ) then
 
+# NAS recommends several_tries for MPT job issues
+# https://www.nas.nasa.gov/hecc/support/kb/mpt-startup-failures-workarounds_526.html
+set SEVERAL_TRIES = '/u/scicon/tools/bin/several_tries'
+
 cat > $HOMDIR/SETENV.commands << EOF
 
 setenv MPI_COLL_REPRODUCIBLE
@@ -2268,6 +2275,9 @@ setenv I_MPI_ADJUST_GATHERV 3
 
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
+
+# This has been found to help with congestion
+setenv FI_PSM3_CONN_TIMEOUT 120
 EOF
 
 endif # if NCCS
@@ -2433,6 +2443,7 @@ s/@USE_IOSERVER/$USE_IOSERVER/g
 s/@NUM_OSERVER_NODES/$NUM_OSERVER_NODES/g
 s/@NUM_BACKEND_PES/$NUM_BACKEND_PES/g
 s/@RESTART_BY_OSERVER/$RESTART_BY_OSERVER/g
+s#@SEVERAL_TRIES#$SEVERAL_TRIES#g
 s/@NCPUS_PER_NODE/$NCPUS_PER_NODE/g
 s/@NUM_READERS/$NUM_READERS/g
 s/@NUM_WRITERS/$NUM_WRITERS/g