Skip to content

v11: Updates for startup and congestion #721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sdrabenh merged 3 commits into from
May 23, 2025
Merged

v11: Updates for startup and congestion #721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5dc9ab4
v11: Updates for startup and congestion
mathomp4 Apr 25, 2025
eb0704c
Fix bad sed
mathomp4 Apr 28, 2025
0d45168
Merge branch 'develop' into feature/network-congestion
mathomp4 May 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion gcm_forecast.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -766,7 +766,7 @@ else
endif

@SINGULARITY_BUILD @OCEAN_PRELOAD $RUN_CMD $TOTAL_PES $SINGULARITY_RUN $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
@NATIVE_BUILD @OCEAN_PRELOAD $RUN_CMD $TOTAL_PES $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
@NATIVE_BUILD @OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $TOTAL_PES $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'

if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null

Expand Down
8 changes: 4 additions & 4 deletions gcm_regress.j
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ if( $RUN_STARTSTOP == TRUE ) then

echo "=== Running test of duration ${test_duration_step1} with NX = $NX and NY = $NY starting at $nymd0 $nhms0 ==="

@OCEAN_PRELOAD $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
@OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'

set date = `cat cap_restart`
set nymde1 = $date[1]
Expand Down Expand Up @@ -517,7 +517,7 @@ set NY = `grep "^ *NY": AGCM.rc | cut -d':' -f2`

echo "=== Running test of duration ${test_duration_step2} with NX = $NX and NY = $NY starting at $nymd0 $nhms0 ==="

@OCEAN_PRELOAD $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
@OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'

set date = `cat cap_restart`
set nymde2 = $date[1]
Expand Down Expand Up @@ -623,7 +623,7 @@ if ($RUN_STARTSTOP == TRUE) then

echo "=== Running test of duration ${test_duration_step3} with NX = $NX and NY = $NY starting at $nymdb $nhmsb ==="

@OCEAN_PRELOAD $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
@OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'

set date = `cat cap_restart`
set nymde3 = $date[1]
Expand Down Expand Up @@ -737,7 +737,7 @@ if ( $RUN_LAYOUT == TRUE) then

echo "=== Running test of duration ${test_duration_step4} with NX = $test_NX and NY = $test_NY starting at $nymd0 $nhms0 ==="

@OCEAN_PRELOAD $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'
@OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml'

set date = `cat cap_restart`
set nymde4 = $date[1]
Expand Down
2 changes: 1 addition & 1 deletion gcm_run.j
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1059,7 +1059,7 @@ else
endif

@SINGULARITY_BUILD @OCEAN_PRELOAD $RUN_CMD $TOTAL_PES $SINGULARITY_RUN $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
@NATIVE_BUILD @OCEAN_PRELOAD $RUN_CMD $TOTAL_PES $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'
@NATIVE_BUILD @OCEAN_PRELOAD @SEVERAL_TRIES $RUN_CMD $TOTAL_PES $GEOSEXE $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml'

if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null

Expand Down
11 changes: 11 additions & 0 deletions gcm_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2157,6 +2157,9 @@ set RESTART_BY_OSERVER = NO

/bin/rm -f $HOMDIR/SETENV.commands

# NAS has a "several_tries" script but we need an empty
# default
set SEVERAL_TRIES = ''

if( $MPI_STACK == openmpi ) then

Expand Down Expand Up @@ -2206,6 +2209,10 @@ EOF

else if( $MPI_STACK == mpt ) then

# NAS recommends several_tries for MPT job issues
# https://www.nas.nasa.gov/hecc/support/kb/mpt-startup-failures-workarounds_526.html
set SEVERAL_TRIES = '/u/scicon/tools/bin/several_tries'

cat > $HOMDIR/SETENV.commands << EOF

setenv MPI_COLL_REPRODUCIBLE
Expand Down Expand Up @@ -2268,6 +2275,9 @@ setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3

# This has been found to help with congestion
setenv FI_PSM3_CONN_TIMEOUT 120
EOF

endif # if NCCS
Expand Down Expand Up @@ -2433,6 +2443,7 @@ s/@USE_IOSERVER/$USE_IOSERVER/g
s/@NUM_OSERVER_NODES/$NUM_OSERVER_NODES/g
s/@NUM_BACKEND_PES/$NUM_BACKEND_PES/g
s/@RESTART_BY_OSERVER/$RESTART_BY_OSERVER/g
s#@SEVERAL_TRIES#$SEVERAL_TRIES#g
s/@NCPUS_PER_NODE/$NCPUS_PER_NODE/g
s/@NUM_READERS/$NUM_READERS/g
s/@NUM_WRITERS/$NUM_WRITERS/g
Expand Down
11 changes: 11 additions & 0 deletions geoschemchem_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2187,6 +2187,9 @@ set RESTART_BY_OSERVER = NO

/bin/rm -f $HOMDIR/SETENV.commands

# NAS has a "several_tries" script but we need an empty
# default
set SEVERAL_TRIES = ''

if( $MPI_STACK == openmpi ) then

Expand Down Expand Up @@ -2236,6 +2239,10 @@ EOF

else if( $MPI_STACK == mpt ) then

# NAS recommends several_tries for MPT job issues
# https://www.nas.nasa.gov/hecc/support/kb/mpt-startup-failures-workarounds_526.html
set SEVERAL_TRIES = '/u/scicon/tools/bin/several_tries'

cat > $HOMDIR/SETENV.commands << EOF

setenv MPI_COLL_REPRODUCIBLE
Expand Down Expand Up @@ -2298,6 +2305,9 @@ setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3

# This has been found to help with congestion
setenv FI_PSM3_CONN_TIMEOUT 120
EOF

endif # if NCCS
Expand Down Expand Up @@ -2463,6 +2473,7 @@ s/@USE_IOSERVER/$USE_IOSERVER/g
s/@NUM_OSERVER_NODES/$NUM_OSERVER_NODES/g
s/@NUM_BACKEND_PES/$NUM_BACKEND_PES/g
s/@RESTART_BY_OSERVER/$RESTART_BY_OSERVER/g
s#@SEVERAL_TRIES#$SEVERAL_TRIES#g
s/@NCPUS_PER_NODE/$NCPUS_PER_NODE/g
s/@NUM_READERS/$NUM_READERS/g
s/@NUM_WRITERS/$NUM_WRITERS/g
Expand Down
11 changes: 11 additions & 0 deletions gmichem_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2359,6 +2359,9 @@ set RESTART_BY_OSERVER = NO

/bin/rm -f $HOMDIR/SETENV.commands

# NAS has a "several_tries" script but we need an empty
# default
set SEVERAL_TRIES = ''

if( $MPI_STACK == openmpi ) then

Expand Down Expand Up @@ -2408,6 +2411,10 @@ EOF

else if( $MPI_STACK == mpt ) then

# NAS recommends several_tries for MPT job issues
# https://www.nas.nasa.gov/hecc/support/kb/mpt-startup-failures-workarounds_526.html
set SEVERAL_TRIES = '/u/scicon/tools/bin/several_tries'

cat > $HOMDIR/SETENV.commands << EOF

setenv MPI_COLL_REPRODUCIBLE
Expand Down Expand Up @@ -2470,6 +2477,9 @@ setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3

# This has been found to help with congestion
setenv FI_PSM3_CONN_TIMEOUT 120
EOF

endif # if NCCS
Expand Down Expand Up @@ -2636,6 +2646,7 @@ s/@USE_IOSERVER/$USE_IOSERVER/g
s/@NUM_OSERVER_NODES/$NUM_OSERVER_NODES/g
s/@NUM_BACKEND_PES/$NUM_BACKEND_PES/g
s/@RESTART_BY_OSERVER/$RESTART_BY_OSERVER/g
s#@SEVERAL_TRIES#$SEVERAL_TRIES#g
s/@NCPUS_PER_NODE/$NCPUS_PER_NODE/g
s/@NUM_READERS/$NUM_READERS/g
s/@NUM_WRITERS/$NUM_WRITERS/g
Expand Down
11 changes: 11 additions & 0 deletions stratchem_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2172,6 +2172,9 @@ set RESTART_BY_OSERVER = NO

/bin/rm -f $HOMDIR/SETENV.commands

# NAS has a "several_tries" script but we need an empty
# default
set SEVERAL_TRIES = ''

if( $MPI_STACK == openmpi ) then

Expand Down Expand Up @@ -2221,6 +2224,10 @@ EOF

else if( $MPI_STACK == mpt ) then

# NAS recommends several_tries for MPT job issues
# https://www.nas.nasa.gov/hecc/support/kb/mpt-startup-failures-workarounds_526.html
set SEVERAL_TRIES = '/u/scicon/tools/bin/several_tries'

cat > $HOMDIR/SETENV.commands << EOF

setenv MPI_COLL_REPRODUCIBLE
Expand Down Expand Up @@ -2283,6 +2290,9 @@ setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3

# This has been found to help with congestion
setenv FI_PSM3_CONN_TIMEOUT 120
EOF

endif # if NCCS
Expand Down Expand Up @@ -2449,6 +2459,7 @@ s/@USE_IOSERVER/$USE_IOSERVER/g
s/@NUM_OSERVER_NODES/$NUM_OSERVER_NODES/g
s/@NUM_BACKEND_PES/$NUM_BACKEND_PES/g
s/@RESTART_BY_OSERVER/$RESTART_BY_OSERVER/g
s#@SEVERAL_TRIES#$SEVERAL_TRIES#g
s/@NCPUS_PER_NODE/$NCPUS_PER_NODE/g
s/@NUM_READERS/$NUM_READERS/g
s/@NUM_WRITERS/$NUM_WRITERS/g
Expand Down