Skip to content

Commit b840c52

Browse files
mathomp4mathomp4
authored
Merge pull request #251 from GEOS-ESM/develop
GitFlow: Merge Develop into main for release.
2 parents 19bd9b1 + ac865a9 commit b840c52

File tree

3 files changed

+146
-127
lines changed

3 files changed

+146
-127
lines changed

‎.circleci/config.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
version: 2.1
22

3-
# Anchors to prevent forgetting to update a version
4-
baselibs_version: &baselibs_version v7.13.0
5-
bcs_version: &bcs_version v11.00.0
3+
# Anchors in case we need to override the defaults from the orb
4+
#baselibs_version: &baselibs_version v7.14.0
5+
#bcs_version: &bcs_version v11.1.0
66

77

88
orbs:
@@ -18,7 +18,7 @@ workflows:
1818
matrix:
1919
parameters:
2020
compiler: [ifort, gfortran]
21-
baselibs_version: *baselibs_version
21+
#baselibs_version: *baselibs_version
2222
repo: GEOSgcm
2323
checkout_fixture: true
2424
mepodevelop: true
@@ -30,7 +30,7 @@ workflows:
3030
matrix:
3131
parameters:
3232
compiler: [gfortran, ifort]
33-
baselibs_version: *baselibs_version
33+
#baselibs_version: *baselibs_version
3434
requires:
3535
- build-GEOSgcm-on-<< matrix.compiler >>
3636
repo: GEOSgcm
@@ -42,7 +42,7 @@ workflows:
4242
parameters:
4343
compiler: [ifort, gfortran]
4444
fv_precision: ["R4", "R8"]
45-
baselibs_version: *baselibs_version
45+
#baselibs_version: *baselibs_version
4646
repo: GEOSfvdycore
4747
checkout_fixture: true
4848
mepodevelop: false
@@ -55,7 +55,7 @@ workflows:
5555
parameters:
5656
compiler: [gfortran, ifort]
5757
fv_precision: ["R4", "R8"]
58-
baselibs_version: *baselibs_version
58+
#baselibs_version: *baselibs_version
5959
requires:
6060
- build-GEOSfvdycore-as-<< matrix.fv_precision >>-on-<< matrix.compiler >>
6161
repo: GEOSfvdycore

‎scripts/fv3.j‎

Lines changed: 54 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -358,65 +358,58 @@ if ($N_OMP > 1) then
358358
setenv KMP_STACKSIZE 16m
359359
endif
360360

361-
#######################################################################
362-
# Settings for Singularity - EXPERIMENTAL
363-
#######################################################################
364-
365-
# Detect if StandAlone_FV3_Dycore.x is in the current directory
366-
# -------------------------------------------------------------
367-
368-
# If you are using singularity, set the path to the singularity sandbox here
369-
setenv SINGULARITY_SANDBOX @SINGULARITY_SANDBOX
370-
371-
# echo if we are running in singularity
372-
if( $SINGULARITY_SANDBOX != "" ) then
373-
echo "We are running under Singularity"
374-
echo ""
375-
endif
376-
377-
# Detect if StandAlone_FV3_Dycore.x is in the experiment directory
378-
if (-e $EXPDIR/StandAlone_FV3_Dycore.x) then
379-
echo "Found StandAlone_FV3_Dycore.x in $EXPDIR"
380-
381-
# If SINGULARITY_SANDBOX is non-empty and StandAlone_FV3_Dycore.x is found in the experiment directory,
382-
# force the use of StandAlone_FV3_Dycore.x in the installation directory
383-
if( $SINGULARITY_SANDBOX != "" ) then
384-
echo "NOTE: Testing has shown Singularity only works when running with"
385-
echo " the StandAlone_FV3_Dycore.x executable directly from the installation bin directory"
386-
echo ""
387-
echo " So, we will *ignore* the local StandAlone_FV3_Dycore.x and "
388-
echo " instead use $GEOSBIN/StandAlone_FV3_Dycore.x"
389-
echo ""
390-
391-
setenv FV3EXE $GEOSBIN/StandAlone_FV3_Dycore.x
392-
else
393-
echo "Copying $EXPDIR/StandAlone_FV3_Dycore.x to $SCRDIR"
394-
/bin/cp $EXPDIR/StandAlone_FV3_Dycore.x $SCRDIR/StandAlone_FV3_Dycore.x
395-
396-
setenv FV3EXE $SCRDIR/StandAlone_FV3_Dycore.x
397-
endif
398-
else
399-
echo "Using StandAlone_FV3_Dycore.x from $GEOSBIN"
400-
401-
setenv FV3EXE $GEOSBIN/StandAlone_FV3_Dycore.x
402-
endif
403-
echo ""
404-
405-
# If SINGULARITY_SANDBOX is non-empty, then run executable in singularity sandbox
406-
if( $SINGULARITY_SANDBOX != "" ) then
407-
# Load the Singularity module
408-
module load singularity
409-
410-
# Set Singularity Bind Paths. Note: These are dependent on where you are running.
411-
# By default, we'll assume you are running this script from NOBACKUP
412-
setenv REAL_BIND_PATH @REAL_BIND_PATH
413-
setenv SINGULARITY_BIND_PATH "-B ${NOBACKUP}:${NOBACKUP},${REAL_BIND_PATH}:${REAL_BIND_PATH}"
414-
415-
# Set a variable to encapsulate all Singularity details
416-
setenv SINGULARITY_RUN "singularity exec $SINGULARITY_BIND_PATH $SINGULARITY_SANDBOX"
417-
else
418-
setenv SINGULARITY_RUN ""
419-
endif
361+
@SINGULARITY_BUILD #######################################################################
362+
@SINGULARITY_BUILD # Settings for Singularity - EXPERIMENTAL
363+
@SINGULARITY_BUILD #######################################################################
364+
@SINGULARITY_BUILD
365+
@SINGULARITY_BUILD # Detect if StandAlone_FV3_Dycore.x is in the current directory
366+
@SINGULARITY_BUILD # -------------------------------------------------------------
367+
@SINGULARITY_BUILD
368+
@SINGULARITY_BUILD # If you are using singularity, set the path to the singularity sandbox here
369+
@SINGULARITY_BUILD setenv SINGULARITY_SANDBOX @SINGULARITY_SANDBOX
370+
@SINGULARITY_BUILD
371+
@SINGULARITY_BUILD echo "We are running under Singularity"
372+
@SINGULARITY_BUILD echo ""
373+
@SINGULARITY_BUILD
374+
@SINGULARITY_BUILD # Detect if StandAlone_FV3_Dycore.x is in the experiment directory
375+
@SINGULARITY_BUILD if (-e $EXPDIR/StandAlone_FV3_Dycore.x) then
376+
@SINGULARITY_BUILD echo "Found StandAlone_FV3_Dycore.x in $EXPDIR"
377+
@SINGULARITY_BUILD
378+
@SINGULARITY_BUILD echo "NOTE: Testing has shown Singularity only works when running with"
379+
@SINGULARITY_BUILD echo " the StandAlone_FV3_Dycore.x executable directly from the installation bin directory"
380+
@SINGULARITY_BUILD echo ""
381+
@SINGULARITY_BUILD echo " So, we will *ignore* the local StandAlone_FV3_Dycore.x and "
382+
@SINGULARITY_BUILD echo " instead use $GEOSBIN/StandAlone_FV3_Dycore.x"
383+
@SINGULARITY_BUILD echo ""
384+
@SINGULARITY_BUILD endif
385+
@SINGULARITY_BUILD
386+
@SINGULARITY_BUILD echo "Using StandAlone_FV3_Dycore.x from $GEOSBIN"
387+
@SINGULARITY_BUILD setenv FV3EXE $GEOSBIN/StandAlone_FV3_Dycore.x
388+
@SINGULARITY_BUILD
389+
@SINGULARITY_BUILD # Load the Singularity module
390+
@SINGULARITY_BUILD module load singularity
391+
@SINGULARITY_BUILD
392+
@SINGULARITY_BUILD # Set Singularity Bind Paths. Note: These are dependent on where you are running.
393+
@SINGULARITY_BUILD # By default, we'll assume you are running this script from NOBACKUP
394+
@SINGULARITY_BUILD setenv REAL_BIND_PATH @REAL_BIND_PATH
395+
@SINGULARITY_BUILD setenv SINGULARITY_BIND_PATH "-B ${NOBACKUP}:${NOBACKUP},${REAL_BIND_PATH}:${REAL_BIND_PATH}"
396+
@SINGULARITY_BUILD
397+
@SINGULARITY_BUILD # Set a variable to encapsulate all Singularity details
398+
@SINGULARITY_BUILD setenv SINGULARITY_RUN "singularity exec $SINGULARITY_BIND_PATH $SINGULARITY_SANDBOX"
399+
400+
@NATIVE_BUILD # Detect if StandAlone_FV3_Dycore.x is in the experiment directory
401+
@NATIVE_BUILD if (-e $EXPDIR/StandAlone_FV3_Dycore.x) then
402+
@NATIVE_BUILD echo "Found StandAlone_FV3_Dycore.x in $EXPDIR"
403+
@NATIVE_BUILD
404+
@NATIVE_BUILD echo "Copying $EXPDIR/StandAlone_FV3_Dycore.x to $SCRDIR"
405+
@NATIVE_BUILD /bin/cp $EXPDIR/StandAlone_FV3_Dycore.x $SCRDIR/StandAlone_FV3_Dycore.x
406+
@NATIVE_BUILD
407+
@NATIVE_BUILD setenv FV3EXE $SCRDIR/StandAlone_FV3_Dycore.x
408+
@NATIVE_BUILD else
409+
@NATIVE_BUILD echo "Using StandAlone_FV3_Dycore.x from $GEOSBIN"
410+
@NATIVE_BUILD
411+
@NATIVE_BUILD setenv FV3EXE $GEOSBIN/StandAlone_FV3_Dycore.x
412+
@NATIVE_BUILD endif
420413

421414
#######################################################################
422415
# Run the Model
@@ -432,7 +425,8 @@ else
432425
set IOSERVER_OPTIONS = ""
433426
endif
434427

435-
$RUN_CMD $NPES $SINGULARITY_RUN $FV3EXE $IOSERVER_OPTIONS --logging_config logging.yaml |& tee ${SCRDIR}.log
428+
@SINGULARITY_BUILD $RUN_CMD $NPES $SINGULARITY_RUN $FV3EXE $IOSERVER_OPTIONS --logging_config logging.yaml |& tee ${SCRDIR}.log
429+
@NATIVE_BUILD $RUN_CMD $NPES $FV3EXE $IOSERVER_OPTIONS --logging_config logging.yaml |& tee ${SCRDIR}.log
436430

437431
if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null
438432

‎scripts/fv3_setup‎

Lines changed: 85 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ endif
3434
# Set Current Working Path to fv3_setup
3535
# -------------------------------------
3636
setenv ARCH `uname -s`
37+
setenv NODE `uname -n`
38+
3739
if ($ARCH == Darwin) then
3840
set FINDPATH = realpath
3941
else
@@ -52,6 +54,56 @@ if (! -x ${BINDIR}/StandAlone_FV3_Dycore.x) then
5254
exit 1
5355
endif
5456

57+
######################################################################
58+
# Singularity Sandbox
59+
#######################################################################
60+
61+
# This section is to determine if we are running in a Singularity
62+
# sandbox by looking for the .singularity.d directory. If we are,
63+
# then we set the SINGULARITY_SANDBOX environment variable to the
64+
# path to the sandbox. If we are not, then we set it to an empty
65+
# string.
66+
#
67+
# We also set the REAL_BIND_PATH environment variable. This is used
68+
# to set the bind paths for Singularity runs. REAL_BIND_PATH is the
69+
# physical path to the NOBACKUP directory
70+
71+
setenv KEYFILE ".singularity.d"
72+
setenv singstat 0
73+
setenv ORIGPATH `pwd -P`
74+
cd $GEOSDEF
75+
while ($singstat == 0)
76+
if (-d $KEYFILE) then
77+
setenv singstat 1
78+
setenv SINGPATH `pwd -P`
79+
endif
80+
if ($singstat == 0) then
81+
if ("$cwd" == "/") then
82+
break
83+
else
84+
cd ..
85+
endif
86+
endif
87+
end
88+
89+
if ($singstat == 1) then
90+
setenv SINGULARITY_SANDBOX $SINGPATH
91+
setenv REAL_BIND_PATH `realpath $NOBACKUP`
92+
93+
set USING_SINGULARITY = TRUE
94+
set SINGULARITY_BUILD = ""
95+
set NATIVE_BUILD = "#DELETE"
96+
else
97+
setenv SINGULARITY_SANDBOX ""
98+
setenv REAL_BIND_PATH ""
99+
100+
set USING_SINGULARITY = FALSE
101+
set SINGULARITY_BUILD = "#DELETE"
102+
set NATIVE_BUILD = ""
103+
endif
104+
105+
cd $ORIGPATH
106+
55107
# Set TMPDIR to /tmp due to issues with heredocs in Singularity sandboxes
56108
# -----------------------------------------------------------------------
57109

@@ -61,7 +113,24 @@ setenv TMPDIR /tmp
61113
# Test for Command Line Flags
62114
#######################################################################
63115

64-
set USER_SITE = ""
116+
set SINGULARITY_RUN_SITE = ""
117+
if ($singstat == 1) then
118+
set USING_SINGULARITY = TRUE
119+
120+
# If we are using Singularity, then where the image was built (say
121+
# AWS) might not match where we run. We at least know NCCS and NAS,
122+
# so we can use that to set the site.
123+
124+
if (($NODE =~ discover*) || ($NODE =~ borg*) || ($NODE =~ warp*)) then
125+
set SINGULARITY_RUN_SITE = "NCCS"
126+
else if (($NODE =~ pfe*) || \
127+
($NODE =~ r[0-9]*i[0-9]*n[0-9]*) || \
128+
($NODE =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*)) then
129+
set SINGULARITY_RUN_SITE = "NAS"
130+
endif
131+
else
132+
set USING_SINGULARITY = FALSE
133+
endif
65134
while ( $#argv > 0 )
66135
set arg = $argv[1]
67136
shift argv
@@ -71,23 +140,6 @@ while ( $#argv > 0 )
71140
case --[Cc][Oo][Ll][Oo][Rr]:
72141
goto SETCOLOR
73142

74-
# Testing with singularity found we need the ability to override
75-
# the SITE variable below because images might be built on, say,
76-
# AWS and thus "block" us from running at NCCS.
77-
#
78-
# So, we make a --site flag that allows us to override the SITE
79-
# variable
80-
case -[Ss]:
81-
case --[Ss][Ii][Tt][Ee]:
82-
set USER_SITE = $1
83-
# if USER_SITE is empty, then we error out
84-
if ( "$USER_SITE" == "" ) then
85-
echo "ERROR: --site flag requires a site name"
86-
exit 1
87-
endif
88-
shift argv
89-
breaksw
90-
91143
# Here any string not above will trigger USAGE
92144
case -[Hh]:
93145
case --[Hh][Ee][Ll][Pp]:
@@ -100,12 +152,10 @@ end
100152
# Determine site
101153
#######################################################################
102154

103-
setenv NODE `uname -n`
104-
setenv ARCH `uname -s`
105-
if ($USER_SITE == "") then
155+
if ($SINGULARITY_RUN_SITE == "") then
106156
setenv SITE `awk '{print $2}' $ETCDIR/SITE.rc`
107157
else
108-
setenv SITE $USER_SITE
158+
setenv SITE $SINGULARITY_RUN_SITE
109159
endif
110160

111161
#######################################################################
@@ -257,7 +307,6 @@ ASKPROC:
257307

258308
if ( $SITE == 'NCCS' ) then
259309
echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
260-
echo " ${C2}hasw (Haswell)${CN}"
261310
echo " ${C2}sky (Skylake)${CN}"
262311
echo " ${C2}cas (Cascade Lake)${CN} (default)"
263312
echo " "
@@ -267,13 +316,10 @@ if ( $SITE == 'NCCS' ) then
267316
set MODEL = 'cas'
268317
endif
269318

270-
if( $MODEL != 'hasw' & \
271-
$MODEL != 'sky' & \
319+
if( $MODEL != 'sky' & \
272320
$MODEL != 'cas' ) goto ASKPROC
273321

274-
if ( $MODEL == 'hasw') then
275-
set NCPUS_PER_NODE = 28
276-
else if ($MODEL == 'sky') then
322+
if ($MODEL == 'sky') then
277323
set NCPUS_PER_NODE = 40
278324
else if ($MODEL == 'cas') then
279325
# NCCS currently recommends that users do not run with
@@ -608,38 +654,6 @@ echo $EXPDIRroot > $HOME/.EXPDIRroot
608654

609655
set GEOSDIR = $GEOSDEF
610656

611-
######################################################################
612-
# Singularity Sandbox
613-
#######################################################################
614-
615-
setenv KEYFILE ".singularity.d"
616-
setenv singstat 0
617-
setenv ORIGPATH `pwd -P`
618-
cd $GEOSDIR
619-
while ($singstat == 0)
620-
if (-d $KEYFILE) then
621-
setenv singstat 1
622-
setenv SINGPATH `pwd -P`
623-
endif
624-
if ($singstat == 0) then
625-
if ("$cwd" == "/") then
626-
break
627-
else
628-
cd ..
629-
endif
630-
endif
631-
end
632-
if ($singstat == 1) then
633-
setenv SINGULARITY_SANDBOX $SINGPATH
634-
else
635-
setenv SINGULARITY_SANDBOX ""
636-
endif
637-
638-
cd $ORIGPATH
639-
640-
setenv REAL_BIND_PATH `realpath $NOBACKUP`
641-
642-
643657
#
644658
# GEOSBIN does point to the bin/ directory in each
645659
#
@@ -751,10 +765,19 @@ if ( $SITE == 'NCCS' ) then
751765
cat >> $EXPDIR/SETENV.commands << EOF
752766
setenv I_MPI_SHM_HEAP_VSIZE 512
753767
setenv PSM2_MEMORY large
768+
EOF
769+
770+
#Testing at NCCS showed these caused crash at higher res
771+
#Crashed at restart read
772+
if ( $USING_SINGULARITY == FALSE ) then
773+
774+
cat >> $EXPDIR/SETENV.commands << EOF
754775
setenv I_MPI_EXTRA_FILESYSTEM 1
755776
setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs
756777
EOF
757778

779+
endif # if NOT Singularity
780+
758781
endif # if NCCS
759782

760783
endif # if mpi
@@ -790,6 +813,8 @@ s?@GEOSSRC?$GEOSSRC?g
790813
s?@GEOSBIN?$GEOSBIN?g
791814
s?@GEOSETC?$GEOSETC?g
792815
s?@GEOSUTIL?$GEOSUTIL?g
816+
s?@SINGULARITY_BUILD?$SINGULARITY_BUILD?g
817+
s?@NATIVE_BUILD?$NATIVE_BUILD?g
793818
s?@SINGULARITY_SANDBOX?$SINGULARITY_SANDBOX?g
794819
s?@REAL_BIND_PATH?$REAL_BIND_PATH?g
795820

0 commit comments

Comments
 (0)