Merge pull request #229 from GEOS-ESM/develop

GitFlow: Merge Develop into Main

Author: mathomp4

.github/workflows/validate_yaml_files.yml

@@ -0,0 +1,31 @@
+---
+
+# Based on code from https://github.com/marketplace/actions/yaml-lint
+
+name: Yaml Lint
+
+on:
+  pull_request:
+      types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+# This validation is equivalent to running on the command line:
+#   yamllint -d relaxed --no-warnings
+# and is controlled by the .yamllint.yml file
+jobs:
+  validate-YAML:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3.3.0
+      - id: yaml-lint
+        name: yaml-lint
+        uses: ibiqlik/action-yamllint@v3
+        with:
+          no_warnings: true
+          format: colored
+          config_file: .yamllint.yml
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: yamllint-logfile
+          path: ${{ steps.yaml-lint.outputs.logfile }}

.yamllint.yml

@@ -0,0 +1,29 @@
+---
+
+extends: default
+
+rules:
+  braces:
+    level: warning
+    max-spaces-inside: 1
+  brackets:
+    level: warning
+    max-spaces-inside: 1
+  colons:
+    level: warning
+  commas:
+    level: warning
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  empty-lines:
+    level: warning
+  hyphens:
+    level: warning
+  indentation:
+    level: warning
+    indent-sequences: consistent
+  line-length:
+    level: warning
+    allow-non-breakable-inline-mappings: true
+  truthy: disable

scripts/fv3.j

@@ -142,11 +142,11 @@ module list
 @SETENVS
 
          setenv EXETAG "$TAG"
-         setenv FV3EXE '@FV_PRECISION'
+         setenv FV3PRC '@FV_PRECISION'
          setenv EXPID   "c${AGCM_IM}_L${AGCM_LM}_T${N_TRACERS}_${NX}x${NY}_${N_OMP}threads"
          setenv EXPDSC  "c${AGCM_IM}_L${AGCM_LM}_T${N_TRACERS}_${NX}x${NY}_${N_OMP}threads"
          setenv EXPDIR  @EXPDIR
-         setenv SCRDIR  $EXPDIR/scratch_${EXPID}_${EXETAG}-${FV3EXE}
+         setenv SCRDIR  $EXPDIR/scratch_${EXPID}_${EXETAG}-${FV3PRC}
 if ($NH) setenv SCRDIR  ${SCRDIR}_NH.$$
 
 #######################################################################
@@ -172,6 +172,32 @@ cat >      ExtData.rc << EOF
 USE_EXTDATA: .false.
 EOF
 
+/bin/rm -f logging.yaml
+cat >      logging.yaml << EOF
+schema_version: 1
+locks:
+   mpi: {class: MpiLock, comm: MPI_COMM_WORLD}
+formatters:
+   plain: {class: Formatter, format: '%(message)a'}
+   basic: {class: Formatter, format: '%(short_name)a15~: %(level_name)a~: %(message)a'}
+   mpi: {class: MpiFormatter, format: '%(mpi_rank)i4.4~: %(name)~: %(level_name)a~: %(message)a', comm: MPI_COMM_WORLD}
+handlers:
+   console: {class: streamhandler, formatter: basic, unit: OUTPUT_UNIT, level: INFO}
+   console_plain: {class: streamhandler, formatter: plain, unit: OUTPUT_UNIT, level: INFO}
+   warnings: {class: FileHandler, filename: warnings_and_errors.log, lock: mpi, level: WARNING, formatter: basic}
+   errors: {class: StreamHandler, formatter: basic, unit: ERROR_UNIT, level: ERROR}
+   mpi_shared: {class: FileHandler, filename: allPEs.log, formatter: mpi, comm: MPI_COMM_WORLD, lock: mpi, rank_keyword: rank, level: DEBUG}
+root:
+   handlers: [warnings, errors, console]
+   level: WARNING
+   root_level: WARNING
+loggers:
+   errors: {handlers: [errors], level: ERROR}
+   CAP: {level: WARNING, root_level: INFO}
+   MAPL: {handlers: [mpi_shared], level: WARNING, root_level: INFO}
+   MAPL.profiler: {handlers: [console_plain], propagate: FALSE, level: WARNING, root_level: INFO}
+EOF
+
 /bin/rm -f CAP.rc
 cat >      CAP.rc << EOF
 MAPLROOT_COMPNAME: DYN
@@ -210,6 +236,7 @@ IOSERVER_NODES: @IOS_NDS
         DYN.LM: ${AGCM_LM}
   DYN.IM_WORLD: ${AGCM_IM}
 FV3_STANDALONE: 1
+    FV3_CONFIG: MONOTONIC
         DYCORE: FV3
      COLDSTART: 1
        CASE_ID: 1
@@ -331,7 +358,6 @@ if ($N_OMP > 1) then
    setenv KMP_AFFINITY compact
    setenv KMP_STACKSIZE 16m
 endif
-#env | grep MPI
 
 #######################################################################
 #          Settings for Singularity - EXPERIMENTAL
@@ -396,7 +422,6 @@ endif
 #                          Run the Model
 #######################################################################
 echo "  "
-#pwd
 echo "***** USING **** $FV3EXE *********************"
 
 if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null
@@ -407,7 +432,7 @@ else
    set IOSERVER_OPTIONS = ""
 endif
 
-$RUN_CMD $NPES $SINGULARITY_RUN $FV3EXE $IOSERVER_OPTIONS |& tee ${SCRDIR}.log
+$RUN_CMD $NPES $SINGULARITY_RUN $FV3EXE $IOSERVER_OPTIONS --logging_config logging.yaml |& tee ${SCRDIR}.log
 
 if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null
 

scripts/fv3_setup

@@ -317,8 +317,43 @@ else if ( $SITE == 'NAS' ) then
       set NCPUS_PER_NODE = 128
    endif
 
+else if( $SITE == 'AWS' | $SITE == 'Azure' ) then
+
+   # Because we do not know the name of the model or the number of CPUs
+   # per node. We ask the user to set these variables in the script
+
+   # AWS and Azure users must set the MODEL and NCPUS_PER_NODE
+   set MODEL = USER_MUST_SET
+   set NCPUS_PER_NODE = USER_MUST_SET
+
+   # Above we need a user to set the MODEL and NCPUS_PER_NODE
+   # variables. Here we check that they have been set. If not,
+   # we ask the user to set them
+   # --------------------------------------------------------
+
+   if ( $MODEL == USER_MUST_SET | $NCPUS_PER_NODE == USER_MUST_SET ) then
+      echo "ERROR: We have detected you are on $SITE. As we do not have"
+      echo "       official fixed node info yet, we ask you to edit $0"
+      echo "       and set the MODEL and NCPUS_PER_NODE variables."
+      echo "       Look for the section that says:"
+      echo " "
+      echo "       # AWS and Azure users must set the MODEL and NCPUS_PER_NODE"
+      exit 1
+   endif
+
 else
    set MODEL = 'UNKNOWN'
+   # As we do not know how many CPUs per node, we detect the number
+   # of CPUs per node by looking at the number of CPUs. This is different
+   # on Linux and macOS
+   if ( $ARCH == 'Linux' ) then
+      set NCPUS_PER_NODE = `grep -c ^processor /proc/cpuinfo`
+   else if ( $ARCH == 'Darwin' ) then
+      set NCPUS_PER_NODE = `sysctl -n hw.ncpu`
+   else
+      echo "ERROR: Unknown architecture $ARCH"
+      exit 1
+   endif
 endif
 
 # Set DEFAULT SHMEM Parameter
@@ -394,9 +429,9 @@ else
    set DEF_IOS_NDS = 5
 endif
 
-# On desktop, we need default to 6 processes
+# On desktop, we default to 6 processes
 # Must be set here due to MODEL_NPES calc below
-if( $SITE != 'NAS' && $SITE != 'NCCS' && $SITE != 'AWS') then
+if( $SITE != 'NAS' && $SITE != 'NCCS' && $SITE != 'AWS' && $SITE != 'Azure' ) then
    set FV_NX = 1
 endif
 
@@ -418,7 +453,7 @@ else
    set IOS_NDS = 0
 endif
 
-setenv     RUN_N  $EXPID # RUN Job Name
+setenv    RUN_N  $EXPID # RUN Job Name
 
 if( $SITE == 'NAS' ) then
    setenv RUN_N            `echo $EXPID | cut -b1-200`_RUN # RUN Job Name
@@ -455,7 +490,7 @@ else if( $SITE == 'NCCS' ) then
 
    setenv WRKDIR /discover/nobackup/$LOGNAME # user work directory
 
-else if( $SITE == 'AWS' ) then
+else if( $SITE == 'AWS' | $SITE == "Azure" ) then
    setenv BATCH_CMD        "sbatch"             # SLURM Batch command
    setenv BATCH_GROUP      "DELETE"             # SLURM Syntax for account name
    setenv BATCH_TIME       "SBATCH --time="     # SLURM Syntax for walltime
@@ -464,15 +499,13 @@ else if( $SITE == 'AWS' ) then
    setenv BATCH_JOINOUTERR "DELETE"             # SLURM joins out and err by default
    setenv RUN_T            "01:00:00"           # Wallclock Time   for fv3.j
 
-   setenv RUN_Q "DELETE"
-   setenv RUN_P "SBATCH --ntasks=${MODEL_NPES}" # PE Configuration for fv3.j
+   @ NODES  = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($IOS_NDS * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc`
 
-   # By default on AWS, just ignore IOSERVER for now until testing
-   set USE_IOSERVER   = 0
-   set IOS_NDS        = 0
-   set NCPUS_PER_NODE = 0
+   setenv RUN_Q "SBATCH --constraint=${MODEL}"                                # batch queue name for fv3.j
+   setenv RUN_P "SBATCH --nodes=${NODES} --ntasks-per-node=${NCPUS_PER_NODE}" # PE Configuration for fv3.j
 
    setenv WRKDIR /home/$LOGNAME # user work directory
+
 else
 # These are defaults for others (assume slurm)
    setenv BATCH_CMD        "sbatch"             # SLURM Batch command
@@ -489,7 +522,6 @@ else
    # By default on desktop, just ignore IOSERVER for now until testing
    set USE_IOSERVER   = 0
    set IOS_NDS        = 0
-   set NCPUS_PER_NODE = 0
 
    setenv WRKDIR /home/$LOGNAME # user work directory
 endif
@@ -751,7 +783,6 @@ foreach FILE ($FILES)
       exit 2
    endif
 
-
    sed -f $EXPDIR/sedfile $EXPDIR/tmpfile > $EXPDIR/$FILE
 
    echo "Creating ${C1}${FILE}${CN} for Experiment: $EXPID "
@@ -791,6 +822,9 @@ chmod   +x $EXPDIR/fv3.j
 echo "Done!"
 echo "-----"
 echo ""
+echo "You can find your experiment in the directory:"
+echo "      ${C2}${EXPDIR}${CN}"
+echo ""
 echo "NOTE: fv3.j by default will run StandAlone_FV3_Dycore.x from the installation directory:"
 echo "      $GEOSBIN"
 echo ""