Merge pull request #136 from GEOS-ESM/develop

Merge develop into main for release

Author: mathomp4

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 
 executors:
-  gcc-build-env:
+  gfortran:
     docker:
       - image: gmao/ubuntu20-geos-env-mkl:v6.2.7-openmpi_4.0.6-gcc_11.2.0
         auth:
@@ -11,73 +11,170 @@ executors:
       OMPI_ALLOW_RUN_AS_ROOT: 1
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
       OMPI_MCA_btl_vader_single_copy_mechanism: none
+      MPIEXEC_PREFLAGS: --oversubscribe
+    resource_class: large
+    #MEDIUM# resource_class: medium
+
+  ifort:
+    docker:
+      - image: gmao/ubuntu20-geos-env:v6.2.7-intelmpi_2021.2.0-intel_2021.2.0
+        auth:
+          username: $DOCKERHUB_USER
+          password: $DOCKERHUB_AUTH_TOKEN
     resource_class: large
     #MEDIUM# resource_class: medium
 
 workflows:
-  version: 2.1
-  build-and-test:
+  build-test:
     jobs:
       - build-GEOSgcm:
+          name: build-GEOSgcm-on-<< matrix.compiler >>
+          matrix:
+            parameters:
+              compiler: [gfortran, ifort]
           context: 
             - docker-hub-creds
-      - make-FV3-experiment:
-          context: 
+      - make-FV3-exp:
+          name: make-FV3-exp-on-<< matrix.compiler >>
+          matrix:
+            parameters:
+              compiler: [gfortran, ifort]
+          context:
             - docker-hub-creds
           requires:
-            - build-GEOSgcm
-      - run-FV3-standalone:
-          context: 
+            - build-GEOSgcm-on-<< matrix.compiler >>
+      - run-FV3:
+          name: run-FV3-on-<< matrix.compiler >>
+          matrix:
+            parameters:
+              compiler: [gfortran, ifort]
+          context:
             - docker-hub-creds
           requires:
-            - make-FV3-experiment
+            - make-FV3-exp-on-<< matrix.compiler >>
+
+
+
+commands:
+  versions:
+    description: "Versions, etc."
+    parameters:
+      compiler:
+        type: string
+    steps:
+      - run: mpirun --version && << parameters.compiler >> --version && echo $BASEDIR && pwd && ls && echo "$(nproc)"
+
+  checkout_fixture:
+    description: "Checkout fixture"
+    parameters:
+      repo:
+        type: string
+    steps:
+      - run: |
+          cd ${CIRCLE_WORKING_DIRECTORY}
+          git clone https://github.com/GEOS-ESM/<< parameters.repo >>.git
+
+  mepoclone:
+    description: "Mepo clone external repos"
+    parameters:
+      repo:
+        type: string
+    steps:
+      - run: |
+          cd ${CIRCLE_WORKING_DIRECTORY}/<< parameters.repo >>
+          mepo clone
+          mepo status
+
+  mepodevelop:
+    description: "Mepo develop GEOSgcm_GridComp GEOSgcm_App GMAO_Shared"
+    parameters:
+      repo:
+        type: string
+    steps:
+      - run: |
+          cd ${CIRCLE_WORKING_DIRECTORY}/<< parameters.repo >>
+          mepo develop GEOSgcm_GridComp GEOSgcm_App GMAO_Shared
+          mepo status
+
+  checkout_feature_branch:
+    description: "Mepo checkout-if-exists feature branch"
+    parameters:
+      repo:
+        type: string
+    steps:
+      - run: |
+          cd ${CIRCLE_WORKING_DIRECTORY}/<< parameters.repo >>
+          echo "${CIRCLE_BRANCH}"
+          if [ "${CIRCLE_BRANCH}" != "develop" ] && [ "${CIRCLE_BRANCH}" != "main" ]
+          then
+              mepo checkout-if-exists ${CIRCLE_BRANCH}
+          fi
+          mepo status
+
+  cmake:
+    description: "Run CMake"
+    parameters:
+      repo:
+        type: string
+      compiler:
+        type: string
+    steps:
+      - run: |
+          mkdir -p /logfiles
+          cd ${CIRCLE_WORKING_DIRECTORY}/<< parameters.repo >>
+          mkdir -p ${CIRCLE_WORKING_DIRECTORY}/workspace/build-<< parameters.repo >>
+          cd ${CIRCLE_WORKING_DIRECTORY}/workspace/build-<< parameters.repo >>
+          cmake ${CIRCLE_WORKING_DIRECTORY}/<< parameters.repo >> -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=<< parameters.compiler >> -DCMAKE_BUILD_TYPE=Debug -DUSE_F2PY=OFF -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} -DCMAKE_INSTALL_PREFIX=${CIRCLE_WORKING_DIRECTORY}/workspace/install-<< parameters.repo >> |& tee /logfiles/cmake.log
+
+  buildinstall:
+    description: "Build and install"
+    parameters:
+      repo:
+        type: string
+    steps:
+      - run: |
+          cd ${CIRCLE_WORKING_DIRECTORY}/workspace/build-<< parameters.repo >>
+          make -j"$(nproc)" install |& tee /logfiles/make.log
+          #MEDIUM# make -j4 install |& tee /logfiles/make.log
 
 jobs:
   build-GEOSgcm:
-    executor: gcc-build-env
+    parameters:
+      compiler:
+        type: string
+    executor: << parameters.compiler >>
     working_directory: /root/project
     steps:
       - run:
-          name: "FVdycoreCubed_GridComp branch"
+          name: "GMAO_Shared branch"
           command: echo ${CIRCLE_BRANCH}
-      - run:
-          name: "Checkout GEOSgcm fixture"
-          command: |
-            cd ${CIRCLE_WORKING_DIRECTORY}
-            git clone https://github.com/GEOS-ESM/GEOSgcm.git
-      - run:
-          name: "Mepo clone GEOSgcm fixture and update FVdycoreCubed_GridComp branch"
-          command: |
-            cd ${CIRCLE_WORKING_DIRECTORY}/GEOSgcm
-            mepo clone
-            mepo develop GEOSgcm_GridComp GEOSgcm_App
-            if [ "${CIRCLE_BRANCH}" != "develop" ] && [ "${CIRCLE_BRANCH}" != "master" ] && [ "${CIRCLE_BRANCH}" != "main" ]
-            then
-               mepo checkout-if-exists ${CIRCLE_BRANCH}
-            fi
-            mepo status
-      - run:
-          name: "CMake"
-          command: |
-            cd ${CIRCLE_WORKING_DIRECTORY}/GEOSgcm
-            mkdir build
-            cd build
-            cmake .. -DBASEDIR=$BASEDIR/Linux -DCMAKE_Fortran_COMPILER=gfortran -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=${CIRCLE_WORKING_DIRECTORY}/workspace/install-GEOSgcm -DUSE_F2PY=OFF
-      - run:
-          name: "Build and install"
-          command: |
-            cd ${CIRCLE_WORKING_DIRECTORY}/GEOSgcm/build
-            make -j"$(nproc)" install
-            #MEDIUM# make -j4 install
-
-      # We need to persist the install for the next step
+      - checkout_fixture:
+          repo: GEOSgcm
+      - versions:
+          compiler: << parameters.compiler >>
+      - mepoclone:
+          repo: GEOSgcm
+      - mepodevelop:
+          repo: GEOSgcm
+      - checkout_feature_branch:
+          repo: GEOSgcm
+      - cmake:
+          repo: GEOSgcm
+          compiler: << parameters.compiler >>
+      - buildinstall:
+          repo: GEOSgcm
+      - store_artifacts:
+          path: /logfiles
       - persist_to_workspace:
           root: workspace
           paths:
             - install-GEOSgcm
 
-  make-FV3-experiment:
-    executor: gcc-build-env
+  make-FV3-exp:
+    parameters:
+      compiler:
+        type: string
+    executor: << parameters.compiler >>
     working_directory: /root/project
     steps:
       - attach_workspace:
@@ -118,17 +215,21 @@ jobs:
           paths:
             - test-fv3-c12
 
-  run-FV3-standalone:
-    executor: gcc-build-env
+  run-FV3:
+    parameters:
+      compiler:
+        type: string
+    executor: << parameters.compiler >>
     working_directory: /root/project
     steps:
       - attach_workspace:
           at: workspace
       - run:
           name: "Run fv3.j"
           command: |
+            mkdir -p /logfiles
             cd ${CIRCLE_WORKING_DIRECTORY}/workspace/test-fv3-c12
-            ./fv3.j
+            ./fv3.j |& tee /logfiles/fv3_run.log
       - run:
           name: "Check for EGRESS"
           command: |
@@ -144,9 +245,6 @@ jobs:
                echo "EGRESS not found!"
                exit 1
             fi
-      - run:
-          name: "Cat log file"
-          command: |
-            cd ${CIRCLE_WORKING_DIRECTORY}/workspace/test-fv3-c12
+      - store_artifacts:
+          path: /logfiles
 
-            cat *.log

CMakeLists.txt

@@ -56,6 +56,9 @@ if (CRAY_POINTER)
   set_target_properties (${this} PROPERTIES COMPILE_FLAGS ${CRAY_POINTER})
 endif()
 
+# Enable OpenMP
+target_link_libraries(${this} PRIVATE OpenMP::OpenMP_Fortran)
+
 add_definitions (-DSPMD -DMAPL_MODE)
 
 foreach(flag ${tmp})