Merging develop into master

Author: Nightly Bot

.github/workflows/cmake.yml

@@ -15,6 +15,7 @@ jobs:
           - { name: GNU, CC: gcc-10, CXX: g++-10 }
           - { name: LLVM, CC: clang, CXX: clang++ }
         build_type: [Debug, Release]
+        no_mpi: [OFF, ON]
 
     steps:
     - uses: actions/checkout@v4
@@ -30,7 +31,7 @@ jobs:
       env:
         MPICH_CXX: ${{matrix.compiler.CXX}}
         MPICH_CC: ${{matrix.compiler.CC}}
-      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install
+      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install -DSCOREC_NO_MPI=${{matrix.no_mpi}}
 
     - name: Build
       env:
@@ -54,3 +55,20 @@ jobs:
       run: |
         cmake -S ${{github.workspace}}/doc -B ${{github.workspace}}/buildExample -DCMAKE_CXX_COMPILER=mpicxx -DSCOREC_PREFIX=${{github.workspace}}/build/install
         cmake --build ${{github.workspace}}/buildExample
+
+    - name: Build MPI-NoMPI Example
+      # Test if a SCOREC_NO_MPI build works with MPI applications.
+      if: >-
+        matrix.compiler.name == 'GNU' && matrix.build_type == 'Release' &&
+        matrix.no_mpi == 'ON'
+      env:
+        MPICH_CXX: ${{matrix.compiler.CXX}}
+        MPICH_CC: ${{matrix.compiler.CC}}
+      run: >
+        cmake -S ${{github.workspace}}/example/mpi-nompi
+        -B ${{github.workspace}}/example/mpi-nompi/build
+        -DCMAKE_CXX_COMPILER=mpicxx
+        -DSCOREC_PREFIX=${{github.workspace}}/build/install ;
+        cmake --build ${{github.workspace}}/example/mpi-nompi/build ;
+        ctest --test-dir ${{github.workspace}}/example/mpi-nompi/build
+        --output-on-failure

.github/workflows/doxygen-gh-pages.yml

@@ -0,0 +1,41 @@
+name: Doxygen GitHub Pages Deploy Action
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  deploy:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      
+      - name: Install dependencies
+        run: |
+              sudo apt update
+              sudo apt install gcc-10 g++-10 mpich doxygen
+
+      - name: Configure CMake
+        env:
+          MPICH_CXX: g++-10
+          MPICH_CC: gcc-10
+        run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install -DSCOREC_NO_MPI=ON
+      
+      - name: Generate Doc
+        run: doxygen ${{github.workspace}}/build/Doxyfile
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./doc/html
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

CMakeLists.txt

@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.8)
 
-project(SCOREC VERSION 3.0.3 LANGUAGES CXX C)
+project(SCOREC VERSION 4.0.0 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)
@@ -22,11 +22,27 @@ endif()
 
 option(ENABLE_CGNS "Enable the CGNS reader: requires c++14 extensions" OFF)
 message(STATUS "ENABLE_CGNS: ${ENABLE_CGNS}")
-if(NOT ENABLE_CGNS)
+option(ENABLE_OMEGA_H "Enable the Omega_h interface" OFF)
+set(PUMI_USE_OMEGA_H_VERSION "10.0.0" CACHE STRING "Specify the Omega_h version PUMI should use")
+message(STATUS "ENABLE_OMEGA_H: ${ENABLE_OMEGA_H}")
+if(NOT ENABLE_CGNS AND NOT ENABLE_OMEGA_H)
+  message(STATUS "enabling cxx11")
   bob_set_cxx_standard(11)
-else()
+elseif(ENABLE_CGNS)
   message(STATUS "enabling cxx14")
   bob_set_cxx_standard(14)
+elseif(ENABLE_OMEGA_H)
+  message(STATUS "enabling cxx17")
+  bob_set_cxx_standard(17)
+endif()
+
+#
+# option related to MPI
+#
+option(SCOREC_NO_MPI "Disable MPI support.  When enabled only serial execution is supported." OFF)
+message(STATUS "SCOREC_NO_MPI: ${SCOREC_NO_MPI}")
+if(SCOREC_NO_MPI AND ENABLE_CGNS)
+  message(FATAL_ERROR "SCOREC_NO_MPI is incompatible with CGNS.")
 endif()
 
 # Set some default compiler flags that should always be used
@@ -114,9 +130,6 @@ option(ENABLE_SIMMETRIX "Build with Simmetrix support" OFF)
 message(STATUS "ENABLE_SIMMETRIX: ${ENABLE_SIMMETRIX}")
 option(ENABLE_CAPSTONE "Build with Capstone support" OFF)
 message(STATUS "ENABLE_CAPSTONE: ${ENABLE_CAPSTONE}")
-option(ENABLE_OMEGA_H "Enable the Omega_h interface" OFF)
-option(PUMI_USE_OMEGA_H_VERSION "Specify the Omega_h version PUMI should use" 10.0.0)
-message(STATUS "ENABLE_OMEGA_H: ${ENABLE_OMEGA_H}")
 
 if(ENABLE_SIMMETRIX)
   add_definitions(-DHAVE_SIMMETRIX)
@@ -133,7 +146,10 @@ endif()
 
 macro(scorec_export_library target)
 bob_export_target(${target})
-install(FILES ${HEADERS} DESTINATION include)
+install(FILES ${HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
+target_include_directories(${target} INTERFACE
+  "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+)
 endmacro(scorec_export_library)
 
 if(ENABLE_SIMMETRIX)
@@ -146,6 +162,8 @@ if(ENABLE_CAPSTONE)
 endif()
 
 if(ENABLE_OMEGA_H)
+  bob_cxx17_flags()
+  bob_set_cxx_standard(17)
   # find the omega_h library
   set(SCOREC_USE_Omega_h_DEFAULT ${ENABLE_OMEGA_H})
   set(Omega_h_REQUIRED_VERSION ${PUMI_USE_OMEGA_H_VERSION})
@@ -162,6 +180,10 @@ if(ENABLE_CGNS)
   add_definitions(-DHAVE_CGNS)
 endif()
 
+configure_file(SCOREC_config.h.in SCOREC_config.h)
+install(FILES "${CMAKE_BINARY_DIR}/SCOREC_config.h" DESTINATION include)
+include_directories(PUBLIC "$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>")
+
 # Include the SCOREC project packages
 add_subdirectory(lion)
 add_subdirectory(pcu)

SCOREC_config.h.in

@@ -0,0 +1,7 @@
+#ifndef SCOREC_CONFIG_H
+#define SCOREC_CONFIG_H
+
+#cmakedefine SCOREC_NO_MPI
+
+#endif // SCOREC_CONFIG_H
+

apf/CMakeLists.txt

@@ -93,7 +93,6 @@ add_library(apf ${SOURCES})
 # Include directories
 target_include_directories(apf PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 # Link this library to these others

apf/apfCGNS.cc

@@ -545,8 +545,9 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe
         cgp_error_exit();
 
       std::vector<int> allNumbersForThisType(m->getPCU()->Peers(), 0);
-      MPI_Allgather(&numbersByElementType[o], 1, MPI_INT, allNumbersForThisType.data(), 1,
-                    MPI_INT, m->getPCU()->GetMPIComm());
+      m->getPCU()->Allgather(
+        &numbersByElementType[o], allNumbersForThisType.data(), 1
+      );
 
       cgsize_t num = 0;
       for (int i = 0; i < m->getPCU()->Self(); i++)
@@ -656,8 +657,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
         }
 
         std::vector<int> allNumbersForThisType(m->getPCU()->Peers(), 0);
-        MPI_Allgather(&number, 1, MPI_INT, allNumbersForThisType.data(), 1,
-                      MPI_INT, m->getPCU()->GetMPIComm());
+        m->getPCU()->Allgather(&number, allNumbersForThisType.data(), 1);
 
         cgsize_t num = 0;
         for (int i = 0; i < m->getPCU()->Self(); i++)
@@ -684,19 +684,16 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
       }
     }
     std::vector<int> cacheStarts(m->getPCU()->Peers(), 0);
-    MPI_Allgather(&cacheStart, 1, MPI_INT, cacheStarts.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&cacheStart, cacheStarts.data(), 1);
     std::vector<int> cacheEnds(m->getPCU()->Peers(), 0);
-    MPI_Allgather(&cacheEnd, 1, MPI_INT, cacheEnds.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&cacheEnd, cacheEnds.data(), 1);
     return std::make_pair(cacheStarts, cacheEnds);
   };
 
   const auto globalElementList = [&m](const std::vector<cgsize_t> &bcList, std::vector<cgsize_t> &allElements) {
     std::vector<int> sizes(m->getPCU()->Peers(), 0); // important initialiser
     const int l = bcList.size();
-    MPI_Allgather(&l, 1, MPI_INT, sizes.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&l, sizes.data(), 1);
 
     int totalLength = 0;
     for (const auto &i : sizes)
@@ -708,9 +705,15 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
       displacement[i] = displacement[i - 1] + sizes[i - 1];
 
     allElements.resize(totalLength);
+    #ifndef SCOREC_NO_MPI
+    PCU_Comm comm;
+    m->getPCU()->DupComm(&comm);
     MPI_Allgatherv(bcList.data(), bcList.size(), MPI_INT, allElements.data(),
-                   sizes.data(), displacement.data(), MPI_INT,
-                   m->getPCU()->GetMPIComm());
+                   sizes.data(), displacement.data(), MPI_INT, comm);
+    MPI_Comm_free(&comm);
+    #else
+    std::copy(bcList.begin(), bcList.end(), allElements.begin());
+    #endif
   };
 
   const auto doVertexBC = [&](const auto &iter) {
@@ -1046,7 +1049,8 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
   sizes[2] = 0;                 // nodes are unsorted, as defined by api
 
   // Copy communicator
-  auto communicator = m->getPCU()->GetMPIComm();
+  PCU_Comm communicator;
+  m->getPCU()->DupComm(&communicator);
   cgp_mpi_comm(communicator);
   //
   cgp_pio_mode(CGP_INDEPENDENT);
@@ -1134,6 +1138,9 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
   destroyGlobalNumbering(gcn);
   //
   cgp_close(cgns.index);
+  #ifndef SCOREC_NO_MPI
+  MPI_Comm_free(&communicator);
+  #endif
 }
 } // namespace
 

apf_cap/CMakeLists.txt

@@ -39,7 +39,6 @@ include(GNUInstallDirs)
 
 target_include_directories(apf_cap PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
 
 scorec_export_library(apf_cap)

apf_sim/CMakeLists.txt

@@ -19,7 +19,6 @@ target_link_libraries(apf_sim PUBLIC ${SIMMODSUITE_LIBS})
 target_include_directories(gmi_sim PUBLIC ${SIMMODSUITE_INCLUDE_DIR})
 target_include_directories(apf_sim PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 option(ENABLE_FIELDSIM "Enable use of FieldSim from Simmetrix SimModSuite" FALSE)

can/CMakeLists.txt

@@ -11,7 +11,6 @@ add_library(can INTERFACE)
 # Include directories
 target_include_directories(can INTERFACE
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 scorec_export_library(can)

cdash/nightly.cmake

@@ -43,6 +43,9 @@ find_program(CTEST_GIT_COMMAND NAMES git)
 set(CTEST_UPDATE_COMMAND "${CTEST_GIT_COMMAND}")
 
 function(setup_repo)
+  if(EXISTS "${CTEST_SOURCE_DIRECTORY}/${CTEST_PROJECT_NAME}")
+    cleanup_merge(develop master)
+  endif()
   if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}/${CTEST_PROJECT_NAME}")
     message("Running \"git clone ${REPO_URL_BASE}.git ${CTEST_SOURCE_DIRECTORY}/${CTEST_PROJECT_NAME}\"")
     execute_process(COMMAND "${CTEST_GIT_COMMAND}" clone ${REPO_URL_BASE}.git
@@ -89,6 +92,18 @@ function(git_exec CMD ACTION)
   endif()
 endfunction(git_exec)
 
+function(remove_branch_if_exists BRANCH_NAME)
+  message("Attempting to delete ${BRANCH_NAME} if it exists")
+  execute_process(COMMAND "${CTEST_GIT_COMMAND}" branch --list ${BRANCH_NAME}
+      WORKING_DIRECTORY"${CTEST_SOURCE_DIRECTORY}/${CTEST_PROJECT_NAME}"
+      RESULT_VARIABLE BRANCH_RET)
+  message("git branch --list returned: ${BRANCH_RET})!")
+  if(BRANCH_RET)
+    message("${BRANCH_NAME} exists.  Attempting to delete it.")
+    git_exec("branch -D ${BRANCH_NAME}" "Deleted ${BRANCH_NAME}.")
+  endif()
+endfunction()
+
 function(create_branch BRANCH_NAME TRACKING_NAME)
   git_exec("branch --track ${BRANCH_NAME} ${TRACKING_NAME}"
            "Creating branch ${BRANCH_NAME}")
@@ -224,6 +239,7 @@ function(start_merge FIRST_BRANCH SECOND_BRANCH NEXT_ACTION)
   update_branch(${FIRST_BRANCH})
   update_branch(${SECOND_BRANCH})
   set(NEW_BRANCH "${SECOND_BRANCH}-into-${FIRST_BRANCH}")
+  remove_branch_if_exists(${NEW_BRANCH})
   create_branch(${NEW_BRANCH} origin/${FIRST_BRANCH})
   checkout_branch(${NEW_BRANCH})
   message("Running \"git merge --no-ff --no-commit ${SECOND_BRANCH}\"")