Merge branch 'develop' into yus/ci_petsc

Author: Sichao25

.github/actions/install-repo/action.yml

@@ -16,21 +16,28 @@ runs:
   using: "composite"
   steps:
 
-  - name: Check Cache
-    if: ${{ inputs.cache == 'true'}}
-    uses: actions/cache@v3
-    id: check-cache
-    with:
-      key: build-${{ inputs.repo-name }}-${{ inputs.repo-ref}}
-      path: ${{ runner.temp }}/build-${{ inputs.repo-name }}
-      
   - name: Checkout Repo
     uses: actions/checkout@v3
     with:
       repository: ${{inputs.repo-path }}
       ref: ${{ inputs.repo-ref }}
       path: ${{ inputs.repo-name }}
 
+  - name: Get commit hash
+    id: get-hash
+    shell: bash
+    run: |
+      cd $GITHUB_WORKSPACE/${{ inputs.repo-name }}
+      echo "commit-hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+  - name: Check Cache
+    if: ${{ inputs.cache == 'true'}}
+    uses: actions/cache@v3
+    id: check-cache
+    with:
+      key: build-${{ inputs.repo-name }}-${{ steps.get-hash.outputs.commit-hash }}
+      path: ${{ runner.temp }}/build-${{ inputs.repo-name }}
+
   - name: Configure CMake
     if: ${{ !steps.check-cache.outputs.cache-hit }}
     shell: bash

.github/workflows/clang-tidy.yml

@@ -82,7 +82,7 @@ jobs:
       with:
         repo-name: 'omega_h-openmpi'
         repo-path: 'SCOREC/omega_h'
-        repo-ref: 'scorec-v11.0.0'
+        repo-ref: '1765836a00b9a64b8b3791f1442ac52f147e43b2'
         cache: true
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DCMAKE_C_COMPILER=`which mpicc`
@@ -98,7 +98,7 @@ jobs:
       with:
         repo-name: 'meshFields-openmpi'
         repo-path: 'SCOREC/meshFields'
-        repo-ref: ''
+        repo-ref: 'b1482bbba288df210784b2345eae08e34faabdc4'
         cache: true
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DCMAKE_C_COMPILER=`which mpicc`
@@ -140,20 +140,26 @@ jobs:
         cache: true
         options: '-DADIOS2_USE_CUDA=OFF'
 
+    - name: clone petsc
+      id: clone-petsc
+      run: |
+        cd ${{ runner.temp }}
+        git clone -b v3.24.2 https://gitlab.com/petsc/petsc.git petsc-openmpi
+        cd petsc-openmpi
+        echo "petsc-commit-hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
     - name: Cache PETSc
       id: cache-petsc
       uses: actions/cache@v3
       with:
         path: ${{ runner.temp }}/petsc-openmpi
-        key: build-petsc-openmpi
+        key: build-petsc-openmpi-${{ steps.clone-petsc.outputs.petsc-commit-hash }}
 
     # PETSc check is removed due to a PETSc test failure using openmpi
     - name: build petsc
       if: steps.cache-petsc.outputs.cache-hit != 'true'
       run: |
-        cd ${{ runner.temp }}
-        git clone -b v3.24.2 https://gitlab.com/petsc/petsc.git petsc-openmpi
-        cd petsc-openmpi
+        cd ${{ runner.temp }}/petsc-openmpi
         ./configure \
           PETSC_ARCH=ubuntu-kokkos \
           --with-kokkos-dir="${{ runner.temp }}/build-kokkos-openmpi/install/" \
@@ -168,7 +174,7 @@ jobs:
       with:
         repo-name: 'redev-openmpi'
         repo-path: 'SCOREC/redev'
-        repo-ref: 'v4.3.1'
+        repo-ref: '1452ec290dc6f8638019e342758325611e16ad77'
         cache: true
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DMPIEXEC_EXECUTABLE=`which mpirun`

.github/workflows/cmake-test.yml

@@ -82,7 +82,7 @@ jobs:
       with:
         repo-name: 'omega_h'
         repo-path: 'SCOREC/omega_h'
-        repo-ref: 'scorec-v11.0.0'
+        repo-ref: '1765836a00b9a64b8b3791f1442ac52f147e43b2'
         cache: true
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DCMAKE_C_COMPILER=`which mpicc`
@@ -98,7 +98,7 @@ jobs:
       with:
         repo-name: 'meshFields'
         repo-path: 'SCOREC/meshFields'
-        repo-ref: ''
+        repo-ref: 'b1482bbba288df210784b2345eae08e34faabdc4'
         cache: true
         options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DCMAKE_C_COMPILER=`which mpicc`
@@ -154,19 +154,25 @@ jobs:
                   -DADIOS2_DIR=${{ runner.temp }}/build-ADIOS2/install/lib/cmake/adios2
                   -Dperfstubs_DIR=${{ runner.temp }}/build-perfstubs/install/lib/cmake'
 
+    - name: clone petsc
+      id: clone-petsc
+      run: |
+        cd ${{ runner.temp }}
+        git clone -b v3.24.2 https://gitlab.com/petsc/petsc.git petsc
+        cd petsc
+        echo "petsc-commit-hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
     - name: Cache PETSc
       id: cache-petsc
       uses: actions/cache@v3
       with:
         path: ${{ runner.temp }}/petsc
-        key: build-petsc
+        key: build-petsc-${{ steps.clone-petsc.outputs.petsc-commit-hash }}
 
     - name: build petsc
       if: steps.cache-petsc.outputs.cache-hit != 'true'
       run: |
-        cd ${{ runner.temp }}
-        git clone -b v3.24.2 https://gitlab.com/petsc/petsc.git petsc
-        cd petsc
+        cd ${{ runner.temp }}/petsc
         ./configure \
           PETSC_ARCH=ubuntu-kokkos \
           --with-kokkos-dir="${{ runner.temp }}/build-kokkos/install/" \
@@ -221,16 +227,25 @@ jobs:
       run: ctest --test-dir ${{ runner.temp }}/build-pcms --output-on-failure
 
     - name: Print Test
+      if: always()
       run: cat ${{ runner.temp }}/build-pcms/Testing/Temporary/LastTest.log
 
     - name: Test PCMS Installation
       shell: bash
       run: |
         # Configure and build the test
         # it should only ask for pcms_DIR/ROOT to find all dependencies
+        ls ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms
+        echo "-------- PCMS CONFIG -------------"
+        cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms-config.cmake
+        echo "-------- PCMS TARGETS -------------"
+        cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms-targets.cmake
+        echo "-------- PCMS INTERPOLATOR TARGETS -------------"
+        cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms_interpolator-targets.cmake
+        export VERBOSE=1
         cmake \
           -B ${{github.workspace}}/examples/external-usage-example/build \
           -S ${{github.workspace}}/examples/external-usage-example/ \
           -Dpcms_DIR=${{ runner.temp }}/build-pcms/install/lib/cmake/pcms \
-
+          --debug-output
         cmake --build ${{github.workspace}}/examples/external-usage-example/build

CITATION.cff

@@ -0,0 +1,62 @@
+# This CITATION.cff file was generated with cffinit.
+# Visit https://bit.ly/cffinit to generate yours today!
+
+cff-version: 1.2.0
+title: Parallel Coupler for Multimodel Simulations
+message: >-
+  If you use this software, please cite it using the
+  metadata from this file.
+type: software
+authors:
+  - given-names: Jacob
+    family-names: Merson
+    email: mersoj2@rpi.edu
+    affiliation: Rensselaer Polytechnic Institute
+    orcid: 'https://orcid.org/0000-0002-6813-6532'
+    name-particle: S
+  - given-names: Cameron
+    name-particle: W
+    family-names: Smith
+    affiliation: Rensselaer Polytechnic Institute
+  - given-names: Mark
+    name-particle: S
+    family-names: Shephard
+    affiliation: Rensselaer Polytechnic Institute
+  - given-names: Fuad
+    family-names: Hasan
+    affiliation: Rensselaer Polytechnic Institute
+  - given-names: Abhiyan
+    family-names: Paudel
+    affiliation: Rensselaer Polytechnic Institute
+  - given-names: Angel
+    family-names: Castillo-Crooke
+    affiliation: Rensselaer Polytechnic Institute
+  - given-names: Joyal
+    family-names: Mathew
+    affiliation: Rensselaer Polytechnic Institute
+  - given-names: Mohammad
+    family-names: Elahi
+    affiliation: Rensselaer Polytechnic Institute
+identifiers:
+  - type: doi
+    value: 10.48550/arXiv.2510.18838
+repository-code: 'https://github.com/SCOREC/pcms'
+abstract: >-
+  PCMS provides infrastructure to couple multiple simulation
+  codes at scale on exascale systems. It uses scalable
+  methodologies for transferring data between distributed
+  applications. PCMS takes a hierarchical approach to field
+  transfer using as much information as possible, such as
+  mesh topology, to accommodate conservation and other
+  physical constraints. In the context of limited
+  information, alternative strategies, such as radial basis
+  functions, are used to support field transfer. PCMS has
+  been demonstrated on coupled fusion simulations that make
+  use of volume coupling of high-dimensional data and
+  complex coordinate systems.
+keywords:
+  - HPC
+  - coupling
+  - supercomputing
+  - unstructured-meshes
+license: BSD-3-Clause

config.cmake.in

@@ -3,6 +3,7 @@
 include(CMakeFindDependencyMacro)
 find_dependency(redev CONFIG HINTS @redev_DIR@)
 find_dependency(Kokkos CONFIG HINTS @Kokkos_DIR@)
+find_dependency(KokkosKernels CONFIG HINTS @KokkosKernels_DIR@)
 find_dependency(meshfields CONFIG HINTS @meshfields_DIR@)
 find_dependency(ADIOS2 CONFIG HINTS @ADIOS2_DIR@)
 find_dependency(MPI)
@@ -16,14 +17,20 @@ if(@PCMS_ENABLE_OMEGA_H@)
 endif()
 
 include("${CMAKE_CURRENT_LIST_DIR}/pcms_core-targets.cmake")
+include("${CMAKE_CURRENT_LIST_DIR}/pcms_interpolator-targets.cmake")
+
 if(@PCMS_ENABLE_C@)
+    include("${CMAKE_CURRENT_LIST_DIR}/pcms_capi_core-targets.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/pcms_capi_interpolator-targets.cmake")
     include("${CMAKE_CURRENT_LIST_DIR}/pcms_capi-targets.cmake")
 endif()
+
 if(@PCMS_ENABLE_Fortran@)
+    include("${CMAKE_CURRENT_LIST_DIR}/pcms_fortranapi_core-targets.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/pcms_fortranapi_interpolator-targets.cmake")
     include("${CMAKE_CURRENT_LIST_DIR}/pcms_fortranapi-targets.cmake")
 endif()
 
 include("${CMAKE_CURRENT_LIST_DIR}/pcms-targets.cmake")
-include("${CMAKE_CURRENT_LIST_DIR}/pcms_interpolator-targets.cmake")
 # must be called at the end of the config file
 check_required_components(pcms)

src/CMakeLists.txt

@@ -148,6 +148,7 @@ if(PCMS_ENABLE_Fortran)
 endif()
 
 add_subdirectory(pcms/interpolator)
+target_link_libraries(pcms_pcms INTERFACE pcms::interpolator)
 
 install(
   TARGETS pcms_pcms

src/pcms/capi/CMakeLists.txt

@@ -1,22 +1,45 @@
-add_library(pcms_capi client.cpp kokkos.cpp)
-add_library(pcms::capi ALIAS pcms_capi)
+add_library(pcms_capi_core client.cpp kokkos.cpp)
+add_library(pcms::capi::core ALIAS pcms_capi_core)
+
 target_link_libraries(
-  pcms_capi
+  pcms_capi_core
   PUBLIC MPI::MPI_C
   PRIVATE pcms::core)
 
 target_include_directories(
-  pcms_capi
+  pcms_capi_core
   PUBLIC "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../..>" # this makes our
                                                                 # include path
                                                                 # pcms/capi
          "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/pcms/capi>")
 
-set(CAPI_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/client.h
+set(CAPI_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/client.h
                  ${CMAKE_CURRENT_SOURCE_DIR}/kokkos.h)
 set_target_properties(
-  pcms_capi PROPERTIES PUBLIC_HEADER "${CAPI_HEADERS}" OUTPUT_NAME pcmscapi
-                       EXPORT_NAME capi)
+  pcms_capi_core PROPERTIES PUBLIC_HEADER "${CAPI_CORE_HEADERS}" OUTPUT_NAME pcmscapicore
+             EXPORT_NAME capi::core)
+
+add_library(pcms_capi_interpolator mesh.cpp interpolator.cpp kokkos.cpp)
+add_library(pcms::capi::interpolator ALIAS pcms_capi_interpolator)
+target_link_libraries(pcms_capi_interpolator PUBLIC MPI::MPI_C PRIVATE pcms::interpolator)
+target_include_directories(pcms_capi_interpolator
+        PUBLIC
+        "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../..>" # this makes the module path cpms/capi
+        "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/pcms/capi>")
+set(CAPI_INTERPOLATOR_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/interpolator.h
+                     ${CMAKE_CURRENT_SOURCE_DIR}/mesh.h)
+set_target_properties(pcms_capi_interpolator
+  PROPERTIES PUBLIC_HEADERS "${CAPI_INTERPOLATOR_HEADERS}"
+             OUTPUT_NAME pcmscapiinterpolator
+             EXPORT_NAME capi::interpolator)
+
+# high level interface target
+add_library(pcms_capi INTERFACE)
+add_library(pcms::capi ALIAS pcms_capi)
+set_target_properties(pcms_capi PROPERTIES EXPORT_NAME capi)
+# link capi libraries to a high level interface library
+target_link_libraries(pcms_capi INTERFACE pcms::capi::core)
+target_link_libraries(pcms_capi INTERFACE pcms::capi::interpolator)
 install(
   TARGETS pcms_capi
   EXPORT pcms_capi-targets
@@ -25,8 +48,37 @@ install(
   RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
   INCLUDES
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pcms/capi/)
+  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pcms)
 install(
   EXPORT pcms_capi-targets
   NAMESPACE pcms::
   DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcms)
+
+# library targets
+install(
+  TARGETS pcms_capi_core
+  EXPORT pcms_capi_core-targets
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  INCLUDES
+  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pcms/capi/)
+install(
+  EXPORT pcms_capi_core-targets
+  NAMESPACE pcms::
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcms)
+
+install(
+  TARGETS pcms_capi_interpolator
+  EXPORT pcms_capi_interpolator-targets
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  INCLUDES
+  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pcms/capi/)
+install(
+  EXPORT pcms_capi_interpolator-targets
+  NAMESPACE pcms::
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcms)