From 32797a1dadc46a73283f12c14bd4f46ce4d16578 Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Tue, 4 Nov 2025 15:06:19 -0500 Subject: [PATCH 1/9] add mpi and two process test --- CMakeLists.txt | 13 +++++++++++++ test/testSprThwaitesAdapt.cpp | 1 + 2 files changed, 14 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c042d51..52ec530 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,7 @@ option(MeshFields_USE_Cabana "Build with the Cabana storage backend" OFF) find_package(Kokkos REQUIRED) find_package(Omega_h REQUIRED) +find_package(MPI REQUIRED) # Verify Omega_h was built with Kokkos enabled if(NOT Omega_h_USE_Kokkos) @@ -234,6 +235,18 @@ add_test_property(OmegahSprAdapt PASS_REGULAR_EXPRESSION "afterAdapt nTri: 14439; solution_1 min -4084.78 max 213.322") +test_func(OmegahSprAdapt_p2 + ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} + ./OmegahSprAdapt ${CMAKE_SOURCE_DIR}/meshes/thwaites_basal_effectiveStrain.osh/ + thwaitesAdapted #output prefix + 0.1 # adapt ratio + 1.0 # min size + 16.0 # max size +) +add_test_property(OmegahSprAdapt_p2 + PASS_REGULAR_EXPRESSION + "afterAdapt nTri: 14439; solution_1 min -4084.78 max 213.322") + #Code Coverage set up ------------------------------------------------------- option(meshfields_ENABLE_COVERAGE_BUILD "Do a coverage build" OFF) diff --git a/test/testSprThwaitesAdapt.cpp b/test/testSprThwaitesAdapt.cpp index 4ed0c32..106de57 100644 --- a/test/testSprThwaitesAdapt.cpp +++ b/test/testSprThwaitesAdapt.cpp @@ -157,6 +157,7 @@ int main(int argc, char **argv) { auto world = lib.world(); Omega_h::Mesh mesh(&lib); Omega_h::binary::read(argv[1], world, &mesh); + mesh.balance(); const auto prefix = std::string(argv[2]); // adaptRatio = 0.1 is used in scorec/core:cws/sprThwaites test/spr_test.cc const MeshField::Real adaptRatio = std::stof(argv[3]); From d902e64a7ec4985e545cc7f383bddbde19a59612 Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Tue, 4 Nov 2025 15:46:15 -0500 Subject: [PATCH 2/9] test name --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 52ec530..319f7ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -238,7 +238,7 @@ add_test_property(OmegahSprAdapt test_func(OmegahSprAdapt_p2 ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} ./OmegahSprAdapt ${CMAKE_SOURCE_DIR}/meshes/thwaites_basal_effectiveStrain.osh/ - thwaitesAdapted #output prefix + thwaitesAdapted_p2 #output prefix 0.1 # adapt ratio 1.0 # min size 16.0 # max size From 632b2b2787e0ccc5a3e514067c41f98e4bc0fbc4 Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Tue, 4 Nov 2025 15:47:21 -0500 Subject: [PATCH 3/9] executes, final mesh does not match serial run --- test/testSprThwaitesAdapt.cpp | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/test/testSprThwaitesAdapt.cpp b/test/testSprThwaitesAdapt.cpp index 106de57..ea44cd2 100644 --- a/test/testSprThwaitesAdapt.cpp +++ b/test/testSprThwaitesAdapt.cpp @@ -157,7 +157,7 @@ int main(int argc, char **argv) { auto world = lib.world(); Omega_h::Mesh mesh(&lib); Omega_h::binary::read(argv[1], world, &mesh); - mesh.balance(); + const auto prefix = std::string(argv[2]); // adaptRatio = 0.1 is used in scorec/core:cws/sprThwaites test/spr_test.cc const MeshField::Real adaptRatio = std::stof(argv[3]); @@ -167,13 +167,20 @@ int main(int argc, char **argv) { argv[5]); // a value of 8 or 16 roughly maintains the boundary shape // in the downstream parts of the domain // where there is significant coarsening - std::cout << "input mesh: " << argv[1] << " outputMeshPrefix: " << prefix - << " adaptRatio: " << adaptRatio << " max_size: " << max_size - << " min_size: " << min_size << "\n"; + if (!mesh.comm()->rank()) { + std::cout << "input mesh: " << argv[1] << " outputMeshPrefix: " << prefix + << " adaptRatio: " << adaptRatio << " max_size: " << max_size + << " min_size: " << min_size << "\n"; + } const auto outname = prefix + "_adaptRatio_" + std::string(argv[3]) + "_maxSz_" + std::to_string(max_size) + "_minSz_" + std::to_string(min_size); + //equally distribute elements among processes; required when loading a serial mesh + mesh.balance(); + //Omega_h::project_by_fit used by spr requires ghosts + mesh.set_parting(Omega_h_Parting::OMEGA_H_GHOSTED); + auto effectiveStrain = getEffectiveStrainRate(mesh); auto recoveredStrain = recoverLinearStrain(mesh, effectiveStrain); mesh.add_tag(VERT, "recoveredStrain", 1, recoveredStrain, false, @@ -204,10 +211,10 @@ int main(int argc, char **argv) { Omega_h::ArrayType::VectorND); { // write vtk - const std::string vtkFileName = "beforeAdapt" + outname + ".vtk"; + const std::string vtkFileName = "beforeAdapt_" + outname + ".vtk"; Omega_h::vtk::write_parallel(vtkFileName, &mesh, 2); const std::string vtkFileName_edges = - "beforeAdapt" + outname + "_edges.vtk"; + "beforeAdapt_" + outname + "_edges.vtk"; Omega_h::vtk::write_parallel(vtkFileName_edges, &mesh, 1); } @@ -224,12 +231,14 @@ int main(int argc, char **argv) { printTriCount(&mesh, "afterAdapt"); const auto sol = mesh.get_array(VERT, "solution_1"); - const auto sol_min = Omega_h::get_min(sol); - const auto sol_max = Omega_h::get_max(sol); - std::cout << "solution_1 min " << sol_min << " max " << sol_max << '\n'; + const auto sol_min = Omega_h::get_min(mesh.comm(), sol); + const auto sol_max = Omega_h::get_max(mesh.comm(), sol); + if(!mesh.comm()->rank()) { + std::cout << "solution_1 min " << sol_min << " max " << sol_max << '\n'; + } { // write vtk and osh for adapted mesh - const std::string outfilename = "afterAdapt" + outname; + const std::string outfilename = "afterAdapt_" + outname; Omega_h::vtk::write_parallel(outfilename + ".vtk", &mesh, 2); Omega_h::binary::write(outfilename + ".osh", &mesh); std::cout << "wrote adapted mesh: " << outfilename + ".osh" From 3822b9f971a0538f96a2ab0ed7c5a6e2d12ad56b Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Tue, 4 Nov 2025 16:08:54 -0500 Subject: [PATCH 4/9] formatting [skip ci] --- src/MeshField_SPR_ErrorEstimator.hpp | 3 ++- test/testSprThwaitesAdapt.cpp | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/MeshField_SPR_ErrorEstimator.hpp b/src/MeshField_SPR_ErrorEstimator.hpp index 97eb726..48a0782 100644 --- a/src/MeshField_SPR_ErrorEstimator.hpp +++ b/src/MeshField_SPR_ErrorEstimator.hpp @@ -270,7 +270,8 @@ template std::tuple, MeshField::Real> getSprSizeField(EstimationT &e, OmegahMeshField &omf, FieldElement &coordFe) { Error errorIntegrator(e, omf); - errorIntegrator.process(coordFe); + errorIntegrator.process( + coordFe); // FIXME - needs to sync results across processes computeSizeFactor(e, omf, coordFe, errorIntegrator); getElementSizeField(e, errorIntegrator); return {averageToVertex(e.mesh, e.element_size), errorIntegrator.r}; diff --git a/test/testSprThwaitesAdapt.cpp b/test/testSprThwaitesAdapt.cpp index ea44cd2..55374ec 100644 --- a/test/testSprThwaitesAdapt.cpp +++ b/test/testSprThwaitesAdapt.cpp @@ -176,9 +176,10 @@ int main(int argc, char **argv) { "_maxSz_" + std::to_string(max_size) + "_minSz_" + std::to_string(min_size); - //equally distribute elements among processes; required when loading a serial mesh + // equally distribute elements among processes; required when loading a serial + // mesh mesh.balance(); - //Omega_h::project_by_fit used by spr requires ghosts + // Omega_h::project_by_fit used by spr requires ghosts mesh.set_parting(Omega_h_Parting::OMEGA_H_GHOSTED); auto effectiveStrain = getEffectiveStrainRate(mesh); @@ -233,7 +234,7 @@ int main(int argc, char **argv) { const auto sol = mesh.get_array(VERT, "solution_1"); const auto sol_min = Omega_h::get_min(mesh.comm(), sol); const auto sol_max = Omega_h::get_max(mesh.comm(), sol); - if(!mesh.comm()->rank()) { + if (!mesh.comm()->rank()) { std::cout << "solution_1 min " << sol_min << " max " << sol_max << '\n'; } From 3203542259b9119482aadd3506d11e0deaaed20c Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Fri, 7 Nov 2025 15:09:53 -0500 Subject: [PATCH 5/9] integrate: add inter-process reduction hook --- src/MeshField_Integrate.hpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/MeshField_Integrate.hpp b/src/MeshField_Integrate.hpp index 100f268..b2b1ebe 100644 --- a/src/MeshField_Integrate.hpp +++ b/src/MeshField_Integrate.hpp @@ -196,9 +196,12 @@ auto getJacobianDeterminants(FieldElement &fes, * - `post`: Called after the integration process ends. * - `atPoints`: Called for each integration point to perform user-defined * computations. + * - `parallelReduce`: Called after `atPoints` to reduce process-local + * integrations into a global mesh integration. * * To use this class, derive from it and implement the `atPoints` method. - * Optionally, override `pre` and `post` for additional setup or cleanup. + * Optionally, override `parallelReduce`, `pre` and `post` for distributed + * memory support, additional setup, or cleanup. */ class Integrator { public: @@ -227,9 +230,18 @@ class Integrator { */ virtual void atPoints(Kokkos::View p, Kokkos::View w, Kokkos::View dV) = 0; + + /** \brief User callback: distributed memory accumulation. + * + * \details If needed, this function supports the use of + * inter-process communication (e.g., MPI) to reduce + * process-local integrations into a global + * mesh integration. + */ + virtual void parallelReduce(){}; + /** \brief Run the Integrator over the local field elements. - * \param fes FieldElement - * FIXME make the sensible + * \param fes FieldElement * */ template void process(FieldElement &fes) { constexpr auto topo = decltype(FieldElement::elm2dof)::getTopology(); @@ -239,9 +251,8 @@ class Integrator { auto weights = getIntegrationPointWeights(fes, ip); auto dV = getJacobianDeterminants(fes, localCoords, ip.size()); atPoints(localCoords, weights, dV); + parallelReduce(); post(); - // TODO support distributed meshes by running a parallel reduction with user - // functor } protected: From 61ff01dd67db74fda90dc5954be4cde0b25dd9b2 Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Fri, 7 Nov 2025 15:28:55 -0500 Subject: [PATCH 6/9] spr: add allreduce element size field (elmSizeField) has obvious discontinuity at part boundary when viewed with log scale in paraview --- src/MeshField_SPR_ErrorEstimator.hpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/MeshField_SPR_ErrorEstimator.hpp b/src/MeshField_SPR_ErrorEstimator.hpp index 48a0782..a412c96 100644 --- a/src/MeshField_SPR_ErrorEstimator.hpp +++ b/src/MeshField_SPR_ErrorEstimator.hpp @@ -19,9 +19,12 @@ static double getNaN() { return std::numeric_limits::quiet_NaN(); } /* common base for Scalar Integrator. */ class SInt : public MeshField::Integrator { public: - SInt(int order) : MeshField::Integrator(order), r(0) {} + SInt(int order, Omega_h::CommPtr comm_in) + : MeshField::Integrator(order), r(0), comm(comm_in) {} void reset() { r = 0; } MeshField::Real r; + Omega_h::CommPtr comm; + void parallelReduce() { comm->allreduce(r, OMEGA_H_SUM); } }; template class Estimation { @@ -83,8 +86,8 @@ template class SelfProduct : public SInt { public: SelfProduct(EstimationT &estimation_in, OmegahMeshField &omf_in) - : SInt(estimation_in.integration_order), estimation(estimation_in), - omf(omf_in) {} + : SInt(estimation_in.integration_order, estimation_in.mesh.comm()), + estimation(estimation_in), omf(omf_in) {} void atPoints(Kokkos::View p, Kokkos::View w, Kokkos::View dV) { @@ -155,8 +158,9 @@ template class Error : public SInt { public: Error(EstimationT &estimation_in, OmegahMeshField &omf_in) - : SInt(estimation_in.integration_order), estimation(estimation_in), - omf(omf_in), errorNorm("errorNorm", estimation_in.mesh.nelems()) {} + : SInt(estimation_in.integration_order, estimation_in.mesh.comm()), + estimation(estimation_in), omf(omf_in), + errorNorm("errorNorm", estimation_in.mesh.nelems()) {} void atPoints(Kokkos::View p, Kokkos::View w, Kokkos::View dV) { From 38d7775606e148655575dd057b5f82a192573051 Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Mon, 10 Nov 2025 12:24:58 -0500 Subject: [PATCH 7/9] spr: save the reduction result duhhhhh --- src/MeshField_SPR_ErrorEstimator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MeshField_SPR_ErrorEstimator.hpp b/src/MeshField_SPR_ErrorEstimator.hpp index a412c96..6503e12 100644 --- a/src/MeshField_SPR_ErrorEstimator.hpp +++ b/src/MeshField_SPR_ErrorEstimator.hpp @@ -24,7 +24,7 @@ class SInt : public MeshField::Integrator { void reset() { r = 0; } MeshField::Real r; Omega_h::CommPtr comm; - void parallelReduce() { comm->allreduce(r, OMEGA_H_SUM); } + void parallelReduce() { r = comm->allreduce(r, OMEGA_H_SUM); } }; template class Estimation { From e869288ad32f7ab9d1522cd7f1d1bb35756b4cb1 Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Mon, 10 Nov 2025 13:15:21 -0500 Subject: [PATCH 8/9] adjust accepted result for p2 spr test meshfields is not deterministic --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 319f7ee..208ccd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -245,7 +245,7 @@ test_func(OmegahSprAdapt_p2 ) add_test_property(OmegahSprAdapt_p2 PASS_REGULAR_EXPRESSION - "afterAdapt nTri: 14439; solution_1 min -4084.78 max 213.322") + "afterAdapt nTri: 14468; solution_1 min -4098.9 max 213.322") #Code Coverage set up ------------------------------------------------------- From 4005dd8468014e457f8780a2548629b3b14918ed Mon Sep 17 00:00:00 2001 From: Cameron Smith Date: Mon, 10 Nov 2025 15:25:30 -0500 Subject: [PATCH 9/9] ci: install openmpi mpich is broken in ubuntu 24.04 --- .github/workflows/cmake.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 9baba4e..6a54f23 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -27,6 +27,13 @@ jobs: sudo apt-get install -yq cmake cmake --version + # There is a known bug using MPICH ubuntu-24.04 (ubuntu-latest as of 11/10/2025) + # https://bugs.launchpad.net/ubuntu/+source/mpich/+bug/2072338 + - name: Install mpi + run: | + sudo apt-get update -yq + sudo apt-get install -yq openmpi-bin libopenmpi-dev + ## Kokkos - name: Kokkos Checkout repo uses: actions/checkout@v4