Skip to content

Commit 9b7bcd2

Browse files
AngelyrAngelyr
committed
set mpi communicator
1 parent e638a1f commit 9b7bcd2

14 files changed

+33
-22
lines changed

‎src/pmpo_MPMesh.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -307,15 +307,16 @@ void MPMesh::reconstructSlices() {
307307
pumipic::RecordTime("PolyMPO_Reconstruct", timer.seconds());
308308
}
309309

310-
bool getAnyIsMigrating(bool isMigrating) {
310+
bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) {
311311
Kokkos::Timer timer;
312+
MPI_Comm comm = p_MPs->getMPIComm();
312313
int comm_rank;
313-
MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank);
314+
MPI_Comm_rank(comm, &comm_rank);
314315
int comm_size;
315-
MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
316+
MPI_Comm_size(comm, &comm_size);
316317

317318
bool anyIsMigrating = false;
318-
MPI_Allreduce(&isMigrating, &anyIsMigrating, 1, MPI_C_BOOL, MPI_LOR, MPI_COMM_WORLD);
319+
MPI_Allreduce(&isMigrating, &anyIsMigrating, 1, MPI_C_BOOL, MPI_LOR, comm);
319320
pumipic::RecordTime("PolyMPO_getAnyIsMigrating", timer.seconds());
320321
return anyIsMigrating;
321322
}
@@ -333,7 +334,7 @@ void MPMesh::push(){
333334
p_MPs->updateMPSlice<MPF_Cur_Pos_XYZ, MPF_Tgt_Pos_XYZ>(); // Tgt_XYZ becomes Cur_XYZ
334335
p_MPs->updateMPSlice<MPF_Cur_Pos_Rot_Lat_Lon, MPF_Tgt_Pos_Rot_Lat_Lon>(); // Tgt becomes Cur
335336
if (elm2Process.size() > 0)
336-
anyIsMigrating = getAnyIsMigrating(p_MPs->migrate());
337+
anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->migrate());
337338
else
338339
p_MPs->rebuild(); //rebuild pumi-pic
339340
p_MPs->updateMPElmID(); //update mpElm IDs slices

‎src/pmpo_c.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,10 @@ void polympo_deleteMPMesh_f(MPMesh_ptr p_mpmesh) {
5252
delete (polyMPO::MPMesh*)p_mpmesh;
5353
}
5454

55-
void polympo_setMPICommunicator_f(MPI_Fint fcomm){
55+
void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm){
5656
MPI_Comm comm = MPI_Comm_f2c(fcomm);
57-
int commSize;
58-
MPI_Comm_size(comm,&commSize);
57+
auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs;
58+
p_MPs->setMPIComm(comm);
5959
}
6060

6161
void polympo_createMPs_f(MPMesh_ptr p_mpmesh,

‎src/pmpo_c.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ MPMesh_ptr polympo_createMPMesh_f(const int setMeshOption, const int setMPOption
1515
void polympo_deleteMPMesh_f(MPMesh_ptr p_mpmesh);
1616

1717
//set MPI communicator
18-
void polympo_setMPICommunicator_f(MPI_Fint fcomm);//TODO:is MPI_Fint best? or something else
18+
void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm);
1919
//TODO: add a function to get communicator
2020

2121
//MP info

‎src/pmpo_fortran.f90‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,10 @@ subroutine polympo_deleteMPMesh(mpMesh) bind(C, NAME='polympo_deleteMPMesh_f')
4545
!> @brief set the MPI communicator used by polympo
4646
!> @param comm(in) MPI communicator
4747
!---------------------------------------------------------------------------
48-
subroutine polympo_setMPICommunicator(comm) &
48+
subroutine polympo_setMPICommunicator(mpMesh, comm) &
4949
bind(C, NAME='polympo_setMPICommunicator_f')
5050
use :: iso_c_binding
51+
type(c_ptr), value :: mpMesh
5152
integer(c_int), value :: comm
5253
end subroutine
5354
!---------------------------------------------------------------------------

‎src/pmpo_materialPoints.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,14 @@ void MaterialPoints::finishRebuild() {
101101
rebuildFields.ongoing = false;
102102
}
103103

104+
MPI_Comm MaterialPoints::getMPIComm() {
105+
return mpi_comm;
106+
}
107+
108+
void MaterialPoints::setMPIComm(MPI_Comm comm) {
109+
mpi_comm = comm;
110+
}
111+
104112
bool MaterialPoints::migrate() {
105113
Kokkos::Timer timer;
106114
auto MPs2Elm = getData<MPF_Tgt_Elm_ID>();
@@ -111,7 +119,7 @@ bool MaterialPoints::migrate() {
111119
IntView isMigrating("isMigrating", 1);
112120

113121
int rank;
114-
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
122+
MPI_Comm_rank(mpi_comm, &rank);
115123
auto setMigrationFields = PS_LAMBDA(const int& e, const int& mp, const bool& mask) {
116124
if (mask) {
117125
new_elem(mp) = MPs2Elm(mp);

‎src/pmpo_materialPoints.hpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ class MaterialPoints {
125125
Operating_Mode operating_mode;
126126
RebuildHelper rebuildFields;
127127
IntFunc getAppID;
128+
MPI_Comm mpi_comm;
128129

129130
public:
130131
MaterialPoints() : MPs(nullptr) {};
@@ -136,7 +137,10 @@ class MaterialPoints {
136137
void startRebuild(IntView tgtElm, int addedNumMPs, IntView addedMP2elm, IntView addedMPAppID, Kokkos::View<const int*> addedMPMask);
137138
void finishRebuild();
138139
bool rebuildOngoing();
140+
139141
bool migrate();
142+
MPI_Comm getMPIComm();
143+
void setMPIComm(MPI_Comm comm);
140144

141145
template<int mpSliceIndex, typename mpSliceData>
142146
typename std::enable_if<mpSliceData::rank==1>::type

‎test/testFortran.f90‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@ program main
3737
call mpi_init(ierr)
3838
call mpi_comm_rank(mpi_comm_handle, self, ierr)
3939

40-
call polympo_setMPICommunicator(mpi_comm_handle) !this is not supported yet! only for showing
4140
call polympo_initialize()
42-
41+
4342
call polympo_checkPrecisionForRealKind(APP_RKIND)
4443
setMeshOption = 1 !create a hard coded planar test mesh
4544
setMPOption = 1 !create some random test MPs that based on the mesh option you give
4645
mpMesh = polympo_createMPMesh(setMeshOption,setMPOption) !creates test mesh
46+
call polympo_setMPICommunicator(mpMesh, mpi_comm_handle) !this is not supported yet! only for showing
4747

4848
!These are hard coded test mesh values
4949
nverts = 19 !todo use getNumVtx from the Mesh object

‎test/testFortranCreateRebuildMPs.f90‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,6 @@ program main
261261
call mpi_init(ierr)
262262
call mpi_comm_rank(mpi_comm_handle, self, ierr)
263263

264-
call polympo_setMPICommunicator(mpi_comm_handle)
265264
call polympo_initialize()
266265

267266
argc = command_argument_count()
@@ -274,6 +273,7 @@ program main
274273
setMeshOption = 0 !create an empty mesh
275274
setMPOption = 0 !create an empty set of MPs
276275
mpMesh = polympo_createMPMesh(setMeshOption, setMPOption)
276+
call polympo_setMPICommunicator(mpMesh, mpi_comm_handle)
277277

278278
call readMPASMeshFromNCFile(filename, maxEdges, vertexDegree, &
279279
nCells, nVertices, nEdgesOnCell, &

‎test/testFortranInit.f90‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ program main
1414
call mpi_comm_rank(mpi_comm_handle, self, ierr)
1515

1616
!polympo start here
17-
call polympo_setMPICommunicator(mpi_comm_handle)
1817
call polympo_initialize()
1918

2019
call polympo_finalize()

‎test/testFortranMPAdvection.f90‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,6 @@ program main
230230
call mpi_comm_rank(mpi_comm_handle, self, ierr)
231231
call mpi_comm_size(mpi_comm_handle, comm_size, ierr)
232232

233-
call polympo_setMPICommunicator(mpi_comm_handle)
234233
call polympo_initialize()
235234
call polympo_enableTiming()
236235

@@ -272,6 +271,8 @@ program main
272271
xCell, yCell, zCell, &
273272
verticesOnCell, cellsOnCell)
274273

274+
call polympo_setMPICommunicator(mpMesh, mpi_comm_handle);
275+
275276
!createMPs
276277
numMPs = 0
277278
do i = 1, nCells

0 commit comments

Comments
 (0)