Skip to content

Commit fefd5e5

Browse files
dhyan1272dhyan1272
committed
Added global ids to polyMPO from MPAS
1 parent 7be04f9 commit fefd5e5

File tree

8 files changed

+81
-45
lines changed

8 files changed

+81
-45
lines changed

‎src/pmpo_MPMesh.cpp‎

Lines changed: 16 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -127,15 +127,21 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){
127127
auto MPs2Elm = p_MPs->getData<MPF_Tgt_Elm_ID>();
128128
auto MPs2Proc = p_MPs->getData<MPF_Tgt_Proc_ID>();
129129
auto elm2Process = p_mesh->getElm2Process();
130-
130+
auto elm2global = p_mesh->getElmGlobal();
131+
132+
133+
MPI_Comm comm = p_MPs->getMPIComm();
134+
int comm_rank;
135+
MPI_Comm_rank(comm, &comm_rank);
131136
assert(cudaDeviceSynchronize() == cudaSuccess);
132137
MPI_Barrier(MPI_COMM_WORLD);
133138
printf("FooTracking \n");
134-
135139
Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){
136140
int pp_id=elm2Process(iElm);
137-
printf("Mesh elm %d owning element %d \n", iElm, pp_id);
141+
if(iElm<3 || iElm==1470 || iElm==1471 || iElm==1472 )
142+
printf("Mesh elm %d Process %d owning proc %d global %d \n", iElm, comm_rank, pp_id, elm2global(iElm));
138143
});
144+
139145

140146
if(printVTPIndex>=0) {
141147
printVTP_mesh(printVTPIndex);
@@ -333,51 +339,37 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) {
333339
}
334340

335341
void MPMesh::push(){
342+
336343
static int count=0;
337344
std::cout<<__FUNCTION__<<" "<<count<<std::endl;
338345
count++;
339-
//if(count>1) exit(1);
340346
Kokkos::Timer timer;
347+
341348
p_mesh->computeRotLatLonIncr();
342349

343-
assert(cudaDeviceSynchronize() == cudaSuccess);
344-
MPI_Barrier(MPI_COMM_WORLD);
345-
printf("FooPush\n");
346-
347350
sphericalInterpolation<MeshF_RotLatLonIncr>(*this);
348-
assert(cudaDeviceSynchronize() == cudaSuccess);
349-
MPI_Barrier(MPI_COMM_WORLD);
350-
printf("FooPush \n");
351-
351+
352352
p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); // set Tgt_XYZ
353-
assert(cudaDeviceSynchronize() == cudaSuccess);
354-
MPI_Barrier(MPI_COMM_WORLD);
355-
printf("FooPush\n");
356-
353+
357354
auto elm2Process = p_mesh->getElm2Process();
358-
assert(cudaDeviceSynchronize() == cudaSuccess);
359-
MPI_Barrier(MPI_COMM_WORLD);
360-
printf("FooPush \n");
361355

362356
bool anyIsMigrating = false;
363357
do {
364358
CVTTrackingElmCenterBased(); // move to Tgt_XYZ
365359
assert(cudaDeviceSynchronize() == cudaSuccess);
366-
printf("FooPushInside\n");
360+
printf("FooPushDoWhile\n");
367361
p_MPs->updateMPSlice<MPF_Cur_Pos_XYZ, MPF_Tgt_Pos_XYZ>(); // Tgt_XYZ becomes Cur_XYZ
368362
p_MPs->updateMPSlice<MPF_Cur_Pos_Rot_Lat_Lon, MPF_Tgt_Pos_Rot_Lat_Lon>(); // Tgt becomes Cur
369363
if (elm2Process.size() > 0)
370364
anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->migrate());
371365
else
372366
p_MPs->rebuild(); //rebuild pumi-pic
367+
printf("Is migrating %d \n", anyIsMigrating);
373368
p_MPs->updateMPElmID(); //update mpElm IDs slices
374369
reconstructSlices();
375370
}
376371
while (anyIsMigrating);
377-
assert(cudaDeviceSynchronize() == cudaSuccess);
378-
MPI_Barrier(MPI_COMM_WORLD);
379-
printf("FooPush\n");
380-
372+
381373
pumipic::RecordTime("PolyMPO_push", timer.seconds());
382374
}
383375

‎src/pmpo_c.cpp‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -75,34 +75,33 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh,
7575
//Find the total no of MPs across all ranks
7676
//And loop over all MPs and find the smallest element id associated across a MP
7777
int numActiveMPs = 0;
78-
int minElmID = numElms+1;
78+
int minElmID = INT_MAX;
7979
for(int i = 0; i < numMPs; i++) {
8080
if(isMPActive[i] == MP_ACTIVE) {
81-
if(mp2Elm[i] < minElmID) {
81+
numActiveMPs++;
82+
if(mp2Elm[i] < minElmID)
8283
minElmID = mp2Elm[i];
83-
numActiveMPs++;
84-
}
8584
}
8685
}
87-
printf("Num Active MPs and minElmId %d %d\n", numActiveMPs, minElmID);
88-
long long globalNumActiveMPs = 0;
86+
int globalNumActiveMPs = 0;
8987
int globalMinElmID;
90-
MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
91-
MPI_Allreduce(&minElmID, &globalMinElmID, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
88+
MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
89+
MPI_Allreduce(&minElmID, &globalMinElmID, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
9290
PMT_ALWAYS_ASSERT(globalNumActiveMPs>0);
9391

9492
//Loop over all mesh elements 0,1,... and find the first element that has an associated MP
95-
int firstElmWithMPs=numElms+1;
93+
int firstElmWithMPs=INT_MAX;
9694
for (int i=0; i<numElms; i++) {
9795
if(mpsPerElm[i]) {
9896
firstElmWithMPs = i;
9997
break;
10098
}
10199
}
102-
printf("First elem with MP %d\n", firstElmWithMPs);
103100
int globalFirstElmWithMPs;
104101
MPI_Allreduce(&firstElmWithMPs, &globalFirstElmWithMPs, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
105102

103+
printf("With a MP, globally smallest mesh elm %d and first elm %d \n", globalMinElmID, globalFirstElmWithMPs);
104+
106105
int offset = -1;
107106
if(globalMinElmID-globalFirstElmWithMPs==1) {
108107
offset = 1;
@@ -123,26 +122,23 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh,
123122
numActiveMPs++;
124123
}
125124
}
125+
auto elm2global = p_mesh->getElmGlobal();
126126

127127
auto mpsPerElm_d = create_mirror_view_and_copy(mpsPerElm, numElms);
128128
auto active_mp2Elm_d = create_mirror_view_and_copy(active_mp2Elm.data(), numActiveMPs);
129129
auto active_mpIDs_d = create_mirror_view_and_copy(active_mpIDs.data(), numActiveMPs);
130130

131131
delete ((polyMPO::MPMesh*)p_mpmesh)->p_MPs;
132132
((polyMPO::MPMesh*)p_mpmesh)->p_MPs =
133-
new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d);
133+
new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d, elm2global);
134134

135135
auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs;
136136
p_MPs->setElmIDoffset(offset);
137137

138-
assert(cudaDeviceSynchronize() == cudaSuccess);
139-
MPI_Barrier(MPI_COMM_WORLD);
140-
printf("Foo1\n");
141-
142138
}
143139

144140
void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh,
145-
const int numMPs, // total number of MPs which is GREATER than or equal to number of active MPs
141+
const int numMPs, // Total # MPs which is GREATER than or equal to number of active MPs
146142
const int* allMP2Elm,
147143
const int* addedMPMask) {
148144
checkMPMeshValid(p_mpmesh);
@@ -1046,6 +1042,23 @@ void polympo_setOwningProc_f(MPMesh_ptr p_mpmesh, const int nCells, const int* a
10461042
p_mesh->setOwningProc(owningProc);
10471043
}
10481044

1045+
void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){
1046+
checkMPMeshValid(p_mpmesh);
1047+
auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh;
1048+
PMT_ALWAYS_ASSERT(p_mesh->meshEditable());
1049+
Kokkos::View<int*, Kokkos::HostSpace> arrayHost("arrayHost", nCells);
1050+
for (int i = 0; i < nCells; i++) {
1051+
arrayHost(i) = array[i] - 1; // Decrease each value by 1
1052+
}
1053+
//check the size
1054+
PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements());
1055+
1056+
Kokkos::View<int*> elmGlobal("elmGlobal",nCells);
1057+
Kokkos::deep_copy(elmGlobal, arrayHost);
1058+
p_mesh->setElmGlobal(elmGlobal);
1059+
}
1060+
1061+
10491062
void polympo_enableTiming_f(){
10501063
pumipic::EnableTiming();
10511064
}

‎src/pmpo_c.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ void polympo_setMeshNumEdgesPerElm_f(MPMesh_ptr p_mpmesh, const int nCells, cons
5757
void polympo_setMeshElm2VtxConn_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int nCells, const int* array);
5858
void polympo_setMeshElm2ElmConn_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int nCells, const int* array);
5959
void polympo_setOwningProc_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array);
60+
void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array);
61+
6062

6163
//Mesh fields
6264
int polympo_getMeshFVtxType_f();

‎src/pmpo_fortran.f90‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -698,6 +698,21 @@ subroutine polympo_setOwningProc(mpMesh, nCells, array) &
698698
integer(c_int), value :: nCells
699699
type(c_ptr), intent(in), value :: array
700700
end subroutine
701+
!---------------------------------------------------------------------------
702+
!> @brief set the owning process array
703+
!> @param mpmesh(in/out) MPMesh object
704+
!> @param nCells(in) number of cells
705+
!> @param array(in) input mesh cell to process array
706+
!---------------------------------------------------------------------------
707+
subroutine polympo_setElmGlobal(mpMesh, nCells, array) &
708+
bind(C, NAME='polympo_setElmGlobal_f')
709+
use :: iso_c_binding
710+
type(c_ptr), value :: mpMesh
711+
integer(c_int), value :: nCells
712+
type(c_ptr), intent(in), value :: array
713+
end subroutine
714+
715+
701716
!---------------------------------------------------------------------------
702717
!> @brief calculate the MPs from given mesh vertices rotational latitude
703718
!> longitude, update the MP slices

‎src/pmpo_materialPoints.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,12 @@ PS* createDPS(int numElms, int numMPs, MPSView<MPF_Cur_Pos_XYZ> positions, IntVi
4040
return dps;
4141
}
4242

43-
PS* createDPS(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID) {
43+
PS* createDPS(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID, IntView elm2global) {
4444
PS::kkGidView elmGids("elementGlobalIds", numElms); //TODO - initialize this to [0..numElms)
45+
Kokkos::parallel_for("setGids", numElms, KOKKOS_LAMBDA(const int elm){
46+
elmGids(elm) = elm2global(elm);
47+
});
48+
4549
auto mpInfo = createInternalMemberViews(numMPs, mp2elm, mpAppID);
4650
Kokkos::TeamPolicy<Kokkos::DefaultExecutionSpace> policy(numElms,Kokkos::AUTO);
4751
auto dps = new DPS<MaterialPointTypes>(policy, numElms, numMPs, mpsPerElm, elmGids, mp2elm, mpInfo);
@@ -57,8 +61,8 @@ MaterialPoints::MaterialPoints(int numElms, int numMPs, MPSView<MPF_Cur_Pos_XYZ>
5761
operating_mode = MP_RELEASE;
5862
};
5963

60-
MaterialPoints::MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID) {
61-
MPs = createDPS(numElms, numMPs, mpsPerElm, mp2elm, mpAppID);
64+
MaterialPoints::MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID, IntView elm2global){
65+
MPs = createDPS(numElms, numMPs, mpsPerElm, mp2elm, mpAppID, elm2global);
6266
updateMaxAppID();
6367
operating_mode = MP_RELEASE;
6468
};

‎src/pmpo_materialPoints.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ class MaterialPoints {
130130
public:
131131
MaterialPoints() : MPs(nullptr) {};
132132
MaterialPoints(int numElms, int numMPs, MPSView<MPF_Cur_Pos_XYZ> positions, IntView mpsPerElm, IntView mp2elm);
133-
MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID);
133+
MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID, IntView elm2global);
134134
~MaterialPoints();
135135

136136
void rebuild(IntView addedMP2elm, IntView addedMPAppID);

‎src/pmpo_mesh.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,5 +77,11 @@ namespace polyMPO{
7777
IntView Mesh::getElm2Process() {
7878
return owningProc_;
7979
}
80+
81+
IntView Mesh::getElmGlobal() {
82+
return globalElm_;
83+
}
84+
85+
8086

8187
} // namespace polyMPO

‎src/pmpo_mesh.hpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ class Mesh {
8888
IntVtx2ElmView elm2VtxConn_;
8989
IntElm2ElmView elm2ElmConn_;
9090
IntView owningProc_;
91-
91+
IntView globalElm_;
9292
//start of meshFields
9393
MeshFView<MeshF_VtxCoords> vtxCoords_;
9494
MeshFView<MeshF_VtxRotLat> vtxRotLat_;
@@ -163,6 +163,10 @@ class Mesh {
163163
void setOwningProc(IntView owningProc) {PMT_ALWAYS_ASSERT(meshEdit_);
164164
owningProc_ = owningProc; }
165165

166+
void setElmGlobal(IntView globalElm) {PMT_ALWAYS_ASSERT(meshEdit_);
167+
globalElm_ = globalElm; }
168+
IntView getElmGlobal();
169+
166170
void computeRotLatLonIncr();
167171
};
168172

0 commit comments

Comments
 (0)