Running on multiple processors

Author: dhyan1272

src/pmpo_MPMesh.cpp

@@ -129,20 +129,13 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){
     auto elm2Process = p_mesh->getElm2Process();
     auto elm2global = p_mesh->getElmGlobal();
 
-    
     MPI_Comm comm = p_MPs->getMPIComm(); 
     int comm_rank;
     MPI_Comm_rank(comm, &comm_rank);
-    assert(cudaDeviceSynchronize() == cudaSuccess);
-    MPI_Barrier(MPI_COMM_WORLD);
-    printf("FooTracking \n");
     Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){
       int pp_id=elm2Process(iElm);
-      if(iElm<3 || iElm==1470 || iElm==1471 || iElm==1472 )
-        printf("Mesh elm %d Process %d owning proc %d global %d \n", iElm, comm_rank, pp_id, elm2global(iElm));
     });
 
-
     if(printVTPIndex>=0) {
       printVTP_mesh(printVTPIndex);
     }
@@ -341,8 +334,9 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) {
 void MPMesh::push(){
 
   static int count=0;
-  std::cout<<__FUNCTION__<<" "<<count<<std::endl;
+  std::cout<<"Push"<<"  "<<count<<std::endl;
   count++;
+  
   Kokkos::Timer timer;
 
   p_mesh->computeRotLatLonIncr();
@@ -357,14 +351,16 @@ void MPMesh::push(){
   do {
     CVTTrackingElmCenterBased(); // move to Tgt_XYZ
     assert(cudaDeviceSynchronize() == cudaSuccess); 
-    printf("FooPushDoWhile\n");
     p_MPs->updateMPSlice<MPF_Cur_Pos_XYZ, MPF_Tgt_Pos_XYZ>(); // Tgt_XYZ becomes Cur_XYZ
     p_MPs->updateMPSlice<MPF_Cur_Pos_Rot_Lat_Lon, MPF_Tgt_Pos_Rot_Lat_Lon>(); // Tgt becomes Cur
-    if (elm2Process.size() > 0)
-        anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->migrate());
+    
+    bool anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->check_migrate());
+   
+    if(anyIsMigrating)
+      p_MPs->migrate();
     else
-        p_MPs->rebuild(); //rebuild pumi-pic
-    printf("Is migrating %d \n", anyIsMigrating);
+      p_MPs->rebuild();
+    
     p_MPs->updateMPElmID(); //update mpElm IDs slices
     reconstructSlices(); 
   } 

src/pmpo_c.cpp

@@ -66,11 +66,10 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh,
                        const int* mp2Elm,
                        const int* isMPActive) {
   checkMPMeshValid(p_mpmesh);
-  std::cout<<__FUNCTION__<<std::endl;
   //the mesh must be fixed/set before adding MPs
   auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh;
   PMT_ALWAYS_ASSERT(!p_mesh->meshEditable());
-  //PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms);
+  PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms);
 
   //Find the total no of MPs across all ranks
   //And loop over all MPs and find the smallest element id associated across a MP
@@ -100,7 +99,7 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh,
   int globalFirstElmWithMPs;
   MPI_Allreduce(&firstElmWithMPs, &globalFirstElmWithMPs, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
 
-  printf("With a MP, globally smallest mesh elm %d and first elm %d \n", globalMinElmID, globalFirstElmWithMPs);
+  //printf("With a MP, globally smallest mesh elm %d and first elm %d \n", globalMinElmID, globalFirstElmWithMPs);
 
   int offset = -1;
   if(globalMinElmID-globalFirstElmWithMPs==1) {

src/pmpo_materialPoints.cpp

@@ -113,22 +113,41 @@ void MaterialPoints::setMPIComm(MPI_Comm comm) {
   mpi_comm = comm;
 }
 
-bool MaterialPoints::migrate() {
+bool MaterialPoints::check_migrate(){
+  Kokkos::Timer timer;
+  auto MPs2Elm = getData<MPF_Tgt_Elm_ID>();
+  auto MPs2Proc = getData<MPF_Tgt_Proc_ID>();
+
+  IntView isMigrating("isMigrating", 1);
+
+  int rank;
+  MPI_Comm_rank(mpi_comm, &rank);
+  auto setMigrationFields = PS_LAMBDA(const int& e, const int& mp, const bool& mask) {
+    if (mask) {
+      if (MPs2Proc(mp) != rank) isMigrating(0) = 1;
+    }
+  };
+  parallel_for(setMigrationFields, "setMigrationFields");
+  if (getOpMode() == polyMPO::MP_DEBUG)
+    printf("Material point check migration: %f\n", timer.seconds());
+  pumipic::RecordTime("PolyMPO_check_migrate", timer.seconds());
+  return pumipic::getLastValue(isMigrating) > 0;
+}
+
+void MaterialPoints::migrate() {
   Kokkos::Timer timer;
   auto MPs2Elm = getData<MPF_Tgt_Elm_ID>();
   auto MPs2Proc = getData<MPF_Tgt_Proc_ID>();
 
   IntView new_elem("new_elem", MPs->capacity());
   IntView new_process("new_process", MPs->capacity());
-  IntView isMigrating("isMigrating", 1);
 
   int rank;
   MPI_Comm_rank(mpi_comm, &rank);
   auto setMigrationFields = PS_LAMBDA(const int& e, const int& mp, const bool& mask) {
     if (mask) {
       new_elem(mp) = MPs2Elm(mp);
       new_process(mp) = MPs2Proc(mp);
-      if (new_process(mp) != rank) isMigrating(0) = 1;
     }
   };
   parallel_for(setMigrationFields, "setMigrationFields");
@@ -137,7 +156,6 @@ bool MaterialPoints::migrate() {
   if (getOpMode() == polyMPO::MP_DEBUG)
     printf("Material point migration: %f\n", timer.seconds());
   pumipic::RecordTime("PolyMPO_migrate", timer.seconds());
-  return pumipic::getLastValue(isMigrating) > 0;
 }
 
 bool MaterialPoints::rebuildOngoing() { return rebuildFields.ongoing; }

src/pmpo_materialPoints.hpp

@@ -138,7 +138,8 @@ class MaterialPoints {
     void finishRebuild();
     bool rebuildOngoing();
 
-    bool migrate();
+    bool check_migrate();
+    void migrate();
     MPI_Comm getMPIComm();
     void setMPIComm(MPI_Comm comm);