Multi process debugging

Author: dhyan1272

src/pmpo_MPMesh.cpp

@@ -127,7 +127,12 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){
     auto MPs2Elm = p_MPs->getData<MPF_Tgt_Elm_ID>();
     auto MPs2Proc = p_MPs->getData<MPF_Tgt_Proc_ID>();
     auto elm2Process = p_mesh->getElm2Process();
-    
+   
+    Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){
+      int pp_id=elm2Process(iElm);
+      printf("Mesh elm %d owning element %d \n", iElm, pp_id);
+    });
+
     if(printVTPIndex>=0) {
       printVTP_mesh(printVTPIndex);
     }
@@ -324,15 +329,37 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) {
 }
 
 void MPMesh::push(){
+  static int count=0;
+  std::cout<<__FUNCTION__<<" "<<count<<std::endl;
+  count++;
+  if(count>1) exit(1);
   Kokkos::Timer timer;
   p_mesh->computeRotLatLonIncr();
+  
+  assert(cudaDeviceSynchronize() == cudaSuccess); 
+  MPI_Barrier(MPI_COMM_WORLD);
+  printf("FooPush\n");
+
   sphericalInterpolation<MeshF_RotLatLonIncr>(*this);
+  assert(cudaDeviceSynchronize() == cudaSuccess); 
+  MPI_Barrier(MPI_COMM_WORLD);
+  printf("FooPush \n");
+
   p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); // set Tgt_XYZ
+  assert(cudaDeviceSynchronize() == cudaSuccess); 
+  MPI_Barrier(MPI_COMM_WORLD);
+  printf("FooPush\n");
+
   auto elm2Process = p_mesh->getElm2Process();
+  assert(cudaDeviceSynchronize() == cudaSuccess); 
+  MPI_Barrier(MPI_COMM_WORLD);
+  printf("FooPush \n");
 
   bool anyIsMigrating = false;
   do {
     CVTTrackingElmCenterBased(); // move to Tgt_XYZ
+    assert(cudaDeviceSynchronize() == cudaSuccess); 
+    printf("FooPushInside\n");
     p_MPs->updateMPSlice<MPF_Cur_Pos_XYZ, MPF_Tgt_Pos_XYZ>(); // Tgt_XYZ becomes Cur_XYZ
     p_MPs->updateMPSlice<MPF_Cur_Pos_Rot_Lat_Lon, MPF_Tgt_Pos_Rot_Lat_Lon>(); // Tgt becomes Cur
     if (elm2Process.size() > 0)
@@ -343,6 +370,9 @@ void MPMesh::push(){
     reconstructSlices(); 
   } 
   while (anyIsMigrating);
+  assert(cudaDeviceSynchronize() == cudaSuccess); 
+  MPI_Barrier(MPI_COMM_WORLD);
+  printf("FooPush\n");
 
   pumipic::RecordTime("PolyMPO_push", timer.seconds());
 }

src/pmpo_MPMesh_assembly.hpp

@@ -104,7 +104,7 @@ void MPMesh::resetPreComputeFlag(){
 }
 
 void MPMesh::computeMatricesAndSolve(){
-
+  Kokkos::Timer timer;
   //Mesh Information
   auto elm2VtxConn = p_mesh->getElm2VtxConn();  
   int numVtx = p_mesh->getNumVertices();
@@ -205,11 +205,12 @@ void MPMesh::computeMatricesAndSolve(){
       VtxCoeffs(vtx,i)=coeff[i];
   });
   this->precomputedVtxCoeffs = VtxCoeffs;
+  pumipic::RecordTime("PolyMPO_Calculate_MLS_Coeff", timer.seconds());
 }
 
 template <MeshFieldIndex meshFieldIndex>
 void MPMesh::assemblyVtx1() {
-   
+  Kokkos::Timer timer; 
   //If no reconstruction till now calculate the coeffs
   if (!isPreComputed) {
     computeMatricesAndSolve();
@@ -270,6 +271,7 @@ void MPMesh::assemblyVtx1() {
     for(int k=0; k<numEntries; k++)
       meshField(vtx, k) = reconVals(vtx,k);
   });
+  pumipic::RecordTime("PolyMPO_Reconstruct_Vtx1", timer.seconds());
 }
 
 template <MeshFieldIndex meshFieldIndex>

src/pmpo_c.cpp

@@ -61,17 +61,19 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm){
 
 void polympo_createMPs_f(MPMesh_ptr p_mpmesh,
                        const int numElms,
-                       const int numMPs, // total number of MPs which is GREATER than or equal to number of active MPs
+                       const int numMPs, // total nof of MPs which is >= no of active MPs
                        int* mpsPerElm,
                        const int* mp2Elm,
                        const int* isMPActive) {
   checkMPMeshValid(p_mpmesh);
-
+  std::cout<<__FUNCTION__<<std::endl;
   //the mesh must be fixed/set before adding MPs
   auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh;
   PMT_ALWAYS_ASSERT(!p_mesh->meshEditable());
   PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms);
 
+  //Find the total no of MPs across all ranks
+  //And loop over all MPs and find the smallest element id associated across a MP
   int numActiveMPs = 0;
   int minElmID = numElms+1;
   for(int i = 0; i < numMPs; i++) {
@@ -82,20 +84,27 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh,
       }
     }
   }
-  //TODO do we care about empty ranks? check just in case...
-  PMT_ALWAYS_ASSERT(numActiveMPs>0);
-
-  int firstElmWithMPs=-1;
+  long long globalNumActiveMPs = 0;
+  int globalMinElmID;
+  MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD); 
+  MPI_Allreduce(&minElmID, &globalMinElmID, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
+  PMT_ALWAYS_ASSERT(globalNumActiveMPs>0);
+  
+  //Loop over all mesh elements 0,1,... and find the first element that has an associated MP
+  int firstElmWithMPs=numElms+1;
   for (int i=0; i<numElms; i++) {
     if(mpsPerElm[i]) {
       firstElmWithMPs = i;
       break;
     }
   }
+  int globalFirstElmWithMPs;
+  MPI_Allreduce(&firstElmWithMPs, &globalFirstElmWithMPs, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
+  
   int offset = -1;
-  if(minElmID-firstElmWithMPs==1) {
+  if(globalMinElmID-globalFirstElmWithMPs==1) {
     offset = 1;
-  }else if (minElmID-firstElmWithMPs==0){
+  }else if (globalMinElmID-globalFirstElmWithMPs==0){
     offset = 0;
   }else {
     fprintf(stderr,"The minElmID is incorrect! Offset is wrong!\n");
@@ -116,12 +125,18 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh,
   auto mpsPerElm_d = create_mirror_view_and_copy(mpsPerElm, numElms);
   auto active_mp2Elm_d = create_mirror_view_and_copy(active_mp2Elm.data(), numActiveMPs);
   auto active_mpIDs_d = create_mirror_view_and_copy(active_mpIDs.data(), numActiveMPs);
-
+  
   delete ((polyMPO::MPMesh*)p_mpmesh)->p_MPs;
   ((polyMPO::MPMesh*)p_mpmesh)->p_MPs =
      new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d);
+
   auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs;
   p_MPs->setElmIDoffset(offset);
+  
+  assert(cudaDeviceSynchronize() == cudaSuccess); 
+  MPI_Barrier(MPI_COMM_WORLD);
+  printf("Foo1\n");
+
 }
 
 void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh,