Towards multi Process

Author: dhyan1272

src/pmpo_MPMesh.cpp

@@ -212,10 +212,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){
     };
     p_MPs->parallel_for(CVTElmCalc,"CVTTrackingElmCenterBasedCalc");
 
-    assert(cudaDeviceSynchronize()==cudaSuccess);
-    MPI_Barrier(MPI_COMM_WORLD);
-    printf("After Tracking \n");
-    
     if(printVTPIndex>=0 && numMPs>0){
         Vec3dView::HostMirror h_history = Kokkos::create_mirror_view(history);
         Vec3dView::HostMirror h_resultLeft = Kokkos::create_mirror_view(resultLeft);
@@ -228,7 +224,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){
         Kokkos::deep_copy(h_resultRight, resultRight);
         Kokkos::deep_copy(h_mpTgtPos, mpTgtPosArray);
 	Kokkos::deep_copy(h_counter, counter);
-        printf("Host counter value: %d\n", h_counter(0));
         // printVTP file
         char* fileOutput = (char *)malloc(sizeof(char) * 256); 
         sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d_%d.vtp", comm_rank, printVTPIndex);
@@ -253,10 +248,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){
         fprintf(pFile,"        </DataArray>\n      </Lines>\n    </Piece>\n  </PolyData>\n</VTKFile>\n");
         fclose(pFile);
     }
-    assert(cudaDeviceSynchronize()==cudaSuccess);
-    MPI_Barrier(MPI_COMM_WORLD);
-    printf("After printing particle paths \n");
-
 
     pumipic::RecordTime("PolyMPO_CVTTrackingElmCenterBased", timer.seconds());
 }

src/pmpo_c.cpp

@@ -212,6 +212,15 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI
   p_MPs->setAppIDFunc(getNextAppID);
 }
 
+void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void*arg2, void*arg3,
+  void* arg4, void* arg5, void*arg6) {
+  checkMPMeshValid(p_mpmesh);
+  auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs;
+  std::function<int()> polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4, arg5, arg6]() 
+    {getMPASAppID(arg1, arg2, arg3, arg4, arg5, arg6); return *(static_cast<int*>(arg5));};
+  p_MPs->setAppIDFunc(polyMPO_getMPASAppID);
+}
+
 void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh,
                            const int numMPs,
                            int* elmIDs){

src/pmpo_c.h

@@ -22,7 +22,11 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm);
 void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int numElms, const int numMPs, int* mpsPerElm, const int* mp2Elm, const int* isMPActive);
 void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, const int numMPs, const int* allTgtMpElmIn, const int* addedMPMask);
 void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh);
+
 void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs);
+void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void* arg2, void*arg3,
+  void* arg4, void*arg5, void* arg6);
+
 void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs);
 void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag);
 

src/pmpo_defines.h

@@ -8,6 +8,7 @@
 typedef void* MPMesh_ptr;
 //Function that receives void* and returns an int
 typedef int (*IntVoidFunc)(void*);
+typedef void (*VoidVoidFunc)(void*, void*, void*, void*, void*, void*);
 
 using space_t = Kokkos::DefaultExecutionSpace::memory_space;
 

src/pmpo_fortran.f90

@@ -110,6 +110,28 @@ subroutine polympo_setAppIDFunc(mpMesh, getNext, appIDs) &
     type(c_funptr), value :: getNext
     type(c_ptr), value :: appIDs
   end subroutine
+  
+  !---------------------------------------------------------------------------
+  !> @brief Stores pointer to appID data structure and a function to retrieve them used in migration
+  !> @param mpmesh(in/out) MPMesh object
+  !> @param getNext(in) Pointer to function that returns next App IDs
+  !> @param appIDs(in) Pointer to opaque data application data structure (that may contain all available app IDs)
+  !---------------------------------------------------------------------------
+  subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, &
+    arg1, arg2, arg3, arg4, arg5, arg6) &
+    bind(C, NAME='polympo_setMPASAppIDFunc_f')
+    use :: iso_c_binding
+    type(c_ptr), value :: mpMesh
+    type(c_funptr), value :: getMPASAppID
+    type(c_ptr), value :: arg1
+    type(c_ptr), value :: arg2
+    type(c_ptr), value :: arg3
+    type(c_ptr), value :: arg4
+    type(c_ptr), value :: arg5
+    type(c_ptr), value :: arg6
+  end subroutine
+  
+  
   !---------------------------------------------------------------------------
   !> @brief get the current element ID MP array from a polympo array
   !> @param mpmesh(in/out) MPMesh object

src/pmpo_materialPoints.cpp

@@ -138,6 +138,7 @@ void MaterialPoints::migrate() {
   Kokkos::Timer timer;
   auto MPs2Elm = getData<MPF_Tgt_Elm_ID>();
   auto MPs2Proc = getData<MPF_Tgt_Proc_ID>();
+  auto mpAppID =  getData<polyMPO::MPF_MP_APP_ID>();
 
   IntView new_elem("new_elem", MPs->capacity());
   IntView new_process("new_process", MPs->capacity());
@@ -148,11 +149,40 @@ void MaterialPoints::migrate() {
     if (mask) {
       new_elem(mp) = MPs2Elm(mp);
       new_process(mp) = MPs2Proc(mp);
+      if(rank!=new_process(mp)){
+        mpAppID(mp)=-1;
+	printf("Particle migrated and so its AppID is -1\n");
+      }
     }
   };
   parallel_for(setMigrationFields, "setMigrationFields");
   MPs->migrate(new_elem, new_process);
 
+  //Since rebuilt
+  mpAppID =  getData<polyMPO::MPF_MP_APP_ID>();
+  //Count MPs that have -1 appID, so that we can count no of MPs received
+  Kokkos::View<int*> numReceivedMPs("numReceivedMPs", 1);
+  Kokkos::deep_copy(numReceivedMPs, 0);
+  auto countnewMPs = PS_LAMBDA(const int& e, const int& mp, const bool& mask) {
+    if(mask){
+      if (mpAppID(mp) == -1)
+        Kokkos::atomic_add(&numReceivedMPs(0), 1);
+    }
+  };
+  parallel_for(countnewMPs, "countReceivedPtcls");
+  Kokkos::fence();
+  
+  auto numReceivedMPs_host = Kokkos::create_mirror_view(numReceivedMPs);
+  Kokkos::deep_copy(numReceivedMPs_host, numReceivedMPs);
+  if(numReceivedMPs_host(0))
+    std::cout <<"Rank "<<rank<<" received "<<numReceivedMPs_host(0)<< "MPs \n";
+  
+  std::vector<int> added_mpIDs(numReceivedMPs_host(0));
+  for(int i=0; i<numReceivedMPs_host(0); i++){
+    printf("Trying to find an ID for a material point\n");
+    auto xx = getNextAppID();
+  }
+
   if (getOpMode() == polyMPO::MP_DEBUG)
     printf("Material point migration: %f\n", timer.seconds());
   pumipic::RecordTime("PolyMPO_migrate", timer.seconds());