New particleID can be found from MPAS in polyMPO

Author: dhyan1272

src/pmpo_MPMesh.cpp

@@ -2,7 +2,7 @@
 #include "pmpo_utils.hpp"
 #include "pmpo_MPMesh.hpp"
 #include "pmpo_wachspressBasis.hpp"
-
+#include <unistd.h>
 namespace polyMPO{
 
 void printVTP_mesh(MPMesh& mpMesh, int printVTPIndex=-1);
@@ -345,7 +345,7 @@ void MPMesh::push(){
 
   static int count=0;
   std::cout<<"Push"<<"  "<<count<<std::endl;
-  if(count>473) exit(-1); 
+  
   Kokkos::Timer timer;
 
   p_mesh->computeRotLatLonIncr();
@@ -358,7 +358,7 @@ void MPMesh::push(){
 
   bool anyIsMigrating = false;
   do {
-    CVTTrackingElmCenterBased(count); // move to Tgt_XYZ
+    CVTTrackingElmCenterBased(); // move to Tgt_XYZ
     assert(cudaDeviceSynchronize() == cudaSuccess); 
     p_MPs->updateMPSlice<MPF_Cur_Pos_XYZ, MPF_Tgt_Pos_XYZ>(); // Tgt_XYZ becomes Cur_XYZ
     p_MPs->updateMPSlice<MPF_Cur_Pos_Rot_Lat_Lon, MPF_Tgt_Pos_Rot_Lat_Lon>(); // Tgt becomes Cur
@@ -369,12 +369,26 @@ void MPMesh::push(){
       p_MPs->migrate();
     else
       p_MPs->rebuild();
+    printf("Done till here 0\n");
 
     p_MPs->updateMPElmID(); //update mpElm IDs slices
     reconstructSlices(); 
   } 
   while (anyIsMigrating);
-  count ++; 
+  
+  count ++;
+
+  volatile int i = 0;
+  char hostname[256];
+  gethostname(hostname, sizeof(hostname));
+  MPI_Comm comm = p_MPs->getMPIComm(); 
+  int comm_rank;
+  MPI_Comm_rank(comm, &comm_rank);
+  printf("Rank %d PID %d on %s ready for attach\n", comm_rank, getpid(), hostname);
+  fflush(stdout);
+  if(count==480) sleep(100);
+
+  printf("Done till here\n");
   pumipic::RecordTime("PolyMPO_push", timer.seconds());
 }
 

src/pmpo_c.cpp

@@ -212,12 +212,13 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI
   //p_MPs->setAppIDFunc(getNextAppID);
 }
 
-void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void*arg2, void*arg3,
-  const int arg4, const int arg5) {
+//arg1 is blockPtr, arg2 is blockSize and arg3 is nCells
+void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void*arg1,
+  const int arg2, const int arg3) {
   checkMPMeshValid(p_mpmesh);
   auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs;
-  std::function<int(int)> polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4, arg5](int iCell) 
-    { int iParticleNew; getMPASAppID(arg1, arg2, arg3, arg4, iCell, iParticleNew, arg5); return iParticleNew;};
+  std::function<int(int)> polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3](int iCell) 
+    { int iParticleNew=-1; getMPASAppID(arg1, arg2, arg3, iCell, iParticleNew); return iParticleNew;};
   p_MPs->setAppIDFunc(polyMPO_getMPASAppID);
 }
 

src/pmpo_c.h

@@ -24,8 +24,7 @@ void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, const int numMPs, const int*
 void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh);
 
 void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs);
-void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void* arg2, void*arg3,
-  const int arg4, const int arg5);
+void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, const int arg2, const int arg3);
 
 void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs);
 void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag);

src/pmpo_defines.h

@@ -8,7 +8,7 @@
 typedef void* MPMesh_ptr;
 //Function that receives void* and returns an int
 typedef int (*IntVoidFunc)(void*);
-typedef void (*VoidVoidFunc)(void*, void*, void*, const int, int, int, const int);
+typedef void (*VoidVoidFunc)(void*, const int, const int, const int, int&);
 
 using space_t = Kokkos::DefaultExecutionSpace::memory_space;
 

src/pmpo_fortran.f90

@@ -118,16 +118,14 @@ subroutine polympo_setAppIDFunc(mpMesh, getNext, appIDs) &
   !> @param appIDs(in) Pointer to opaque data application data structure (that may contain all available app IDs)
   !---------------------------------------------------------------------------
   subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, &
-    arg1, arg2, arg3, arg4, arg5) &
+    arg1, arg2, arg3) &
     bind(C, NAME='polympo_setMPASAppIDFunc_f')
     use :: iso_c_binding
     type(c_ptr), value :: mpMesh
     type(c_funptr), value :: getMPASAppID
     type(c_ptr), value :: arg1
-    type(c_ptr), value :: arg2
-    type(c_ptr), value :: arg3
-    integer(c_int), intent(in), value :: arg4
-    integer(c_int), intent(in), value :: arg5
+    integer(c_int), intent(in), value :: arg2
+    integer(c_int), intent(in), value :: arg3
   end subroutine
 
 

src/pmpo_materialPoints.cpp

@@ -158,9 +158,9 @@ void MaterialPoints::migrate() {
   parallel_for(setMigrationFields, "setMigrationFields");
   MPs->migrate(new_elem, new_process);
 
-  //SINCE REBUILT, mpAppID needs to be be recalled
+  //AS REBUILT, mpAppID needs to be be recalled
   mpAppID =  getData<polyMPO::MPF_MP_APP_ID>();
-  //Count MPs that have -1 appID, so that we can count no of MPs received
+  //Count MPs that have -1 appID, so that we can count no of MPs received by a rank
   Kokkos::View<int*> numReceivedMPs("numReceivedMPs", 1);
   Kokkos::deep_copy(numReceivedMPs, 0);
   auto countnewMPs = PS_LAMBDA(const int& e, const int& mp, const bool& mask) {
@@ -170,34 +170,56 @@ void MaterialPoints::migrate() {
     }
   };
   parallel_for(countnewMPs, "countReceivedPtcls");
-  Kokkos::fence();
-  //Bring them to CPU and print particles received
   auto numReceivedMPs_host = Kokkos::create_mirror_view(numReceivedMPs);
   Kokkos::deep_copy(numReceivedMPs_host, numReceivedMPs);
-  if(numReceivedMPs_host(0))
-    std::cout <<"Rank "<<rank<<" received "<<numReceivedMPs_host(0)<< "MPs \n";
 
-  //Another array that contains element id of the the MPs that have migrated
+  //Another array that contains new element id of the the MPs that have migrated
+  //Array size is #elements received
   Kokkos::View<int*> receivedMPs2Elm("ReceivedMPs2Elm", numReceivedMPs_host(0));
   Kokkos::View<int*> counter("counter", 1);
+  Kokkos::deep_copy(counter, 0);
   auto set_new_elem = PS_LAMBDA(const int& e, const int& mp, const bool& mask) {
     if(mask){
       if (mpAppID(mp) == -1){
-        auto xx=Kokkos::atomic_fetch_add(&counter(0), 1);
-        receivedMPs2Elm(xx) = e;
+        auto count_temp=Kokkos::atomic_fetch_add(&counter(0), 1);
+        receivedMPs2Elm(count_temp) = e;
       }
     }
   };
-  parallel_for(set_new_elem, "countReceivedPtcls");  
-  //NEED AN ASSERT ELEMNT DOING COUNTER=#RECEIVED MPs
+  parallel_for(set_new_elem, "countReceivedPtcls");
+  //Bring them to CPU so that elm_id can be passed and a new mpAppID can be found from CPU
   auto receivedMPs2Elm_host = Kokkos::create_mirror_view(receivedMPs2Elm);
   Kokkos::deep_copy(receivedMPs2Elm_host, receivedMPs2Elm);
-
+  auto counter_host = Kokkos::create_mirror_view(counter);
+  Kokkos::deep_copy(counter_host, counter);
+  assert(numReceivedMPs_host(0)==counter_host(0));
+  if(counter_host(0))
+    std::cout <<"Rank "<<rank<<" received "<<counter_host(0)<<"  "<< numReceivedMPs_host(0) <<" MPs \n";
+ 
+  //Find new AppIDS from CPU-MPAS and store so that they can be added to MPs that have appID=-1
+  std::vector<int> appIDs;
   for(int i=0; i<numReceivedMPs_host(0); i++){
-    printf("Trying to find an ID for a material point\n");
-    auto xx = getNextAppID(receivedMPs2Elm_host(i)+1);
+    auto app_id_new = getNextAppID(receivedMPs2Elm_host(i)+1);
+    printf("Trying to find ID for MP located in %d and it's %d \n", receivedMPs2Elm_host(i), app_id_new);
+    appIDs.push_back(app_id_new);
   }
+  kkViewHostU<int*> appIDs_host(appIDs.data(), appIDs.size());
+  Kokkos::View<int*> appIDs_d("appIDsDevice", appIDs.size());
+  Kokkos::deep_copy(appIDs_d, appIDs_host);
 
+  //If the mpAppID is -1 assign a new mpAppID
+  Kokkos::deep_copy(counter, 0);
+  auto set_appID = PS_LAMBDA(const int& e, const int& mp, const bool& mask) {
+    if(mask){
+      if (mpAppID(mp) == -1){
+        auto count_temp=Kokkos::atomic_fetch_add(&counter(0), 1);
+        mpAppID(mp)=appIDs_d(count_temp)-1;
+	printf("Count_temp %d and new appID %d \n", count_temp, mpAppID(mp));
+      }
+    }
+  };
+  parallel_for(set_appID, "setApplicationIDs");
+   
   if (getOpMode() == polyMPO::MP_DEBUG)
     printf("Material point migration: %f\n", timer.seconds());
   pumipic::RecordTime("PolyMPO_migrate", timer.seconds());