use sets of point marker pairs for periodic comm preprocessing

Author: bmunguia

Common/src/geometry/CGeometry.cpp

@@ -792,15 +792,18 @@ void CGeometry::PreprocessPeriodicComms(CGeometry* geometry, CConfig* config) {
   nPoint_Send_All[0] = 0;
   int* nPoint_Recv_All = new int[size + 1];
   nPoint_Recv_All[0] = 0;
-  int* nPoint_Flag = new int[size];
 
-  for (iRank = 0; iRank < size; iRank++) {
-    nPoint_Send_All[iRank] = 0;
-    nPoint_Recv_All[iRank] = 0;
-    nPoint_Flag[iRank] = -1;
-  }
-  nPoint_Send_All[size] = 0;
-  nPoint_Recv_All[size] = 0;
+  /*--- Store a set of unique (point, marker) pairs per destination rank. ---*/
+  using PointMarkerPair = std::pair<unsigned long, unsigned long>;
+
+  struct PairHash {
+    std::size_t operator()(const PointMarkerPair& p) const {
+      return std::hash<unsigned long>{}(p.first) ^ (std::hash<unsigned long>{}(p.second) << 1);
+    }
+  };
+
+  using PointMarkerSet = std::unordered_set<PointMarkerPair, PairHash>;
+  std::vector<PointMarkerSet> Points_Send_All(size);
 
   /*--- Loop through all of our periodic markers and track
    our sends with each rank. ---*/
@@ -821,19 +824,19 @@ void CGeometry::PreprocessPeriodicComms(CGeometry* geometry, CConfig* config) {
 
           iRank = static_cast<int>(geometry->vertex[iMarker][iVertex]->GetDonorProcessor());
 
-          /*--- If we have not visited this point last, increment our
-           number of points that must be sent to a particular proc. ---*/
-
-          if ((nPoint_Flag[iRank] != static_cast<int>(iPoint))) {
-            nPoint_Flag[iRank] = static_cast<int>(iPoint);
-            nPoint_Send_All[iRank + 1] += 1;
-          }
+          /*--- Store the (point, marker) pair in the set for the destination rank. ---*/
+          Points_Send_All[iRank].insert(std::make_pair(iPoint, static_cast<unsigned long>(iMarker)));
         }
       }
     }
   }
 
-  delete[] nPoint_Flag;
+  for (iRank = 0; iRank < size; iRank++) {
+    nPoint_Send_All[iRank + 1] = Points_Send_All[iRank].size();
+    nPoint_Recv_All[iRank + 1] = 0;
+  }
+  // nPoint_Send_All[size] = 0;
+  nPoint_Recv_All[size] = 0;
 
   /*--- Communicate the number of points to be sent/recv'd amongst
    all processors. After this communication, each proc knows how
@@ -943,11 +946,15 @@ void CGeometry::PreprocessPeriodicComms(CGeometry* geometry, CConfig* config) {
   auto* idSend = new unsigned long[nPoint_PeriodicSend[nPeriodicSend] * nPackets];
   for (iSend = 0; iSend < nPoint_PeriodicSend[nPeriodicSend] * nPackets; iSend++) idSend[iSend] = 0;
 
+  /*--- Re-use set of unique points per destination rank. ---*/
+  for (iRank = 0; iRank < size; iRank++) Points_Send_All[iRank].clear();
+
   /*--- Build the lists of local index and periodic marker index values. ---*/
 
   ii = 0;
   jj = 0;
   for (iSend = 0; iSend < nPeriodicSend; iSend++) {
+    int destRank = Neighbors_PeriodicSend[iSend];
     for (iMarker = 0; iMarker < config->GetnMarker_All(); iMarker++) {
       if (config->GetMarker_All_KindBC(iMarker) == PERIODIC_BOUNDARY) {
         iPeriodic = config->GetMarker_All_PerBound(iMarker);
@@ -969,14 +976,22 @@ void CGeometry::PreprocessPeriodicComms(CGeometry* geometry, CConfig* config) {
              index on the matching periodic point and the periodic marker
              index to be communicated to the recv rank. ---*/
 
-            if (iRank == Neighbors_PeriodicSend[iSend]) {
-              Local_Point_PeriodicSend[ii] = iPoint;
-              Local_Marker_PeriodicSend[ii] = static_cast<unsigned long>(iMarker);
-              jj = ii * nPackets;
-              idSend[jj] = geometry->vertex[iMarker][iVertex]->GetDonorPoint();
-              jj++;
-              idSend[jj] = static_cast<unsigned long>(iPeriodic);
-              ii++;
+            if (iRank == destRank) {
+              /*--- Check if we have already added this (point, marker) pair 
+               for this destination rank. ---*/
+              auto pointMarkerPair = std::make_pair(iPoint, static_cast<unsigned long>(iMarker));
+              if (Points_Send_All[destRank].find(pointMarkerPair) == Points_Send_All[destRank].end()) {
+                Local_Point_PeriodicSend[ii] = iPoint;
+                Local_Marker_PeriodicSend[ii] = static_cast<unsigned long>(iMarker);
+                jj = ii * nPackets;
+                idSend[jj] = geometry->vertex[iMarker][iVertex]->GetDonorPoint();
+                jj++;
+                idSend[jj] = static_cast<unsigned long>(iPeriodic);
+                ii++;
+
+                /*--- Store the (point, marker) pair in the set for the destination rank. ---*/
+                Points_Send_All[destRank].insert(pointMarkerPair);
+              }
             }
           }
         }