some more MG MPI fixes

bigfooted · bigfooted · commit 2468017ca018 · 2025-12-15T12:22:59.000+01:00
diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp
@@ -503,6 +503,7 @@ class CConfig {
   bool MG_Smooth_Output; /*!< \brief Output per-iteration multigrid smoothing info. */
   bool MG_Implicit_Lines; /*!< \\brief Enable implicit-lines agglomeration from walls. */
   bool MG_Implicit_Debug; /*!< \brief Enable debug output for implicit-lines agglomeration. */
+  bool MG_DebugHaloCoordinates; /*!< \brief Enable halo CV coordinate validation for multigrid. */
   su2double MG_Smooth_Coeff; /*!< \brief Smoothing coefficient for multigrid correction smoothing. */
   su2double *LocationStations;        /*!< \brief Airfoil sections in wing slicing subroutine. */
 
@@ -3875,6 +3876,12 @@ class CConfig {
    */
   bool GetMG_Implicit_Debug() const { return MG_Implicit_Debug; }
 
+  /*!\
+   * \brief Get whether halo CV coordinate validation is enabled for multigrid.
+   * \return True if halo coordinate validation is enabled.
+   */
+  bool GetMG_DebugHaloCoordinates() const { return MG_DebugHaloCoordinates; }
+
   /*!\
    * \brief Get the minimum mesh size threshold used to compute effective MG levels.
    * \return Minimum mesh size per coarsest level.
diff --git a/Common/include/geometry/CMultiGridGeometry.hpp b/Common/include/geometry/CMultiGridGeometry.hpp
@@ -181,4 +181,11 @@ class CMultiGridGeometry final : public CGeometry {
    * \param[in] val_marker - Index of the boundary marker.
    */
   void SetMultiGridWallTemperature(const CGeometry* fine_grid, unsigned short val_marker) override;
+
+  /*!
+   * \brief Validate that halo CV coordinates match corresponding domain CVs on remote ranks (debug feature).
+   * \param[in] config - Definition of the particular problem.
+   * \param[in] iMesh - Multigrid level for reporting.
+   */
+  void ValidateHaloCoordinates(const CConfig* config, unsigned short iMesh) const;
 };
diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp
@@ -1969,6 +1969,9 @@ void CConfig::SetConfig_Options() {
   /*!\brief MG_IMPLICIT_DEBUG\n DESCRIPTION: Enable debug output for implicit-lines agglomeration. DEFAULT: NO \ingroup Config*/
   addBoolOption("MG_IMPLICIT_DEBUG", MG_Implicit_Debug, false);
 
+  /*!\brief MG_DEBUG_HALO_COORDINATES\n DESCRIPTION: Enable halo CV coordinate validation for multigrid (expensive MPI check). DEFAULT: NO \ingroup Config*/
+  addBoolOption("MG_DEBUG_HALO_COORDINATES", MG_DebugHaloCoordinates, false);
+
   /*!\brief MG_MIN_MESHSIZE
    \ DESCRIPTION: Minimum global mesh size (points) to allow another multigrid level. DEFAULT: 1000 \ingroup Config*/
   addUnsignedLongOption("MG_MIN_MESHSIZE", MG_Min_MeshSize, 1000);
diff --git a/Common/src/geometry/CMultiGridGeometry.cpp b/Common/src/geometry/CMultiGridGeometry.cpp
@@ -758,6 +758,107 @@ CMultiGridGeometry::CMultiGridGeometry(CGeometry* fine_grid, CConfig* config, un
            << " orphaned halo coarse CVs found - this indicates a logic error!" << endl;
     }
   }
+
+  /*--- Debug validation: Verify halo CV coordinates match domain CVs on remote ranks ---*/
+  /*--- Note: This validation is deferred until after SetVertex() is called, as vertices ---*/
+  /*--- are not yet initialized at the end of the constructor. The validation is performed ---*/
+  /*--- by calling ValidateHaloCoordinates() after SetVertex() in the driver. ---*/
+
+  /*--- For now, we skip this validation in the constructor to avoid segfaults. ---*/
+  /*--- TODO: Move this to a separate validation function called after SetVertex(). ---*/
+
+#if 0  // Disabled - causes segfault as vertex array not yet initialized
+  if (size > SINGLE_NODE && config->GetMG_DebugHaloCoordinates()) {
+
+    if (rank == MASTER_NODE) {
+      cout << "\n--- MG Halo Coordinate Validation (Level " << iMesh << ") ---" << endl;
+    }
+
+    /*--- For each SEND_RECEIVE marker pair, exchange coordinates and validate ---*/
+    for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) {
+      if ((config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) && (config->GetMarker_All_SendRecv(iMarker) > 0)) {
+        const auto MarkerS = iMarker;
+        const auto MarkerR = iMarker + 1;
+
+        const auto send_to = config->GetMarker_All_SendRecv(MarkerS) - 1;
+        const auto receive_from = abs(config->GetMarker_All_SendRecv(MarkerR)) - 1;
+
+        const auto nVertexS = nVertex[MarkerS];
+        const auto nVertexR = nVertex[MarkerR];
+
+        /*--- Allocate buffers for coordinate exchange ---*/
+        vector<su2double> Buffer_Send_Coord(nVertexS * nDim);
+        vector<su2double> Buffer_Receive_Coord(nVertexR * nDim);
+
+        /*--- Pack SEND coordinates (domain CVs being sent) ---*/
+        for (auto iVertex = 0ul; iVertex < nVertexS; iVertex++) {
+          const auto iPoint = vertex[MarkerS][iVertex]->GetNode();
+          const auto* Coord = nodes->GetCoord(iPoint);
+          for (auto iDim = 0u; iDim < nDim; iDim++) {
+            Buffer_Send_Coord[iVertex * nDim + iDim] = Coord[iDim];
+          }
+        }
+
+        /*--- Exchange coordinates ---*/
+        SU2_MPI::Sendrecv(Buffer_Send_Coord.data(), nVertexS * nDim, MPI_DOUBLE, send_to, 0,
+                          Buffer_Receive_Coord.data(), nVertexR * nDim, MPI_DOUBLE, receive_from, 0,
+                          SU2_MPI::GetComm(), MPI_STATUS_IGNORE);
+
+        /*--- Validate RECEIVE coordinates against local halo CVs ---*/
+        unsigned long nMismatch = 0;
+        su2double maxError = 0.0;
+        su2double tolerance = 1e-10;
+
+        for (auto iVertex = 0ul; iVertex < nVertexR; iVertex++) {
+          const auto iPoint = vertex[MarkerR][iVertex]->GetNode();
+          const auto* Coord_Local = nodes->GetCoord(iPoint);
+
+          su2double error = 0.0;
+          for (auto iDim = 0u; iDim < nDim; iDim++) {
+            su2double coord_remote = Buffer_Receive_Coord[iVertex * nDim + iDim];
+            su2double diff = fabs(Coord_Local[iDim] - coord_remote);
+            error += diff * diff;
+          }
+          error = sqrt(error);
+
+          if (error > tolerance) {
+            nMismatch++;
+            maxError = max(maxError, error);
+
+            if (nMismatch <= 5) {  // Only print first 5 mismatches
+              cout << "COORD MISMATCH [Rank " << rank << ", Marker " << MarkerS << "/" << MarkerR
+                   << ", Vertex " << iVertex << ", Point " << iPoint << "]: ";
+              cout << "Local=(";
+              for (auto iDim = 0u; iDim < nDim; iDim++) {
+                cout << Coord_Local[iDim];
+                if (iDim < nDim - 1) cout << ", ";
+              }
+              cout << "), Remote=(";
+              for (auto iDim = 0u; iDim < nDim; iDim++) {
+                cout << Buffer_Receive_Coord[iVertex * nDim + iDim];
+                if (iDim < nDim - 1) cout << ", ";
+              }
+              cout << "), Error=" << error << endl;
+            }
+          }
+        }
+
+        if (nMismatch > 0) {
+          cout << "WARNING [Rank " << rank << ", Marker " << MarkerS << "/" << MarkerR
+               << "]: " << nMismatch << " coordinate mismatches detected (max error: "
+               << maxError << ")" << endl;
+        } else if (nVertexR > 0) {
+          cout << "INFO [Rank " << rank << ", Marker " << MarkerS << "/" << MarkerR
+               << "]: All " << nVertexR << " halo CV coordinates match (tol=" << tolerance << ")" << endl;
+        }
+      }
+    }
+
+    if (rank == MASTER_NODE) {
+      cout << "--- End MG Halo Coordinate Validation ---\n" << endl;
+    }
+  }
+#endif  // Disabled validation code
 #endif  // HAVE_MPI
 
   /*--- Update the number of points after the MPI agglomeration ---*/
@@ -1835,3 +1936,104 @@ su2double CMultiGridGeometry::ComputeLocalCurvature(const CGeometry* fine_grid,
 
   return max_angle;
 }
+
+void CMultiGridGeometry::ValidateHaloCoordinates(const CConfig* config, unsigned short iMesh) const {
+#ifdef HAVE_MPI
+
+  int size = SU2_MPI::GetSize();
+  int rank = SU2_MPI::GetRank();
+
+  if (size == SINGLE_NODE || !config->GetMG_DebugHaloCoordinates()) {
+    return;  // Skip if single-node or debug option disabled
+  }
+
+  if (rank == MASTER_NODE) {
+    cout << "\n--- MG Halo Coordinate Validation (Level " << iMesh << ") ---" << endl;
+  }
+
+  /*--- For each SEND_RECEIVE marker pair, exchange coordinates and validate ---*/
+  for (auto iMarker = 0u; iMarker < config->GetnMarker_All(); iMarker++) {
+    if ((config->GetMarker_All_KindBC(iMarker) == SEND_RECEIVE) && (config->GetMarker_All_SendRecv(iMarker) > 0)) {
+      const auto MarkerS = iMarker;
+      const auto MarkerR = iMarker + 1;
+
+      const auto send_to = config->GetMarker_All_SendRecv(MarkerS) - 1;
+      const auto receive_from = abs(config->GetMarker_All_SendRecv(MarkerR)) - 1;
+
+      const auto nVertexS = nVertex[MarkerS];
+      const auto nVertexR = nVertex[MarkerR];
+
+      /*--- Allocate buffers for coordinate exchange ---*/
+      vector<su2double> Buffer_Send_Coord(nVertexS * nDim);
+      vector<su2double> Buffer_Receive_Coord(nVertexR * nDim);
+
+      /*--- Pack SEND coordinates (domain CVs being sent) ---*/
+      for (auto iVertex = 0ul; iVertex < nVertexS; iVertex++) {
+        const auto iPoint = vertex[MarkerS][iVertex]->GetNode();
+        const auto* Coord = nodes->GetCoord(iPoint);
+        for (auto iDim = 0u; iDim < nDim; iDim++) {
+          Buffer_Send_Coord[iVertex * nDim + iDim] = Coord[iDim];
+        }
+      }
+
+      /*--- Exchange coordinates ---*/
+      SU2_MPI::Sendrecv(Buffer_Send_Coord.data(), nVertexS * nDim, MPI_DOUBLE, send_to, 0,
+                        Buffer_Receive_Coord.data(), nVertexR * nDim, MPI_DOUBLE, receive_from, 0,
+                        SU2_MPI::GetComm(), MPI_STATUS_IGNORE);
+
+      /*--- Validate RECEIVE coordinates against local halo CVs ---*/
+      unsigned long nMismatch = 0;
+      su2double maxError = 0.0;
+      su2double tolerance = 1e-10;
+
+      for (auto iVertex = 0ul; iVertex < nVertexR; iVertex++) {
+        const auto iPoint = vertex[MarkerR][iVertex]->GetNode();
+        const auto* Coord_Local = nodes->GetCoord(iPoint);
+
+        su2double error = 0.0;
+        for (auto iDim = 0u; iDim < nDim; iDim++) {
+          su2double coord_remote = Buffer_Receive_Coord[iVertex * nDim + iDim];
+          su2double diff = fabs(Coord_Local[iDim] - coord_remote);
+          error += diff * diff;
+        }
+        error = sqrt(error);
+
+        if (error > tolerance) {
+          nMismatch++;
+          maxError = max(maxError, error);
+
+          if (nMismatch <= 5) {  // Only print first 5 mismatches
+            cout << "COORD MISMATCH [Rank " << rank << ", Marker " << MarkerS << "/" << MarkerR
+                 << ", Vertex " << iVertex << ", Point " << iPoint << "]: ";
+            cout << "Local=(";
+            for (auto iDim = 0u; iDim < nDim; iDim++) {
+              cout << Coord_Local[iDim];
+              if (iDim < nDim - 1) cout << ", ";
+            }
+            cout << "), Remote=(";
+            for (auto iDim = 0u; iDim < nDim; iDim++) {
+              cout << Buffer_Receive_Coord[iVertex * nDim + iDim];
+              if (iDim < nDim - 1) cout << ", ";
+            }
+            cout << "), Error=" << error << endl;
+          }
+        }
+      }
+
+      if (nMismatch > 0) {
+        cout << "WARNING [Rank " << rank << ", Marker " << MarkerS << "/" << MarkerR
+             << "]: " << nMismatch << " coordinate mismatches detected (max error: "
+             << maxError << ")" << endl;
+      } else if (nVertexR > 0) {
+        cout << "INFO [Rank " << rank << ", Marker " << MarkerS << "/" << MarkerR
+             << "]: All " << nVertexR << " halo CV coordinates match (tol=" << tolerance << ")" << endl;
+      }
+    }
+  }
+
+  if (rank == MASTER_NODE) {
+    cout << "--- End MG Halo Coordinate Validation ---\n" << endl;
+  }
+
+#endif  // HAVE_MPI
+}
diff --git a/SU2_CFD/include/solvers/CSolver.hpp b/SU2_CFD/include/solvers/CSolver.hpp
@@ -1480,7 +1480,9 @@ class CSolver {
   void PerformCFLReductions(CGeometry *geometry, CConfig *config, unsigned short iMesh);
 
   void ApplyCFLToCoarseGrid(CGeometry *geometry, CSolver **solver_container,
-                           CConfig *config, unsigned short iMesh);
+                           CConfig *config, unsigned short iMesh,
+                           bool reduceCFL, bool resetCFL, bool canIncrease,
+                           su2double startingIter);
 
 public:
 
diff --git a/SU2_CFD/src/drivers/CDriver.cpp b/SU2_CFD/src/drivers/CDriver.cpp
@@ -881,6 +881,11 @@ void CDriver::InitializeGeometryFVM(CConfig *config, CGeometry **&geometry) {
     geometry[iMGlevel]->SetEdges();
     geometry[iMGlevel]->SetVertex(geometry[iMGlevel-1], config);
 
+    /*--- Validate halo CV coordinates if debug option enabled ---*/
+    if (auto* mg_geometry = dynamic_cast<CMultiGridGeometry*>(geometry[iMGlevel])) {
+      mg_geometry->ValidateHaloCoordinates(config, iMGlevel);
+    }
+
     /*--- Create the control volume structures ---*/
 
     geometry[iMGlevel]->SetControlVolume(geometry[iMGlevel-1], ALLOCATE);
diff --git a/SU2_CFD/src/solvers/CSolver.cpp b/SU2_CFD/src/solvers/CSolver.cpp
