Add nested FGMRES inner BiCGSTAB options and logging

Author: eorbay-metu

Common/include/CConfig.hpp

@@ -53,6 +53,12 @@
 
 using namespace std;
 
+/*Inner solver for nested linear solver:*/
+enum ENUM_LINEAR_SOLVER_INNER {
+  INNER_SOLVER_NONE = 0,
+  INNER_SOLVER_BCGSTAB
+};
+
 /*!
  * \class CConfig
  * \brief Main class for defining the problem; basically this class reads the configuration file, and
@@ -430,6 +436,7 @@ class CConfig {
   unsigned short nQuasiNewtonSamples;  /*!< \brief Number of samples used in quasi-Newton solution methods. */
   bool UseVectorization;       /*!< \brief Whether to use vectorized numerics schemes. */
   bool NewtonKrylov;           /*!< \brief Use a coupled Newton method to solve the flow equations. */
+  bool Nested_Linear_Solver;   /*!< \brief Enable nested Krylov linear solver (e.g. FGMRES + inner solver). */
   array<unsigned short,3> NK_IntParam{{20, 3, 2}}; /*!< \brief Integer parameters for NK method. */
   array<su2double,5> NK_DblParam{{-2.0, 0.1, -3.0, 1e-4, 1.0}}; /*!< \brief Floating-point parameters for NK method. */
   su2double NK_Relaxation = 1.0;
@@ -525,6 +532,7 @@ class CConfig {
   Kind_Deform_Linear_Solver,             /*!< Numerical method to deform the grid */
   Kind_Deform_Linear_Solver_Prec,        /*!< \brief Preconditioner of the linear solver. */
   Kind_Linear_Solver,                    /*!< \brief Numerical solver for the implicit scheme. */
+  Kind_Linear_Solver_Inner,              /*!< \brief Inner solver used in nested Krylov schemes. */
   Kind_Linear_Solver_Prec,               /*!< \brief Preconditioner of the linear solver. */
   Kind_DiscAdj_Linear_Solver,            /*!< \brief Linear solver for the discrete adjoint system. */
   Kind_DiscAdj_Linear_Prec,              /*!< \brief Preconditioner of the discrete adjoint linear solver. */
@@ -4281,6 +4289,16 @@ class CConfig {
    */
   unsigned short GetKind_Linear_Solver(void) const { return Kind_Linear_Solver; }
 
+  /*!
+   * \brief Check whether nested Krylov linear solver is enabled.
+   */
+  bool GetNested_Linear_Solver(void) const { return Nested_Linear_Solver; }
+
+  /*!
+   * \brief Get the inner linear solver used in nested Krylov schemes.
+   */
+  unsigned short GetKind_Linear_Solver_Inner(void) const { return Kind_Linear_Solver_Inner; }
+
 
   /*!
    * \brief Get the kind of preconditioner for the implicit solver.

Common/include/linear_algebra/CSysSolve.hpp

@@ -348,21 +348,6 @@ class CSysSolve {
   unsigned long BCGSTAB_LinSolver(const VectorType& b, VectorType& x, const ProductType& mat_vec,
                                   const PrecondType& precond, ScalarType tol, unsigned long m, ScalarType& residual,
                                   bool monitoring, const CConfig* config) const;
-  /*!
-   * \brief The nested FGMRES and BCGSTAB solver
-   * \param[in] b - the right hand size vector
-   * \param[in,out] x - on entry the intial guess, on exit the solution
-   * \param[in] mat_vec - object that defines matrix-vector product
-   * \param[in] precond - object that defines preconditioner
-   * \param[in] tol - tolerance with which to solve the system
-   * \param[in] m - maximum size of the search subspace
-   * \param[out] residual - final normalized residual
-   * \param[in] monitoring - turn on priting residuals from solver to screen.
-   * \param[in] config - Definition of the particular problem.
-   */
-  unsigned long FGMRESandBCGSTAB2_LinSolver(const VectorType& b, VectorType& x, const ProductType& mat_vec,
-                                            const PrecondType& precond, ScalarType tol, unsigned long m,
-                                            ScalarType& residual, bool monitoring, const CConfig* config) const;
 
   /*!
    * \brief Generic smoother (modified Richardson iteration with preconditioner)

Common/include/option_structure.hpp

@@ -2354,7 +2354,6 @@ enum ENUM_LINEAR_SOLVER {
   CONJUGATE_GRADIENT,   /*!< \brief Preconditionated conjugate gradient method for grid deformation. */
   FGMRES,               /*!< \brief Flexible Generalized Minimal Residual method. */
   BCGSTAB,              /*!< \brief BCGSTAB - Biconjugate Gradient Stabilized Method (main solver). */
-  FGMRESandBCGSTAB2,    /*!< \brief FGMRESandBCGSTAB (main solver). */
   RESTARTED_FGMRES,     /*!< \brief Flexible Generalized Minimal Residual method with restart. */
   SMOOTHER,             /*!< \brief Iterative smoother. */
   PASTIX_LDLT,          /*!< \brief PaStiX LDLT (complete) factorization. */
@@ -2363,7 +2362,6 @@ enum ENUM_LINEAR_SOLVER {
 static const MapType<std::string, ENUM_LINEAR_SOLVER> Linear_Solver_Map = {
   MakePair("CONJUGATE_GRADIENT", CONJUGATE_GRADIENT)
   MakePair("BCGSTAB", BCGSTAB)
-  MakePair("FGMRESandBCGSTAB2", FGMRESandBCGSTAB2)
   MakePair("FGMRES", FGMRES)
   MakePair("RESTARTED_FGMRES", RESTARTED_FGMRES)
   MakePair("SMOOTHER", SMOOTHER)

Common/src/CConfig.cpp

@@ -62,6 +62,12 @@ vector<double> GEMM_Profile_MaxTime;      /*!< \brief Maximum time spent for thi
 //#pragma omp threadprivate(Profile_Function_tp, Profile_Time_tp, Profile_ID_tp, Profile_Map_tp)
 
 
+/* --- Map from config string to inner solver enum --- */
+std::map<std::string, ENUM_LINEAR_SOLVER_INNER> Inner_Linear_Solver_Map = {
+  {"NONE",    INNER_SOLVER_NONE},
+  {"BCGSTAB", INNER_SOLVER_BCGSTAB}
+};
+
 CConfig::CConfig(char case_filename[MAX_STRING_SIZE], SU2_COMPONENT val_software, bool verb_high) {
 
   /*--- Set the case name to the base config file name without extension ---*/
@@ -1906,6 +1912,20 @@ void CConfig::SetConfig_Options() {
   addEnumOption("DISCADJ_LIN_PREC", Kind_DiscAdj_Linear_Prec, Linear_Solver_Prec_Map, ILU);
   /* DESCRIPTION: Linear solver for the discete adjoint systems */
 
+  /*!\brief LINEAR_SOLVER_NESTED
+   *  \n DESCRIPTION: Enable nested Krylov linear solver (e.g. FGMRES with inner solver).
+   *  \n OPTIONS: YES, NO \ingroup Config */
+  addBoolOption("LINEAR_SOLVER_NESTED", Nested_Linear_Solver, false);
+
+  /*!\brief LINEAR_SOLVER_INNER
+   *  \n DESCRIPTION: Inner linear solver used when LINEAR_SOLVER_NESTED = YES.
+   *  \n OPTIONS: see \link Inner_Linear_Solver_Map \endlink
+   *  \n DEFAULT: BCGSTAB \ingroup Config */
+  addEnumOption("LINEAR_SOLVER_INNER",
+                Kind_Linear_Solver_Inner,
+                Inner_Linear_Solver_Map,
+                INNER_SOLVER_BCGSTAB);
+
   /* DESCRIPTION: Maximum update ratio value for flow density and energy variables */
   addDoubleOption("MAX_UPDATE_FLOW", MaxUpdateFlow, 0.2);
   /* DESCRIPTION: Maximum update ratio value for SA turbulence variable nu_tilde */
@@ -7255,15 +7275,19 @@ void CConfig::SetOutput(SU2_COMPONENT val_software, unsigned short val_izone) {
             SU2_MPI::Error("Implicit time scheme is not yet implemented with Mutation++. Use EULER_EXPLICIT.", CURRENT_FUNCTION);
           switch (Kind_Linear_Solver) {
             case BCGSTAB:
-            case FGMRESandBCGSTAB2:
             case FGMRES:
             case RESTARTED_FGMRES:
-              if (Kind_Linear_Solver == BCGSTAB)
+              if (Kind_Linear_Solver == BCGSTAB){
                 cout << "BCGSTAB is used for solving the linear system." << endl;
-              else if (Kind_Linear_Solver == FGMRESandBCGSTAB2)
-                cout << "FGMRES and BCGSTAB is used for solving the linear system." << endl;
-              else
-                cout << "FGMRES is used for solving the linear system." << endl;
+              }  
+              else {
+                    if (GetNested_Linear_Solver()){
+                        cout << "Nested FGMRES (FGMRES with inner BiCGSTAB) is used for solving the linear system." << endl;
+                    }    
+                    else { 
+                       cout << "FGMRES is used for solving the linear system." << endl;
+                    }   
+               }     
               switch (Kind_Linear_Solver_Prec) {
                 case ILU: cout << "Using a ILU("<< Linear_Solver_ILU_n <<") preconditioning."<< endl; break;
                 case LINELET: cout << "Using a linelet preconditioning."<< endl; break;
@@ -7316,12 +7340,6 @@ void CConfig::SetOutput(SU2_COMPONENT val_software, unsigned short val_izone) {
               cout << "Convergence criteria of the linear solver: "<< Linear_Solver_Error <<"."<< endl;
               cout << "Max number of iterations: "<< Linear_Solver_Iter <<"."<< endl;
               break;
-            case FGMRESandBCGSTAB2:
-              cout << "FGMRES and BCGSTAB is used for solving the linear system." << endl;
-              cout << "Convergence criteria of the linear solver: "<< Linear_Solver_Error <<"."<< endl;
-              cout << "Max number of iterations: "<< Linear_Solver_Iter <<"."<< endl;
-              break;
-
           }
           break;
       }