Feature: Make BPCG support band parallelism

source/Makefile.Objects

@@ -333,6 +333,7 @@ OBJS_HSOLVER=diago_cg.o\
     diago_david.o\
     diago_dav_subspace.o\
     diago_bpcg.o\
+    para_linear_transform.o\
     hsolver.o\
     hsolver_pw.o\
     hsolver_lcaopw.o\

source/driver.cpp

@@ -152,9 +152,9 @@ void Driver::reading()
                                 GlobalV::MY_RANK,
                                 PARAM.inp.bndpar,
                                 GlobalV::KPAR,
-                                GlobalV::NPROC_IN_STOGROUP,
-                                GlobalV::RANK_IN_STOGROUP,
-                                GlobalV::MY_STOGROUP,
+                                GlobalV::NPROC_IN_BNDGROUP,
+                                GlobalV::RANK_IN_BPGROUP,
+                                GlobalV::MY_BNDGROUP,
                                 GlobalV::NPROC_IN_POOL,
                                 GlobalV::RANK_IN_POOL,
                                 GlobalV::MY_POOL);

source/module_base/global_file.cpp

@@ -153,36 +153,32 @@ void ModuleBase::Global_File::make_dir_out(
 #endif
     }
 
-    std::stringstream ss,ss1;
-
     // mohan add 2010-09-12
     if(out_alllog)
     {
-	    ss << "running_" << calculation << "_" << rank + 1;
-	    open_log(GlobalV::ofs_running, ss.str(), calculation, restart);
+	    open_log(GlobalV::ofs_running, PARAM.globalv.log_file, calculation, restart);
         #if defined(__CUDA) || defined(__ROCM)
-        open_log(GlobalV::ofs_device, "device" + std::to_string(rank), calculation, restart);
+        open_log(GlobalV::ofs_device, "device" + std::to_string(rank) + ".log", calculation, restart);
         #endif
     }
     else
     {
 	    if(rank==0)
 	    {
-		    ss << "running_" << calculation;
-		    open_log(GlobalV::ofs_running, ss.str(), calculation, restart);
+		    open_log(GlobalV::ofs_running, PARAM.globalv.log_file, calculation, restart);
             #if defined(__CUDA) || defined(__ROCM)
-            open_log(GlobalV::ofs_device, "device", calculation, restart);
+            open_log(GlobalV::ofs_device, "device.log", calculation, restart);
             #endif
 	    }
     }
 
     if(rank==0)
     {
-        open_log(GlobalV::ofs_warning, "warning", calculation, restart);
+        open_log(GlobalV::ofs_warning, "warning.log", calculation, restart);
     }
 
 #ifdef GATHER_INFO
-    open_log(GlobalV::ofs_info, "math_info_" + std::to_string(rank), calculation, restart);
+    open_log(GlobalV::ofs_info, "math_info_" + std::to_string(rank) + ".log", calculation, restart);
 #endif
 
     return;
@@ -206,7 +202,7 @@ void ModuleBase::Global_File::open_log(std::ofstream &ofs, const std::string &fn
 // PARAM.globalv.global_out_dir : (default dir to store "*.log" file)
 //----------------------------------------------------------
     std::stringstream ss;
-    ss << PARAM.globalv.global_out_dir << fn << ".log";
+    ss << PARAM.globalv.global_out_dir << fn;
 
     if(calculation == "md" && restart)
     {

source/module_base/global_variable.cpp

@@ -18,14 +18,13 @@ namespace GlobalV
 
 int NPROC = 1; ///< global number of process
 int KPAR = 1;  ///< global number of pools
-int KPAR_LCAO = 1; ///< global number of pools for LCAO diagonalization only
 int MY_RANK = 0; ///< global index of process
 int MY_POOL = 0; ///< global index of pool (count in pool)
-int MY_STOGROUP = 0;
+int MY_BNDGROUP = 0;
 int NPROC_IN_POOL = 1; ///< local number of process in a pool
-int NPROC_IN_STOGROUP = 1;
+int NPROC_IN_BNDGROUP = 1;
 int RANK_IN_POOL = 0; ///< global index of pool (count in process), my_rank in each pool
-int RANK_IN_STOGROUP = 0;
+int RANK_IN_BPGROUP = 0;
 int DRANK = -1; ///< mohan add 2012-01-13, must be -1, so we can recognize who
                 ///< didn't in DIAG_WORLD
 int DSIZE = KPAR;

source/module_base/global_variable.h

@@ -28,23 +28,21 @@ namespace GlobalV
 // NAME : DCOLOR( color of each group)
 // NAME : GRANK( index of grid world)
 // NAME : GSIZE( number of processors in each grid world)
-// NAME : KPAR_LCAO ( global number of pools for LCAO diagonalization only)
 //========================================================================
 extern int NPROC;
 extern int KPAR;
 extern int MY_RANK;
 extern int MY_POOL;
-extern int MY_STOGROUP;
+extern int MY_BNDGROUP;
 extern int NPROC_IN_POOL;
-extern int NPROC_IN_STOGROUP;
+extern int NPROC_IN_BNDGROUP;
 extern int RANK_IN_POOL;
-extern int RANK_IN_STOGROUP;
+extern int RANK_IN_BPGROUP;
 extern int DRANK;
 extern int DSIZE;
 extern int DCOLOR;
 extern int GRANK;
 extern int GSIZE;
-extern int KPAR_LCAO;
 
 //==========================================================
 // NAME : ofs_running( contain information during runnnig)

source/module_base/kernels/math_kernel_op.cpp

@@ -382,11 +382,13 @@ template struct line_minimize_with_block_op<std::complex<float>, base_device::DE
 
 template struct scal_op<double, base_device::DEVICE_CPU>;
 template struct axpy_op<std::complex<double>, base_device::DEVICE_CPU>;
+template struct axpy_op<double, base_device::DEVICE_CPU>;
 template struct gemv_op<std::complex<double>, base_device::DEVICE_CPU>;
 template struct gemv_op<double, base_device::DEVICE_CPU>;
 template struct gemm_op<std::complex<double>, base_device::DEVICE_CPU>;
 template struct gemm_op<double, base_device::DEVICE_CPU>;
 template struct dot_real_op<std::complex<double>, base_device::DEVICE_CPU>;
+template struct dot_real_op<double, base_device::DEVICE_CPU>;
 template struct vector_div_constant_op<std::complex<double>, base_device::DEVICE_CPU>;
 template struct vector_mul_vector_op<std::complex<double>, base_device::DEVICE_CPU>;
 template struct vector_div_vector_op<std::complex<double>, base_device::DEVICE_CPU>;
@@ -397,8 +399,6 @@ template struct calc_grad_with_block_op<std::complex<double>, base_device::DEVIC
 template struct line_minimize_with_block_op<std::complex<double>, base_device::DEVICE_CPU>;
 
 #ifdef __LCAO
-template struct axpy_op<double, base_device::DEVICE_CPU>;
-template struct dot_real_op<double, base_device::DEVICE_CPU>;
 template struct vector_mul_vector_op<double, base_device::DEVICE_CPU>;
 template struct vector_div_constant_op<double, base_device::DEVICE_CPU>;
 template struct vector_div_vector_op<double, base_device::DEVICE_CPU>;