Merge branch 'cusolver-kpar' of https://github.com/dzzz2001/abacus-develop into cusolver-kpar

Author: dzzz2001

CMakeLists.txt

@@ -10,39 +10,45 @@ project(
   HOMEPAGE_URL "https://github.com/deepmodeling/abacus-develop"
   LANGUAGES CXX)
 
-option(ENABLE_LCAO "Enable LCAO calculation." ON)
-option(ENABLE_DEEPKS "Enable DeePKS functionality" OFF)
-option(ENABLE_MLKEDF "Enable Machine Learning based KEDF for OFDFT" OFF)
-option(ENABLE_LIBXC "Enable LibXC functionality" OFF)
-option(USE_CUDA "Enable support to CUDA for ABACUS." OFF)
-option(ENABLE_FLOAT_FFTW "Enable support to single precision FFTW library." OFF)
-option(USE_ROCM "Enable support to ROCm." OFF)
-option(USE_OPENMP "Enable OpenMP in ABACUS." ON)
+option(ENABLE_MPI "Enable MPI" ON)
+option(USE_OPENMP "Enable OpenMP" ON)
+option(USE_CUDA "Enable CUDA" OFF)
+option(USE_CUDA_MPI "Enable CUDA-aware MPI" OFF)
+option(USE_CUDA_ON_DCU "Enable CUDA on DCU" OFF)
+option(USE_ROCM "Enable ROCm" OFF)
+option(USE_DSP "Enable DSP" OFF)
+
+option(USE_ABACUS_LIBM "Build libmath from source to speed up" OFF)
+option(ENABLE_LIBXC "Enable using the LibXC package" OFF)
+option(ENABLE_FLOAT_FFTW "Enable using single-precision FFTW library." OFF)
+option(ENABLE_DEEPKS "Enable the DeePKS algorithm" OFF)
+option(ENABLE_MLKEDF "Enable the Machine-Learning-based KEDF for OFDFT" OFF)
+
+option(ENABLE_LCAO "Enable LCAO algorithm" ON)
+option(USE_ELPA "Enable ELPA for LCAO" ON)
+option(ENABLE_LIBRI "Enable LibRI for hybrid functional" OFF)
+option(ENABLE_LIBCOMM "Enable LibComm" OFF)
+option(ENABLE_PEXSI "Enable PEXSI for LCAO" OFF)
+
+option(BUILD_TESTING "Build unittests" OFF)
+option(DEBUG_INFO "Print message to debug" OFF)
 option(ENABLE_ASAN "Enable AddressSanitizer" OFF)
-option(BUILD_TESTING "Build ABACUS unit tests" OFF)
-option(INFO "Enable gathering of math library information" OFF)
-option(ENABLE_COVERAGE "Enable coverage build." OFF)
-option(ENABLE_LIBRI "Enable EXX with LibRI." OFF)
-option(ENABLE_LIBCOMM "Enable communicate with LibComm." OFF)
-option(ENABLE_PAW "Enable PAW calculation" OFF)
-option(ENABLE_MPI "Enable compilation with or without MPI." ON)
-option(USE_ELPA "Enable ELPA" ON)
-option(USE_ABACUS_LIBM "Build libmath from source to speed up." OFF)
+option(INFO "Enable gathering math library information" OFF)
+option(ENABLE_COVERAGE "Enable coverage build" OFF)
 option(GIT_SUBMODULE "Check submodules during build" ON)
-option(DEBUG_INFO "Print message for developers to debug." OFF)
+
+option(ENABLE_PAW "Enable PAW method" OFF)
+
 # Do not enable it if generated code will run on different CPUs
 option(ENABLE_NATIVE_OPTIMIZATION
        "Enable compilation optimization for the native machine's CPU type" OFF)
+
 option(COMMIT_INFO "Print commit information in log" ON)
-option(ENABLE_FFT_TWO_CENTER "Enable FFT-based two-center integral method." ON)
-option(ENABLE_GOOGLEBENCH "Enable GOOGLE-benchmark usage." OFF)
-option(ENABLE_RAPIDJSON "Enable rapid-json usage." OFF)
-option(ENABLE_CNPY "Enable cnpy usage." OFF)
-option(ENABLE_PEXSI "Enable support for PEXSI." OFF)
-option(ENABLE_CUSOLVERMP "Enable cusolvermp." OFF)
-option(USE_DSP "Enable DSP usage." OFF)
-option(USE_CUDA_ON_DCU "Enable CUDA on DCU" OFF)
-option(USE_CUDA_MPI "Enable CUDA-aware MPI" OFF)
+option(ENABLE_FFT_TWO_CENTER "Enable FFT-based two-center integral method" ON)
+option(ENABLE_GOOGLEBENCH "Enable GOOGLE-benchmark usage" OFF)
+option(ENABLE_RAPIDJSON "Enable rapid-json usage" OFF)
+option(ENABLE_CNPY "Enable cnpy usage" OFF)
+option(ENABLE_CUSOLVERMP "Enable cusolvermp" OFF)
 
 # enable json support
 if(ENABLE_RAPIDJSON)

docs/advanced/acceleration/cuda.md

@@ -46,6 +46,7 @@ We provides [examples](https://github.com/deepmodeling/abacus-develop/tree/devel
 PW basis:
 - Only k point parallelization is supported, so the input keyword `kpar` will be set to match the number of MPI tasks automatically.
 - By default, CUDA architectures 60, 70, 75, 80, 86, and 89 are compiled (if supported). It can be overriden using the CMake variable [`CMAKE_CUDA_ARCHITECTURES`](https://cmake.org/cmake/help/latest/variable/CMAKE_CUDA_ARCHITECTURES.html) or the environmental variable [`CUDAARCHS`](https://cmake.org/cmake/help/latest/envvar/CUDAARCHS.html).
+
 LCAO basis:
 - Unless there is a specific reason, avoid using multiple GPUs, as it can be slower than using a single GPU. This is because the generalized eigenvalue solution of the LCAO basis set will incur additional communication overhead when calculated on multiple cards. When the memory limit of a GPU card makes it insufficient to complete the task, it is recommended to use multiple cards for calculation.
 - When using elpa on GPUs, some ELPA internal logs will be output.

docs/advanced/input_files/input-main.md

@@ -140,6 +140,7 @@
     - [out\_wfc\_r](#out_wfc_r)
     - [out\_wfc\_lcao](#out_wfc_lcao)
     - [out\_dos](#out_dos)
+    - [out\_ldos](#out_ldos)
     - [out\_band](#out_band)
     - [out\_proj\_band](#out_proj_band)
     - [out\_stru](#out_stru)
@@ -154,6 +155,7 @@
     - [out\_mat\_dh](#out_mat_dh)
     - [out\_mat\_xc](#out_mat_xc)
     - [out\_mat\_xc2](#out_mat_xc2)
+    - [out\_mat\_l](#out_mat_l)
     - [out\_eband\_terms](#out_eband_terms)
     - [out\_hr\_npz/out\_dm\_npz](#out_hr_npzout_dm_npz)
     - [dm\_to\_rho](#dm_to_rho)
@@ -174,6 +176,7 @@
     - [dos\_emin\_ev](#dos_emin_ev)
     - [dos\_emax\_ev](#dos_emax_ev)
     - [dos\_nche](#dos_nche)
+    - [stm\_bias](#stm_bias)
   - [NAOs](#naos)
     - [bessel\_nao\_ecut](#bessel_nao_ecut)
     - [bessel\_nao\_tolerence](#bessel_nao_tolerence)
@@ -1702,10 +1705,15 @@ These variables are used to control the output of properties.
 - **Description**: Whether to output the density of states (DOS). For more information, refer to the [dos.md](../elec_properties/dos.md).
   - 0: no output
   - 1: output the density of states (DOS)
-  - 2: 
-    - lcao-only: output the density of states (DOS) and the projected density of states (PDOS)
+  - 2: (lcao-only) output the density of states (DOS) and the projected density of states (PDOS)
 - **Default**: 0
 
+### out_ldos
+
+- **Type**: Boolean
+- **Description**: Whether to output the local density of states for given bias in cube file format, which is controlled by [stm_bias](#stm_bias). 
+- **Default**: False
+
 ### out_band
 
 - **Type**: Boolean \[Integer\](optional)
@@ -1807,6 +1815,13 @@ The band (KS orbital) energy for each (k-point, spin, band) will be printed in t
 - **Description**: Whether to print the exchange-correlation matrices in **numerical orbital representation** (unit: Ry): $\braket{\phi_i|V_\text{xc}^\text{(semi-)local}+V_\text{exx}+V_\text{DFTU}|\phi_j}(\mathbf{R})$ in CSR format (the same format as [out_mat_hs2](../elec_properties/hs_matrix.md#out_mat_hs2)) in the directory `OUT.${suffix}`. (Note that currently DeePKS term is not included. ) The files are named `Vxc_R_spin$s`.
 - **Default**: False
 
+### out_mat_l
+
+- **Type**: Boolean [Integer\](optional)
+- **Availability**: Numerical atomic orbital (NAO) basis
+- **Description**: Whether to print the expectation value of the angular momentum operator $\hat{L}_x$, $\hat{L}_y$, and $\hat{L}_z$ in the basis of the localized atomic orbitals. The files are named `OUT.${suffix}/${suffix}_Lx.dat`, `OUT.${suffix}/${suffix}_Ly.dat`, and `OUT.${suffix}/${suffix}_Lz.dat`. The second integer controls the precision of the output.
+- **Default**: False 8
+
 ### out_eband_terms
 
 - **Type**: Boolean
@@ -1955,9 +1970,20 @@ These variables are used to control the calculation of DOS. [Detailed introducti
 ### dos_nche
 
 - **Type**: Integer
-The order of Chebyshev expansions when using Stochastic Density Functional Theory (SDFT) to calculate DOS.
+- **Description**: The order of Chebyshev expansions when using Stochastic Density Functional Theory (SDFT) to calculate DOS.
 - **Default**: 100
 
+### stm_bias
+
+- **Type**: Real Real(optional) Integer(optional)
+- **Description**: The bias voltage used to calculate local density of states to simulate scanning tunneling microscope, see details in [out_ldos](#out_ldos). When using three parameters:
+
+  - The first parameter specifies the initial bias voltage value. 
+  - The second parameter defines the voltage increment (step size between consecutive bias values). 
+  - The third parameter determines the total number of voltage points
+- **Default**: 1.0
+- **Unit**: V
+
 [back to top](#full-list-of-input-keywords)
 
 ## NAOs
@@ -2004,10 +2030,13 @@ Warning: this function is not robust enough for the current version. Please try
 
 ### deepks_out_labels
 
-- **Type**: Boolean
+- **Type**: Integer
 - **Availability**: numerical atomic orbital basis
-- **Description**: Print labels and descriptors for DeePKS training in OUT.${suffix}. The names of these files start with "deepks".
-- **Note**: In `LCAO` calculation, the path of a numerical descriptor (an `orb` file) is needed to be specified under the `NUMERICAL_DESCRIPTOR` tag in the `STRU` file. For example:
+- **Description**: Print labels and descriptors for DeePKS in OUT.${suffix}. The names of these files start with "deepks".
+  - 0 : No output.
+  - 1 : Output intermediate files needed during DeePKS training.
+  - 2 : Output target labels for label preperation. The label files are named as `deepks_<property>.npy`, where the units and formats are the same as label files `<property>.npy` required for training, except that the first dimension (`nframes`) is excluded. System structrue files are also given in `deepks_atom.npy` and `deepks_box.npy` in the unit of *Bohr*, which means `lattice_constant` should be set to 1 when training. 
+- **Note**: When `deepks_out_labels` equals **1**, the path of a numerical descriptor (an `orb` file) is needed to be specified under the `NUMERICAL_DESCRIPTOR` tag in the `STRU` file. For example:
 
   ```text
   NUMERICAL_ORBITAL
@@ -2017,8 +2046,8 @@ Warning: this function is not robust enough for the current version. Please try
   NUMERICAL_DESCRIPTOR
   jle.orb
   ```
-
-- **Default**: False
+  This is not needed when `deepks_out_labels` equals 2. 
+- **Default**: 0
 
 ### deepks_scf
 
@@ -3858,12 +3887,6 @@ These variables are used to control berry phase and wannier90 interface paramete
   - 1: Yes.
 - **Default**: 0
 
-- **Type**: Boolean
-- **Description**: Specify whether to set the colorful output in terminal.
-  - 0: No.
-  - 1: Yes.
-- **Default**: 0
-
 ### test_skip_ewald
 
 - **Type**: Boolean

source/Makefile.Objects

@@ -497,6 +497,9 @@ OBJS_IO=input_conv.o\
     write_dos_pw.o\
     nscf_band.o\
     cal_dos.o\
+    cal_pdos_gamma.o\
+    cal_pdos_multik.o\
+    cal_ldos.o\
     cif_io.o\
     dos_nao.o\
     numerical_descriptor.o\
@@ -554,6 +557,7 @@ OBJS_IO=input_conv.o\
     read_input_item_output.o\
     read_set_globalv.o\
     orb_io.o\
+    cal_pLpR.o\
 
 OBJS_IO_LCAO=cal_r_overlap_R.o\
       write_orb_info.o\

source/module_base/parallel_global.cpp

@@ -405,14 +405,16 @@ void Parallel_Global::divide_mpi_groups(const int& procs,
         exit(1);
     }
 
-    if(rank < extra_procs)
+    if(rank < extra_procs * (procs_in_group + 1))
     {
+        // The first extra_procs groups have procs_in_group + 1 processes.
         procs_in_group++;
         my_group = rank / procs_in_group;
         rank_in_group = rank % procs_in_group;
     }
     else
     {
+        // The remaining groups have procs_in_group processes.
         my_group = (rank - extra_procs) / procs_in_group;
         rank_in_group = (rank - extra_procs) % procs_in_group;
     }

source/module_base/test/timer_test.cpp

@@ -59,19 +59,22 @@ TEST_F(TimerTest, Tick)
 	EXPECT_GT(ModuleBase::timer::timer_pool["wavefunc"]["evc"].cpu_second,0.0001);
 }
 
+
 TEST_F(TimerTest, Start)
 {
 	ModuleBase::timer::start();
 	// start() called tick() once
 	EXPECT_FALSE(ModuleBase::timer::timer_pool[""]["total"].start_flag);
 }
 
+
 TEST_F(TimerTest, write_to_json)
 {
 	ModuleBase::timer::tick("wavefunc","evc");
 	std::this_thread::sleep_for(std::chrono::microseconds(T_Elapse)); // 0.1 ms
 	ModuleBase::timer::tick("wavefunc","evc");
 	ModuleBase::timer::write_to_json("tmp.json");
+
 	// check if tmp.json exists
 	ifs.open("tmp.json");
 	EXPECT_TRUE(ifs.good());
@@ -89,9 +92,6 @@ TEST_F(TimerTest, write_to_json)
 		content += tmp;
 	}
 
-	// check if the content is correct
-	// shuold be like this:
-	// {"total": 0, "sub":[{"class_name": "wavefunc","sub":[{"name":"evc","cpu_second": 0.000318,"calls":2,"cpu_second_per_call":0.000159,"cpu_second_per_total": null}]}]}
 	EXPECT_THAT(content,testing::HasSubstr("\"total\":"));
 	EXPECT_THAT(content,testing::HasSubstr("\"sub\":[{\"class_name\":\"wavefunc\",\"sub\":[{\"name\":\"evc\",\"cpu_second\":"));
 	EXPECT_THAT(content,testing::HasSubstr("\"calls\":2,\"cpu_second_per_call\":"));
@@ -106,25 +106,29 @@ TEST_F(TimerTest, PrintAll)
 	ModuleBase::timer::tick("wavefunc","evc");
 	std::this_thread::sleep_for(std::chrono::microseconds(T_Elapse)); // 0.1 ms
 	ModuleBase::timer::tick("wavefunc","evc");
+
 	// call print_all
 	ofs.open("tmp");
 	testing::internal::CaptureStdout();
 	ModuleBase::timer::print_all(ofs);
 	output = testing::internal::GetCapturedStdout();
 	ofs.close();
+
 	// checout output on screen
-	std::cout << "Get captured stdout: \n" << std::endl;
-	std::cout << output << std::endl;
+	// std::cout << "Get captured stdout: \n" << std::endl;
+	// std::cout << output << std::endl;
 	EXPECT_THAT(output,testing::HasSubstr("TIME STATISTICS"));
 	EXPECT_THAT(output,testing::HasSubstr("CLASS_NAME"));
 	EXPECT_THAT(output,testing::HasSubstr("NAME"));
 	EXPECT_THAT(output,testing::HasSubstr("TIME/s"));
 	EXPECT_THAT(output,testing::HasSubstr("CALLS"));
 	EXPECT_THAT(output,testing::HasSubstr("AVG/s"));
 	EXPECT_THAT(output,testing::HasSubstr("PER/%"));
+
 	// check output in file
 	ifs.open("tmp");
-	std::cout << "Capture contents line by line from output file: \n" << std::endl;
+	// std::cout << "Capture contents line by line from output file: \n" << std::endl;
+	getline(ifs,output);
 	getline(ifs,output);
 	EXPECT_THAT(output,testing::HasSubstr("TIME STATISTICS"));
 	getline(ifs,output);
@@ -139,36 +143,41 @@ TEST_F(TimerTest, PrintAll)
 	remove("time.json");
 }
 
+
 TEST_F(TimerTest, PrintUntilNow)
 {
 	long double time = ModuleBase::timer::print_until_now();
 	EXPECT_TRUE(time>0.0);
 }
 
+
 TEST_F(TimerTest, Finish)
 {
 	ModuleBase::timer::tick("wavefunc","evc");
 	std::this_thread::sleep_for(std::chrono::microseconds(T_Elapse)); // 0.1 ms
 	ModuleBase::timer::tick("wavefunc","evc");
+
 	// call print_all
 	ofs.open("tmp");
 	testing::internal::CaptureStdout();
 	ModuleBase::timer::finish(ofs);
 	output = testing::internal::GetCapturedStdout();
 	ofs.close();
 	// checout output on screen
-	std::cout << "Get captured stdout: \n" << std::endl;
-	std::cout << output << std::endl;
+	//std::cout << "Get captured stdout: \n" << std::endl;
+	//std::cout << output << std::endl;
 	EXPECT_THAT(output,testing::HasSubstr("TIME STATISTICS"));
 	EXPECT_THAT(output,testing::HasSubstr("CLASS_NAME"));
 	EXPECT_THAT(output,testing::HasSubstr("NAME"));
 	EXPECT_THAT(output,testing::HasSubstr("TIME/s"));
 	EXPECT_THAT(output,testing::HasSubstr("CALLS"));
 	EXPECT_THAT(output,testing::HasSubstr("AVG/s"));
 	EXPECT_THAT(output,testing::HasSubstr("PER/%"));
+
 	// check output in file
 	ifs.open("tmp");
-	std::cout << "Capture contents line by line from output file: \n" << std::endl;
+	//std::cout << "Capture contents line by line from output file: \n" << std::endl;
+	getline(ifs,output);
 	getline(ifs,output);
 	EXPECT_THAT(output,testing::HasSubstr("TIME STATISTICS"));
 	getline(ifs,output);
@@ -182,7 +191,6 @@ TEST_F(TimerTest, Finish)
 	ifs.close();
 }
 
-// use __MPI to activate parallel environment
 #ifdef __MPI
 int main(int argc, char **argv)
 {

source/module_base/test/tool_quit_test.cpp

@@ -93,7 +93,7 @@ TEST_F(ToolQuitTest,warningquit)
 	ifs.open("running.log");
 	getline(ifs,output);
 	// test output in running.log file
-	EXPECT_THAT(output,testing::HasSubstr("!!!!!!!"));
+	EXPECT_THAT(output,testing::HasSubstr("-------"));
 	ifs.close();
 }
 
@@ -116,7 +116,7 @@ TEST_F(ToolQuitTest,warningquit_with_ret)
 	ifs.open("running.log");
 	getline(ifs,output);
 	// test output in running.log file
-	EXPECT_THAT(output,testing::HasSubstr("!!!!!!!"));
+	EXPECT_THAT(output,testing::HasSubstr("-------"));
 	ifs.close();
 }
 // use __MPI to activate parallel environment

source/module_base/test_parallel/blacs_connector_test.sh

@@ -10,6 +10,11 @@ do
     fi
     echo "TEST in parallel, nprocs=$i"
     mpirun -np $i ./blacs_connector
+    if [[ $? -ne 0 ]]; then
+        echo -e "\e[1;33m [  FAILED  ] \e[0m"\
+			"execute UT with $i cores error."
+        exit 1
+    fi
     break    
 done
 

source/module_base/test_parallel/parallel_2d_test.sh

@@ -9,4 +9,9 @@ for i in 2 3 4; do
     fi
     echo "TEST in parallel, nprocs=$i"
     mpirun -np $i ./parallel_2d_test
+    if [[ $? -ne 0 ]]; then
+        echo -e "\e[1;33m [  FAILED  ] \e[0m"\
+			"execute UT with $i cores error."
+        exit 1
+    fi
 done