Merge branch 'develop' into dsp1

Author: A-006

CMakeLists.txt

@@ -17,6 +17,7 @@ option(USE_CUDA_MPI "Enable CUDA-aware MPI" OFF)
 option(USE_CUDA_ON_DCU "Enable CUDA on DCU" OFF)
 option(USE_ROCM "Enable ROCm" OFF)
 option(USE_DSP "Enable DSP" OFF)
+option(USE_SW "Enable SW Architecture" OFF)
 
 option(USE_ABACUS_LIBM "Build libmath from source to speed up" OFF)
 option(ENABLE_LIBXC "Enable using the LibXC package" OFF)
@@ -281,8 +282,23 @@ if (USE_DSP)
   target_link_libraries(${ABACUS_BIN_NAME} ${MT_HOST_DIR}/hthreads/lib/libhthread_device.a)
   target_link_libraries(${ABACUS_BIN_NAME} ${MT_HOST_DIR}/hthreads/lib/libhthread_host.a)
 endif()
+
 target_link_libraries(${ABACUS_BIN_NAME} ${SCALAPACK_LIBRARY_DIR})
 
+if (USE_SW)
+  add_compile_definitions(__SW)
+  set(SW ON)
+  include_directories(${SW_MATH}/include)
+  include_directories(${SW_FFT}/include)
+  
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_FFT}/lib/libfftw3.a)
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswfft.a)
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswscalapack.a)
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswlapack.a)
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswblas.a)
+endif()
+
+
 find_package(Threads REQUIRED)
 target_link_libraries(${ABACUS_BIN_NAME} Threads::Threads)
 
@@ -460,6 +476,9 @@ if(MKLROOT)
   if(CMAKE_CXX_COMPILER_ID MATCHES Intel)
     list(APPEND math_libs ifcore)
   endif()
+elseif(USE_SW)
+   list(APPEND math_libs gfortran)
+  # SW architecture can only use its own math library
 else()
   find_package(FFTW3 REQUIRED)
   find_package(Lapack REQUIRED)

docs/advanced/input_files/input-main.md

@@ -8,6 +8,7 @@
     - [symmetry](#symmetry)
     - [symmetry\_prec](#symmetry_prec)
     - [symmetry\_autoclose](#symmetry_autoclose)
+    - [cal_symm_repr](#cal_symm_repr)
     - [kpar](#kpar)
     - [bndpar](#bndpar)
     - [latname](#latname)
@@ -375,6 +376,8 @@
     - [out\_wannier\_unk](#out_wannier_unk)
     - [out\_wannier\_wvfn\_formatted](#out_wannier_wvfn_formatted)
   - [rt-TDDFT: Real-time time dependent density functional theory](#tddft-time-dependent-density-functional-theory)
+    - [estep\_per\_md](#estep_per_md)
+    - [td\_dt](#td_dt)
     - [td\_edm](#td_edm)
     - [td\_print\_eij](#td_print_eij)
     - [td\_propagator](#td_propagator)
@@ -510,6 +513,7 @@ These variables are used to control general system parameters.
   - get_wf: obtain real space wave functions (for LCAO basis only). See `out_wfc_norm` and `out_wfc_re_im` for more information
   - get_s: obtain the overlap matrix formed by localized orbitals (for LCAO basis with multiple k points). the file name is `SR.csr` with file format being the same as that generated by [out_mat_hs2](#out_mat_hs2). Note: in the 3.10-LTS version, the command was named `get_S`
   - gen_bessel: generates projectors, i.e., a series of Bessel functions, for the DeePKS method (for LCAO basis only); see also keywords `bessel_descriptor_lmax`, `bessel_descriptor_rcut` and `bessel_descriptor_tolerence`. A file named `jle.orb` will be generated which contains the projectors. An example is provided in examples/H2O-deepks-pw
+  - gen_opt_abfs: generate opt-ABFs as discussed in this [article](https://pubs.acs.org/doi/abs/10.1021/acs.jpclett.0c00481).
   - test_memory: obtain a rough estimation of memory consuption for the calculation
   - test_neighbour: obtain information of neighboring atoms (for LCAO basis only), please specify a positive [search_radius](#search_radius) manually
 - **Default**: scf
@@ -538,7 +542,7 @@ These variables are used to control general system parameters.
 - **Default**:
   - 0:
     - if [calculation](#calculation)==md/nscf/get_pchg/get_wf/get_s or [gamma_only](#gamma_only)==True;
-    - If ([dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0/opt_orb or [rpa](#rpa)==True). 
+    - If ([dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0 or [rpa](#rpa)==True). 
     - If [efield_flag](#efield_flag)==1
   - 1: else
 
@@ -559,6 +563,11 @@ These variables are used to control general system parameters.
   - True: automatically set symmetry to 0 and continue running without symmetry analysis
 - **Default**: True
 
+### cal_symm_repr
+- **Type**: Integer [Integer]\(optional\)
+- **Description**: Whether to print the matrix representation of symmetry operation to running log file. If the first value is given as 1, then all matrix representations will be printed. The second optional parameter controls the precision (number of digits) to print, default is 3, which is enough for a quick check.
+- **Default**: 1 3
+
 ### kpar
 
 - **Type**: Integer
@@ -761,7 +770,7 @@ These variables are used to control parameters related to input files.
 - **Availability**: Used only when numerical atomic orbitals are employed as basis set.
 - **Description**: If [restart_save](#restart_save) is set to true and an electronic iteration is finished, calculations can be restarted from the charge density file, which are saved in the former calculation. Please ensure [read_file_dir](#read_file_dir) is correct, and  the charge density file exist.
 
-  If EXX(exact exchange) is calculated (i.e. *[dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0/opt_orb* or *[rpa](#rpa)==True*), the Hexx(R) files in the same folder for each processor will also be read.
+  If EXX(exact exchange) is calculated (i.e. *[dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0* or *[rpa](#rpa)==True*), the Hexx(R) files in the same folder for each processor will also be read.
 - **Default**: False
 
 ### wannier_card
@@ -1055,7 +1064,6 @@ calculations.
 
   Furthermore, the old INPUT parameter exx_hybrid_type for hybrid functionals has been absorbed into dft_functional. Options are `hf` (pure Hartree-Fock), `pbe0`(PBE0), `hse` (Note: in order to use HSE functional, LIBXC is required). Note also that HSE has been tested while PBE0 has NOT been fully tested yet, and the maximum CPU cores for running exx in parallel is $N(N+1)/2$, with N being the number of atoms.
 
-  If set to `opt_orb`, the program will not perform hybrid functional calculation. Instead, it is going to generate opt-ABFs as discussed in this [article](https://pubs.acs.org/doi/abs/10.1021/acs.jpclett.0c00481).
 - **Default**: Used the same as DFT functional as specified in the pseudopotential files.
 
 ### xc_temperature
@@ -1643,7 +1651,7 @@ These variables are used to control the output of properties.
   ---
   The circle order of the charge density on real space grids is: x is the outer loop, then y and finally z (z is moving fastest).
 
-  In EXX(exact exchange) calculations, (i.e. *[dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0/opt_orb* or *[rpa](#rpa)==True*), the Hexx(R) files will be output in the folder `OUT.${suffix}` too, which can be read in NSCF calculation.
+  In EXX(exact exchange) calculations, (i.e. *[dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0* or *[rpa](#rpa)==True*), the Hexx(R) files will be output in the folder `OUT.${suffix}` too, which can be read in NSCF calculation.
 
   In molecular dynamics simulations, the output frequency is controlled by [out_interval](#out_interval).
 - **Default**: 0 3
@@ -1728,7 +1736,7 @@ These variables are used to control the output of properties.
 
 - **Type**: Integer
 - **Availability**: Numerical atomic orbital basis
-- **Description**: Whether to output the electronic wavefunction coefficients into files and store them in the folder `OUT.${suffix}`. The files are named as `wf{s}{spin index}{k(optional)}{k-point index}{g(optional)}{geometry index1}{_nao} + {".txt"/".dat"}`. Here, 's' refers to spin, where s1 means spin up channel while s2 means spin down channel, and 's12' refer to spinor wave functions that contains both spin channels with spin-orbital coupling or noncollinear calculations enabled. In addition, if 'gamma_only' is set to 0, then the optinoal k-point sampling index appears with the k-point index attached to the electronic wave function file names. Finally, if [out_app_flag](#out_app_flag) is set to false, the file name contains the optional 'g' index for each ionic step that may have different geometries, and if [out_app_flag](#out_app_flag) is set to true, the wave functions accumulate during ionic steps.
+- **Description**: Whether to output the electronic wavefunction coefficients into files and store them in the folder `OUT.${suffix}`. The files are named as `wf{s}{spin index}{k(optional)}{k-point index}{g(optional)}{geometry index1}{_nao} + {".txt"/".dat"}`. Here, 's' refers to spin, where s1 means spin up channel while s2 means spin down channel, and 's12' refer to spinor wave functions that contains both spin channels with spin-orbital coupling or noncollinear calculations enabled. In addition, if 'gamma_only' is set to 0, then the optinoal k-point sampling index appears with the k-point index attached to the electronic wave function file names. Finally, if [out_app_flag](#out_app_flag) is set to false, the file name contains the optional 'g' index for each ionic step that may have different geometries, and if [out_app_flag](#out_app_flag) is set to true, the wave functions accumulate during ionic steps. If the out_app_flag is set to false, a new folder named WFC will be created, and the wave function files will be saved into it.
   - 0: no output
   - 1: (txt format) 
     - gamma-only: `wfs1_nao.txt` or `wfs2_nao.txt`, ...;
@@ -1963,7 +1971,7 @@ These variables are used to control the output of properties.
   - auto: These files are saved in folder `OUT.${suffix}/restart/`;
   - other: These files are saved in folder `${read_file_dir}/restart/`.
 
-  If EXX(exact exchange) is calculated (i.e. *[dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0/opt_orb* or *[rpa](#rpa)==True*), the Hexx(R) files for each processor will also be saved in the above folder, which can be read in EXX calculation with *[restart_load](#restart_load)==True*.
+  If EXX(exact exchange) is calculated (i.e. *[dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0* or *[rpa](#rpa)==True*), the Hexx(R) files for each processor will also be saved in the above folder, which can be read in EXX calculation with *[restart_load](#restart_load)==True*.
 - **Default**: False
 
 ### rpa (Under Development Feature)
@@ -2925,22 +2933,22 @@ These variables are relevant when using hybrid functionals with *[basis_type](#b
 ### exx_opt_orb_lmax
 
 - **Type**: Integer
-- **Availability**: *[dft_functional](#dft_functional)==opt_orb*
+- **Availability**: *[calculation](#calculation)==gen_opt_abfs*
 - **Description**: The maximum l of the spherical Bessel functions, when the radial part of opt-ABFs are generated as linear combinations of spherical Bessel functions. A reasonable choice is 2.
 - **Default**: 0
 
 ### exx_opt_orb_ecut
 
 - **Type**: Real
-- **Availability**: *[dft_functional](#dft_functional)==opt_orb*
+- **Availability**: *[calculation](#calculation)==gen_opt_abfs*
 - **Description**: The cut-off of plane wave expansion, when the plane wave basis is used to optimize the radial ABFs. A reasonable choice is 60.
 - **Default**: 0
 - **Unit**: Ry
 
 ### exx_opt_orb_tolerence
 
 - **Type**: Real
-- **Availability**: *[dft_functional](#dft_functional)==opt_orb*
+- **Availability**: *[calculation](#calculation)==gen_opt_abfs*
 - **Description**: The threshold when solving for the zeros of spherical Bessel functions. A reasonable choice is 1e-12.
 - **Default**: 0
 
@@ -2973,7 +2981,7 @@ These variables are relevant when using hybrid functionals with *[basis_type](#b
 ### exx_symmetry_realspace
 
 - **Type**: Boolean
-- **Availability**: *[symmetry](#symmetry)==1* and exx calculation  (*[dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0/opt_orb* or *[rpa](#rpa)==True*)
+- **Availability**: *[symmetry](#symmetry)==1* and exx calculation  (*[dft_fuctional](#dft_functional)==hse/hf/pbe0/scan0* or *[rpa](#rpa)==True*)
 - **Description**: 
   - False: only rotate k-space density matrix D(k) from irreducible k-points to accelerate diagonalization
   - True: rotate both D(k) and Hexx(R) to accelerate both diagonalization and EXX calculation
@@ -3687,6 +3695,18 @@ These variables are used to control berry phase and wannier90 interface paramete
 
 ## TDDFT: time dependent density functional theory
 
+### estep_per_md
+
+- **Type**: Integer
+- **Description**: The number of electron propagation steps between two ionic steps.
+- **Default**: 1
+
+### td_dt
+
+- **Type**: Real
+- **Description**: The time step used in electron propagation. Setting td_dt will reset the md_dt value to td_dt * estep_per_md.
+- **Default**: md_dt/estep_per_md
+
 ### td_edm
 
 - **Type**: Integer
@@ -3739,6 +3759,7 @@ These variables are used to control berry phase and wannier90 interface paramete
   Type of electric field in space domain
   - 0: length gauge.
   - 1: velocity gauge.
+  - 2: hybrid gauge.
 - **Default**: 0
 
 ### td_ttype

source/CMakeLists.txt

@@ -16,7 +16,7 @@ add_subdirectory(source_io)
 add_subdirectory(source_relax)
 add_subdirectory(source_lcao/module_ri)
 add_subdirectory(source_io/module_parameter)
-add_subdirectory(module_lr)
+add_subdirectory(source_lcao/module_lr)
 
 # add by jghan
 add_subdirectory(source_lcao/module_rdmft)

source/Makefile.Objects

@@ -20,6 +20,7 @@ VPATH=./src_global:\
 ./source_cell:\
 ./source_base:\
 ./source_base/kernels:\
+./source_base/module_external:\
 ./source_base/module_container/base/core:\
 ./source_base/module_container/ATen/core:\
 ./source_base/module_container/ATen/kernels:\
@@ -32,7 +33,7 @@ VPATH=./src_global:\
 ./source_esolver:\
 ./source_hsolver:\
 ./source_hsolver/kernels:\
-./source_hsolver/genelpa:\
+./source_hsolver/module_genelpa:\
 ./source_hsolver/module_pexsi:\
 ./source_estate:\
 ./source_estate/kernels:\
@@ -69,12 +70,12 @@ VPATH=./src_global:\
 ./src_ri:\
 ./source_lcao/module_ri:\
 ./source_io/module_parameter:\
-./module_lr:\
-./module_lr/ao_to_mo_transformer:\
-./module_lr/dm_trans:\
-./module_lr/operator_casida:\
-./module_lr/potentials:\
-./module_lr/utils:\
+./source_lcao/module_lr:\
+./source_lcao/module_lr/ao_to_mo_transformer:\
+./source_lcao/module_lr/dm_trans:\
+./source_lcao/module_lr/operator_casida:\
+./source_lcao/module_lr/potentials:\
+./source_lcao/module_lr/utils:\
 ./source_lcao/module_rdmft:\
 ./\
 
@@ -356,6 +357,7 @@ OBJS_HAMILT_LCAO=hamilt_lcao.o\
     overlap_new.o\
     td_ekinetic_lcao.o\
     td_nonlocal_lcao.o\
+    td_pot_hybrid.o\
     veff_lcao.o\
     meta_lcao.o\
     op_dftu_lcao.o\
@@ -624,8 +626,9 @@ OBJS_LCAO=evolve_elec.o\
       propagator_cn2.o\
       propagator_taylor.o\
       propagator_etrs.o\
-      td_velocity.o\
-      td_current.o\
+      td_folding.o\
+      td_info.o\
+      velocity_op.o\
       snap_psibeta_half_tddft.o\
       solve_propagation.o\
       upsi.o\

source/source_base/CMakeLists.txt

@@ -10,9 +10,9 @@ add_library(
     base
     OBJECT
     assoc_laguerre.cpp
-    blas_connector_base.cpp
-    blas_connector_vector.cpp
-    blas_connector_matrix.cpp
+    module_external/blas_connector_base.cpp
+    module_external/blas_connector_vector.cpp
+    module_external/blas_connector_matrix.cpp
     clebsch_gordan_coeff.cpp
     complexarray.cpp
     complexmatrix.cpp
@@ -85,7 +85,7 @@ if(BUILD_TESTING)
     add_subdirectory(kernels/test)
     add_subdirectory(module_mixing/test)
     add_subdirectory(module_device/test)
-    add_subdirectory(grid/test)
+    add_subdirectory(module_grid/test)
     if (USE_ABACUS_LIBM)
       add_subdirectory(libm/test)
     endif()

source/source_base/complexmatrix.cpp

@@ -7,7 +7,7 @@
 
 #ifdef __NORMAL
 #else
-#include "blas_connector.h"
+#include "module_external/blas_connector.h"
 #endif
 
 namespace ModuleBase

source/source_base/gather_math_lib_info.cpp

@@ -8,8 +8,8 @@ Results will output to OUT/math_info.log, see ModuleBase::Global_File::make_dir_
 
 #include <ostream>
 #undef GATHER_INFO
-#include "source_base/blas_connector.h"
-#include "source_base/lapack_connector.h"
+#include "source_base/module_external/blas_connector.h"
+#include "source_base/module_external/lapack_connector.h"
 
 #include <iostream>
 

source/source_base/global_file.cpp

@@ -25,6 +25,7 @@ void ModuleBase::Global_File::make_dir_out(
     const std::string &suffix,
 	const std::string &calculation,
     const bool &out_dir,
+    const bool &out_wfc_dir,
     const int rank,
     const bool &restart,
     const bool out_alllog)
@@ -153,6 +154,45 @@ void ModuleBase::Global_File::make_dir_out(
 #endif
     }
 
+    if(out_wfc_dir)
+    {
+        int make_dir_wfc = 0;
+        std::string command1 =  "test -d " + PARAM.globalv.global_wfc_dir + " || mkdir " + PARAM.globalv.global_wfc_dir;
+
+        times = 0;
+        while(times<GlobalV::NPROC)
+        {
+            if(rank==times)
+            {
+                if ( system( command1.c_str() ) == 0 )
+                {
+                    std::cout << " MAKE THE WFC DIR    : " << PARAM.globalv.global_wfc_dir << std::endl;
+                    make_dir_wfc = 1;
+                }
+                else
+                {
+                    std::cout << " PROC " << rank << " CAN NOT MAKE THE WFC DIR !!! " << std::endl;
+                    make_dir_wfc = 0;
+                }
+            }
+#ifdef __MPI
+            Parallel_Reduce::reduce_all(make_dir_wfc);
+#endif
+            if(make_dir_wfc>0) { break;
+}
+            ++times;
+        }
+
+#ifdef __MPI
+        if(make_dir_wfc==0)
+        {
+            std::cout << " CAN NOT MAKE THE WFC DIR......." << std::endl;
+            ModuleBase::QUIT();
+        }
+        MPI_Barrier(MPI_COMM_WORLD);
+#endif
+    }
+
     if(PARAM.inp.of_ml_gene_data == 1)
     {
         int make_dir_descrip = 0;

source/source_base/global_file.h

@@ -23,6 +23,7 @@ namespace Global_File
 	void make_dir_out(const std::string &suffix,
 		const std::string &calculation,
         const bool &out_dir,
+		const bool &out_wfc_dir,
 		const int rank,
         const bool &restart,
 		const bool out_alllog = false); 

source/source_base/global_function.cpp

@@ -61,6 +61,7 @@ void OUT(std::ofstream &ofs, const std::string &name)
 void MAKE_DIR(const std::string &fn)
 {
     //	ModuleBase::TITLE("global_function","MAKE_DIR");
+    #ifndef __SW
     if (GlobalV::MY_RANK == 0)
     {
         std::stringstream ss;
@@ -73,6 +74,7 @@ void MAKE_DIR(const std::string &fn)
             ModuleBase::WARNING_QUIT("MAKE_DIR", fn);
         }
     }
+    #endif
     return;
 }