refactor: move parallel_2d to module_base

CMakeLists.txt

@@ -421,10 +421,8 @@ else()
   include_directories(${FFTW3_INCLUDE_DIRS})
   list(APPEND math_libs FFTW3::FFTW3 LAPACK::LAPACK BLAS::BLAS)
 
-  if(ENABLE_LCAO)
-    find_package(ScaLAPACK REQUIRED)
-    list(APPEND math_libs ScaLAPACK::ScaLAPACK)
-  endif()
+  find_package(ScaLAPACK REQUIRED)
+  list(APPEND math_libs ScaLAPACK::ScaLAPACK)
 
   if(USE_OPENMP)
     list(APPEND math_libs FFTW3::FFTW3_OMP)

source/Makefile.Objects

@@ -170,6 +170,7 @@ OBJS_BASE=abfs-vector3_order.o\
     broyden_mixing.o\
     memory_op.o\
     device.o\
+    parallel_2d.o\
 
 OBJS_CELL=atom_pseudo.o\
     atom_spec.o\
@@ -378,7 +379,6 @@ OBJS_ORBITAL=ORB_atomic.o\
       ORB_nonlocal.o\
       ORB_nonlocal_lm.o\
       ORB_read.o\
-      parallel_2d.o\
       parallel_orbitals.o\
 	  atomic_radials.o\
       hydrogen_radials.o\

source/module_base/CMakeLists.txt

@@ -53,6 +53,7 @@ add_library(
     parallel_device.cpp
     spherical_bessel_transformer.cpp
     cubic_spline.cpp
+    parallel_2d.cpp
     module_mixing/mixing_data.cpp
     module_mixing/mixing.cpp
     module_mixing/plain_mixing.cpp

source/module_base/test_parallel/CMakeLists.txt

@@ -34,7 +34,20 @@ add_test(NAME base_parallel_reduce_test
       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )
 
-if(ENABLE_LCAO)
+AddTest(
+    TARGET parallel_2d_test
+    SOURCES parallel_2d_test.cpp ../parallel_2d.cpp
+    LIBS parameter ${math_libs} 
+)
+
+install(FILES parallel_2d_test.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+find_program(BASH bash)
+add_test(NAME parallel_2d_test_para
+      COMMAND ${BASH} parallel_2d_test.sh
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+
   # figure out the lib that provides BLACS
   if(MKLROOT)
     list(APPEND BLACS_LIB MKL::MKL MKL::MKL_SCALAPACK)
@@ -52,4 +65,4 @@ if(ENABLE_LCAO)
         COMMAND ${BASH} blacs_connector_test.sh
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   )
-endif()
+

source/module_basis/module_ao/CMakeLists.txt

@@ -8,7 +8,6 @@ if(ENABLE_LCAO)
       ORB_nonlocal.cpp
       ORB_nonlocal_lm.cpp
       ORB_read.cpp
-      parallel_2d.cpp
       parallel_orbitals.cpp
   )
 

source/module_basis/module_ao/parallel_orbitals.h

@@ -1,6 +1,6 @@
 #ifndef _PARALLEL_ORBITALS_H_
 #define _PARALLEL_ORBITALS_H_
-#include "parallel_2d.h"
+#include "module_base/parallel_2d.h"
 #include <fstream>
 
 /// This class packs the information of 2D-block-cyclic for LCAO code:

source/module_basis/module_ao/test/CMakeLists.txt

@@ -27,13 +27,13 @@ list(APPEND depend_files
   ../../../module_base/libm/sincos.cpp
   ../../../module_base/spherical_bessel_transformer.cpp
   ../../../module_base/cubic_spline.cpp
+  ../../../module_base/parallel_2d.cpp
   ../ORB_read.cpp
   ../ORB_atomic.cpp
   ../ORB_atomic_lm.cpp
   ../ORB_nonlocal.cpp
   ../ORB_nonlocal_lm.cpp
   ../ORB_gaunt_table.cpp
-  ../parallel_2d.cpp
   ../parallel_orbitals.cpp
   ../../../module_hamilt_lcao/hamilt_lcaodft/center2_orb-orb11.cpp
   ../../../module_hamilt_lcao/hamilt_lcaodft/center2_orb.cpp
@@ -75,22 +75,9 @@ AddTest(
 )
 
 
-AddTest(
-    TARGET parallel_2d_test
-    SOURCES parallel_2d_test.cpp ../parallel_2d.cpp
-    LIBS parameter ${math_libs} device base
-)
-
-install(FILES parallel_2d_test.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-find_program(BASH bash)
-add_test(NAME parallel_2d_test_para
-      COMMAND ${BASH} parallel_2d_test.sh
-      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-)
-
 AddTest(
     TARGET parallel_orbitals_test 
-    SOURCES parallel_orbitals_test.cpp ../parallel_2d.cpp ../parallel_orbitals.cpp
+    SOURCES parallel_orbitals_test.cpp ../parallel_orbitals.cpp
     LIBS parameter ${math_libs} device base
 )
 

source/module_elecstate/module_dm/test/CMakeLists.txt

@@ -15,7 +15,6 @@ AddTest(
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/base_matrix.cpp
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/hcontainer.cpp
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/atom_pair.cpp
-  ${ABACUS_SOURCE_DIR}/module_basis/module_ao/parallel_2d.cpp
   ${ABACUS_SOURCE_DIR}/module_basis/module_ao/parallel_orbitals.cpp
   ${ABACUS_SOURCE_DIR}/module_io/output.cpp
 )
@@ -27,7 +26,6 @@ AddTest(
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/base_matrix.cpp
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/hcontainer.cpp
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/atom_pair.cpp
-  ${ABACUS_SOURCE_DIR}/module_basis/module_ao/parallel_2d.cpp
   ${ABACUS_SOURCE_DIR}/module_basis/module_ao/parallel_orbitals.cpp
 )
 
@@ -38,7 +36,6 @@ AddTest(
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/base_matrix.cpp
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/hcontainer.cpp
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/atom_pair.cpp
-  ${ABACUS_SOURCE_DIR}/module_basis/module_ao/parallel_2d.cpp
   ${ABACUS_SOURCE_DIR}/module_basis/module_ao/parallel_orbitals.cpp
 )
 
@@ -49,6 +46,5 @@ AddTest(
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/base_matrix.cpp
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/hcontainer.cpp
   ${ABACUS_SOURCE_DIR}/module_hamilt_lcao/module_hcontainer/atom_pair.cpp
-  ${ABACUS_SOURCE_DIR}/module_basis/module_ao/parallel_2d.cpp
   ${ABACUS_SOURCE_DIR}/module_basis/module_ao/parallel_orbitals.cpp
 )

source/module_hamilt_lcao/hamilt_lcaodft/operator_lcao/test/CMakeLists.txt

@@ -6,7 +6,7 @@ AddTest(
   LIBS parameter ${math_libs} psi base device container
   SOURCES test_overlapnew.cpp ../overlap_new.cpp ../../../module_hcontainer/func_folding.cpp 
   ../../../module_hcontainer/base_matrix.cpp ../../../module_hcontainer/hcontainer.cpp ../../../module_hcontainer/atom_pair.cpp  
-  ../../../../module_basis/module_ao/parallel_2d.cpp ../../../../module_basis/module_ao/parallel_orbitals.cpp 
+  ../../../../module_basis/module_ao/parallel_orbitals.cpp 
   ../../../../module_basis/module_ao/ORB_atomic_lm.cpp
   tmp_mocks.cpp ../../../../module_hamilt_general/operator.cpp
 )
@@ -16,7 +16,7 @@ AddTest(
   LIBS parameter ${math_libs} psi base device container
   SOURCES test_overlapnew_cd.cpp ../overlap_new.cpp ../../../module_hcontainer/func_folding.cpp 
   ../../../module_hcontainer/base_matrix.cpp ../../../module_hcontainer/hcontainer.cpp ../../../module_hcontainer/atom_pair.cpp  
-  ../../../../module_basis/module_ao/parallel_2d.cpp ../../../../module_basis/module_ao/parallel_orbitals.cpp 
+  ../../../../module_basis/module_ao/parallel_orbitals.cpp 
   ../../../../module_basis/module_ao/ORB_atomic_lm.cpp
   tmp_mocks.cpp ../../../../module_hamilt_general/operator.cpp
 )
@@ -26,7 +26,7 @@ AddTest(
   LIBS parameter ${math_libs} psi base device container
   SOURCES test_ekineticnew.cpp ../ekinetic_new.cpp ../../../module_hcontainer/func_folding.cpp 
   ../../../module_hcontainer/base_matrix.cpp ../../../module_hcontainer/hcontainer.cpp ../../../module_hcontainer/atom_pair.cpp  
-  ../../../../module_basis/module_ao/parallel_2d.cpp ../../../../module_basis/module_ao/parallel_orbitals.cpp 
+  ../../../../module_basis/module_ao/parallel_orbitals.cpp 
   ../../../../module_basis/module_ao/ORB_atomic_lm.cpp
   tmp_mocks.cpp ../../../../module_hamilt_general/operator.cpp
 )
@@ -36,7 +36,7 @@ AddTest(
   LIBS parameter ${math_libs} psi base device container
   SOURCES test_nonlocalnew.cpp ../nonlocal_new.cpp ../../../module_hcontainer/func_folding.cpp 
   ../../../module_hcontainer/base_matrix.cpp ../../../module_hcontainer/hcontainer.cpp ../../../module_hcontainer/atom_pair.cpp  
-  ../../../../module_basis/module_ao/parallel_2d.cpp ../../../../module_basis/module_ao/parallel_orbitals.cpp 
+  ../../../../module_basis/module_ao/parallel_orbitals.cpp 
   ../../../../module_basis/module_ao/ORB_atomic_lm.cpp
   tmp_mocks.cpp ../../../../module_hamilt_general/operator.cpp
 )
@@ -46,7 +46,7 @@ AddTest(
   LIBS parameter ${math_libs} psi base device container 
   SOURCES test_T_NL_cd.cpp ../nonlocal_new.cpp ../ekinetic_new.cpp ../../../module_hcontainer/func_folding.cpp 
   ../../../module_hcontainer/base_matrix.cpp ../../../module_hcontainer/hcontainer.cpp ../../../module_hcontainer/atom_pair.cpp  
-  ../../../../module_basis/module_ao/parallel_2d.cpp ../../../../module_basis/module_ao/parallel_orbitals.cpp 
+  ../../../../module_basis/module_ao/parallel_orbitals.cpp 
   ../../../../module_basis/module_ao/ORB_atomic_lm.cpp
   tmp_mocks.cpp ../../../../module_hamilt_general/operator.cpp
 )
@@ -56,7 +56,7 @@ AddTest(
   LIBS parameter ${math_libs} psi base device container 
   SOURCES test_dftu.cpp ../dftu_lcao.cpp ../../../module_hcontainer/func_folding.cpp 
   ../../../module_hcontainer/base_matrix.cpp ../../../module_hcontainer/hcontainer.cpp ../../../module_hcontainer/atom_pair.cpp  
-  ../../../../module_basis/module_ao/parallel_2d.cpp ../../../../module_basis/module_ao/parallel_orbitals.cpp 
+  ../../../../module_basis/module_ao/parallel_orbitals.cpp 
   ../../../../module_basis/module_ao/ORB_atomic_lm.cpp
   tmp_mocks.cpp ../../../../module_hamilt_general/operator.cpp
 )

source/module_hamilt_lcao/module_deltaspin/test/CMakeLists.txt

@@ -12,7 +12,6 @@ AddTest(
   LIBS ${math_libs} base device parameter
   SOURCES spin_constrain_test.cpp
     ../spin_constrain.cpp
-    ../../../module_basis/module_ao/parallel_2d.cpp
     ../../../module_basis/module_ao/parallel_orbitals.cpp
 )
 

source/module_hamilt_lcao/module_hcontainer/test/CMakeLists.txt

@@ -4,35 +4,35 @@ AddTest(
   TARGET hcontainer_test
   LIBS parameter ${math_libs} psi base device
   SOURCES test_hcontainer.cpp ../base_matrix.cpp ../hcontainer.cpp ../atom_pair.cpp  
-  ../../../module_basis/module_ao/parallel_2d.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
+  ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
 )
 
 AddTest(
   TARGET hcontainer_complex_test
   LIBS parameter ${math_libs} psi base device
   SOURCES test_hcontainer_complex.cpp ../base_matrix.cpp ../hcontainer.cpp ../atom_pair.cpp  
-  ../../../module_basis/module_ao/parallel_2d.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
+  ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
 )
 
 AddTest(
   TARGET hcontainer_cost_test
   LIBS parameter ${math_libs} psi base device
   SOURCES test_hcontainer_time.cpp ../base_matrix.cpp ../hcontainer.cpp ../atom_pair.cpp  
-  ../../../module_basis/module_ao/parallel_2d.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
+  ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
 )
 
 AddTest(
   TARGET hcontainer_folding_test
   LIBS parameter ${math_libs} psi base device
   SOURCES test_func_folding.cpp ../base_matrix.cpp ../hcontainer.cpp ../atom_pair.cpp  
-  ../func_folding.cpp ../../../module_basis/module_ao/parallel_2d.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
+  ../func_folding.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
 )
 
 AddTest(
   TARGET hcontainer_transfer_test
   LIBS parameter ${math_libs} psi base device
   SOURCES test_transfer.cpp ../func_transfer.cpp ../base_matrix.cpp ../hcontainer.cpp ../atom_pair.cpp  
-  ../transfer.cpp ../../../module_basis/module_ao/parallel_2d.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
+  ../transfer.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp tmp_mocks.cpp
 )
 
 install(FILES parallel_hcontainer_tests.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

source/module_hamilt_lcao/module_tddft/test/CMakeLists.txt

@@ -13,7 +13,7 @@ AddTest(
 AddTest(
   TARGET tddft_bandenergy_test
   LIBS parameter ${math_libs} base device tddft_test_lib  
-  SOURCES bandenergy_test.cpp ../bandenergy.cpp ../../../module_basis/module_ao/parallel_2d.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp
+  SOURCES bandenergy_test.cpp ../bandenergy.cpp ../../../module_basis/module_ao/parallel_orbitals.cpp
 )
 
 AddTest(

source/module_hsolver/parallel_k2d.h

@@ -1,7 +1,7 @@
 #ifndef PARALLEL_K2D_H
 #define PARALLEL_K2D_H
 
-#include "module_basis/module_ao/parallel_2d.h"
+#include "module_base/parallel_2d.h"
 #include "module_cell/parallel_kpoints.h"
 #include "module_hamilt_general/matrixblock.h"
 #ifdef __MPI

source/module_hsolver/test/CMakeLists.txt

@@ -7,7 +7,7 @@ if (ENABLE_MPI)
   AddTest(
     TARGET hsolver_parak2d_test
     LIBS parameter  ${math_libs} base device MPI::MPI_CXX
-    SOURCES parallel_k2d_test.cpp ../parallel_k2d.cpp ../../module_cell/parallel_kpoints.cpp  ../../module_basis/module_ao/parallel_2d.cpp
+    SOURCES parallel_k2d_test.cpp ../parallel_k2d.cpp ../../module_cell/parallel_kpoints.cpp
   )
   AddTest(
     TARGET HSolver_bpcg
@@ -105,7 +105,7 @@ if (ENABLE_MPI)
     AddTest(
       TARGET HSolver_LCAO_PEXSI
       LIBS parameter  ${math_libs} ${PEXSI_LIBRARY} ${SuperLU_DIST_LIBRARY} ${ParMETIS_LIBRARY} ${METIS_LIBRARY} MPI::MPI_CXX base psi device pexsi
-      SOURCES diago_pexsi_test.cpp ../diago_pexsi.cpp ../../module_basis/module_ao/parallel_orbitals.cpp ../../module_basis/module_ao/parallel_2d.cpp
+      SOURCES diago_pexsi_test.cpp ../diago_pexsi.cpp ../../module_basis/module_ao/parallel_orbitals.cpp
     )
     endif()
   endif()

source/module_io/io_dmk.cpp

@@ -139,7 +139,7 @@ bool ModuleIO::read_dmk(const int nspin,
 
     int nlocal = pv.get_global_row_size();
     bool gamma_only = std::is_same<double, T>::value;
-    std::vector<std::vector<T>> dmk_global;
+    std::vector<std::vector<T>> dmk_global(nspin * nk, std::vector<T>(nlocal * nlocal,0));
 
     // write a lambda function to check the consistency of the data
     auto check_consistency = [&](const std::string& fn,
@@ -160,8 +160,6 @@ bool ModuleIO::read_dmk(const int nspin,
     bool read_success = true;
     std::string tmp;
     if (my_rank == 0) {
-        dmk_global.resize(nspin * nk, std::vector<T>(nlocal * nlocal));
-
         for (int ispin = 0; ispin < nspin; ispin++) {
             for (int ik = 0; ik < nk; ik++) {
                 std::string fn = dmk_dir + dmk_gen_fname(gamma_only, ispin, ik);

source/module_io/io_dmk.h

@@ -1,7 +1,7 @@
 #ifndef DM_IO_H
 #define DM_IO_H
 
-#include "module_basis/module_ao/parallel_2d.h"
+#include "module_base/parallel_2d.h"
 #include "module_cell/unitcell.h"
 
 #include <string>

source/module_io/read_wfc_lcao.h

@@ -10,7 +10,7 @@
 #ifdef __MPI
 // parallelization
 #include "module_base/scalapack_connector.h"
-#include "module_basis/module_ao/parallel_2d.h"
+#include "module_base/parallel_2d.h"
 #endif
 
 /**

source/module_io/test/CMakeLists.txt

@@ -85,7 +85,7 @@ AddTest(
 AddTest(
   TARGET io_write_wfc_nao
   LIBS parameter  ${math_libs} base psi device
-  SOURCES write_wfc_nao_test.cpp ../write_wfc_nao.cpp ../../module_basis/module_ao/parallel_2d.cpp ../../module_basis/module_ao/parallel_orbitals.cpp ../binstream.cpp
+  SOURCES write_wfc_nao_test.cpp ../write_wfc_nao.cpp ../../module_basis/module_ao/parallel_orbitals.cpp ../binstream.cpp
 )
 
 install(FILES write_wfc_nao_para.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
@@ -198,7 +198,6 @@ AddTest(
   LIBS parameter  base ${math_libs} device
   SOURCES output_mulliken_test.cpp output_mulliken_mock.cpp ../output_mulliken.cpp
   ../../module_cell/cell_index.cpp
-  ../../module_basis/module_ao/parallel_2d.cpp
   ../../module_basis/module_ao/parallel_orbitals.cpp
   ../orb_io.cpp
 )
@@ -207,7 +206,7 @@ if(ENABLE_LCAO)
 AddTest(
   TARGET read_wfc_lcao_test
   LIBS parameter  base ${math_libs} device
-  SOURCES read_wfc_lcao_test.cpp ../read_wfc_lcao.cpp ../../module_basis/module_ao/parallel_2d.cpp
+  SOURCES read_wfc_lcao_test.cpp ../read_wfc_lcao.cpp
 )
 
 add_test(NAME read_wfc_lcao_test_parallel
@@ -237,4 +236,28 @@ AddTest(
 add_test(NAME orb_io_test_parallel
       COMMAND mpirun -np 4 ./orb_io_test
       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+AddTest(
+  TARGET io_dmk_io
+  LIBS parameter ${math_libs} base device cell_info 
+  SOURCES io_dmk_test.cpp ../io_dmk.cpp ../output.cpp
+)
+
+add_test(
+  NAME io_dmk_io_parallel
+  COMMAND mpirun -np 2 ./io_dmk_io
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+AddTest(
+  TARGET io_read_wfc_nao_test
+  LIBS parameter ${math_libs} base device
+  SOURCES read_wfc_nao_test.cpp ../read_wfc_nao.cpp ../../module_psi/psi.cpp ../../module_basis/module_ao/parallel_orbitals.cpp
+)
+
+add_test(
+  NAME io_read_wfc_nao_test_parallel
+  COMMAND mpirun -np 2 ./io_read_wfc_nao_test
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )