add RALL

Author: A-006

CMakeLists.txt

@@ -269,7 +269,6 @@ if(ENABLE_MPI)
   add_compile_definitions(__MPI)
   list(APPEND math_libs MPI::MPI_CXX)
 endif()
-target_link_libraries(${ABACUS_BIN_NAME} ${SCALAPACK_LIBRARY_DIR})
 
 
 if (USE_DSP)
@@ -459,10 +458,8 @@ else()
   find_package(Lapack REQUIRED)
   include_directories(${FFTW3_INCLUDE_DIRS})
   list(APPEND math_libs FFTW3::FFTW3 LAPACK::LAPACK BLAS::BLAS)
-  if (ENBALE_LCAO)
   find_package(ScaLAPACK REQUIRED)
   list(APPEND math_libs ScaLAPACK::ScaLAPACK)
-  endif()
   if(USE_OPENMP)
     list(APPEND math_libs FFTW3::FFTW3_OMP)
   endif()

source/module_basis/module_pw/module_fft/fft_bundle.h

@@ -203,5 +203,18 @@ class FFT_Bundle
     std::string device = "cpu";
     std::string precision = "double";
 };
+// Use RAII (Resource Acquisition Is Initialization) to 
+// control the resources used by hthread when setting the DSP
+struct FFT_RALL
+  {
+      const FFT_Bundle& fft_;
+      FFT_RALL(const FFT_Bundle& fft) : fft_(fft) 
+        {fft_.resource_handler(1);}
+      ~FFT_RALL()
+      {
+        fft_.resource_handler(0);
+      }
+  };
+
 } // namespace ModulePW
 #endif // FFT_H

source/module_basis/module_pw/module_fft/fft_dsp.cpp

@@ -63,15 +63,7 @@ void FFT_DSP<double>::setupFFT()
 template <>
 void FFT_DSP<double>::resource_handler(const int flag) const
 {
-    switch (ResourceState)
-    {
-    case constant expression:
-        /* code */
-        break;
-    
-    default:
-        break;
-    }
+    if (flag == 0)
     {
         hthread_barrier_destroy(b_id);
         hthread_group_destroy(thread_id_for);
@@ -84,6 +76,8 @@ void FFT_DSP<double>::resource_handler(const int flag) const
         b_id = hthread_barrier_create(cluster_id);
         args_for[0] = b_id;
         args_back[0] = b_id;
+    }else{
+        ModuleBase::WARNING_QUIT("FFT_DSP", "Error use of fft resource handle");
     }
 }
 template <>

source/module_basis/module_pw/module_fft/fft_dsp.h

@@ -12,12 +12,7 @@
 
 namespace ModulePW
 {
-    enum class ResourceState
-    {
-        Destroyed = 0,
-        Created = 1,
-        INVALID
-    }
+    
 template <typename FPTYPE>
 class FFT_DSP : public FFT_BASE<FPTYPE>
 {
@@ -30,7 +25,11 @@ class FFT_DSP : public FFT_BASE<FPTYPE>
         void clear() override;
 
         void cleanFFT() override;
-
+        /**
+         * @brief Control the allocation or deallocation of hthread 
+         * resource 
+         * @param flag  0: deallocate, 1: allocate
+         */
         void resource_handler(const int flag) const override;
         /** 
         * @brief Initialize the fft parameters

source/module_hamilt_pw/hamilt_pwdft/operator_pw/veff_pw.cpp

@@ -57,9 +57,38 @@ void Veff<OperatorPW<T, Device>>::act(
     const int current_spin = this->isk[this->ik];
 
 #ifdef __DSP
-    // wfcpw->fft_bundle.resource_handler(1);
+    ModulePW::FFT_RALL guard(wfcpw->fft_bundle);
+    for (int ib = 0; ib<nbands ; ib += npol)
+    {
+        if (npol == 1)
+        {
+            wfcpw->convolution(this->ctx,
+            this->ik,
+            this->veff_col,
+            tmpsi_in,
+            this->veff+ current_spin* this->veff_col,
+            tmhpsi,
+            true);
+        }else{
+            // Should be replaced in the Convolution way.
+            wfcpw->recip_to_real<T,Device>(tmpsi_in, this->porter, this->ik);
+            wfcpw->recip_to_real<T,Device>(tmpsi_in + max_npw, this->porter1, this->ik);
+            if(this->veff_col != 0)
+            {
+                /// denghui added at 20221109
+				const Real* current_veff[4];
+				for(int is = 0; is < 4; is++) 
+				{
+					current_veff[is] = this->veff + is * this->veff_col ; // for CPU device
+				}
+                veff_op()(this->ctx, this->veff_col, this->porter, this->porter1, current_veff);
+            }
+            // FFT back to G space.
+            wfcpw->real_to_recip<T,Device>(this->porter, tmhpsi, this->ik, true);
+            wfcpw->real_to_recip<T,Device>(this->porter1, tmhpsi + max_npw, this->ik, true);
+        }
+    }
 #endif
-    // std::cout<<"the Device is "<<Device;
     for (int ib = 0; ib < nbands; ib += npol)
     {
         if (npol == 1)
@@ -73,13 +102,6 @@ void Veff<OperatorPW<T, Device>>::act(
                 veff_op()(this->ctx, this->veff_col, this->porter, this->veff + current_spin * this->veff_col);
             }
             wfcpw->real_to_recip<T,Device>(this->porter, tmhpsi, this->ik, true);
-            // wfcpw->convolution(this->ctx,
-            //         this->ik,
-            //         this->veff_col,
-            //         tmpsi_in,
-            //         this->veff+ current_spin* this->veff_col,
-            //         tmhpsi,
-            //         true);
         }
         else
         {
@@ -103,9 +125,6 @@ void Veff<OperatorPW<T, Device>>::act(
         tmhpsi += max_npw * npol;
         tmpsi_in += max_npw * npol;
     }
-#ifdef __DSP
-    // wfcpw->fft_bundle.resource_handler(0);
-#endif
     ModuleBase::timer::tick("Operator", "veff_pw");
 }