Merge branch 'deepmodeling:develop' into develop

Author: ddhhss

.github/workflows/cuda.yml

@@ -0,0 +1,74 @@
+name: CUDA Test
+
+on:
+  workflow_dispatch:
+
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.PAT }}
+          ec2-image-id: ami-04cd9fec4a7a39019
+          ec2-instance-type: g4dn.xlarge
+          subnet-id: subnet-72d3e53e
+          security-group-id: sg-06b0c93122c08aeab
+
+  test:
+    name: Do the job on the runner
+    needs: start-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+    container: 
+      image: ghcr.io/deepmodeling/abacus-development-kit:cuda
+      options: --gpus all
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build cuSolver
+        run: |
+          nvidia-smi
+          cmake -B build -DUSE_CUSOLVER_LCAO=ON
+          cmake --build build -j4
+          cmake --install build
+      - name: Test
+        run: |
+          export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib64
+          cd tests/integrate
+          echo "ks_solver cusolver" >> ./270_NO_MD_2O/INPUT
+          ./Autotest.sh -r 270_NO_MD_2O
+
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - test # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.PAT }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

.github/workflows/container.yml renamed to .github/workflows/devcontainer.yml

@@ -8,6 +8,7 @@ on:
 jobs:
   build_container_and_push:
     runs-on: ubuntu-latest
+    if: github.repository_owner == 'deepmodeling'
     strategy:
       matrix:
         dockerfile: ["gnu","intel","cuda"]
@@ -25,10 +26,19 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Login to Aliyun Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.DP_HARBOR_REGISTRY }}
+          username: ${{ secrets.DP_HARBOR_USERNAME }}
+          password: ${{ secrets.DP_HARBOR_PASSWORD }}
+
       - name: Build and Push Container
         uses: docker/build-push-action@v2
         with:
-          tags: ghcr.io/${{ github.repository_owner }}/abacus-development-kit:${{ matrix.dockerfile }}
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/abacus-development-kit:${{ matrix.dockerfile }}
+            ${{ secrets.DP_HARBOR_REGISTRY }}/dplc/abacus-${{ matrix.dockerfile }}:latest
           file: Dockerfile.${{ matrix.dockerfile }}
           cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/abacus-development-kit:${{matrix.dockerfile}}
           cache-to: type=inline

.github/workflows/image.yml

@@ -0,0 +1,42 @@
+name: Build Image
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  build_container_and_push:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'deepmodeling'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to Aliyun Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.DP_HARBOR_REGISTRY }}
+          username: ${{ secrets.DP_HARBOR_USERNAME }}
+          password: ${{ secrets.DP_HARBOR_PASSWORD }}
+
+      - name: Build and Push Container
+        uses: docker/build-push-action@v2
+        with:
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/abacus:latest
+            ${{ secrets.DP_HARBOR_REGISTRY }}/dplc/abacus:latest
+          file: Dockerfile
+          cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/abacus:latest
+          cache-to: type=inline
+          push: true

.github/workflows/performance.yml

@@ -0,0 +1,34 @@
+name: Performance test
+
+on:
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Performance test
+    runs-on: self-hosted
+    strategy:
+      matrix:
+        tag: ["gnu", "intel"]
+    if: github.repository_owner == 'deepmodeling'
+    container: ghcr.io/${{ github.repository_owner }}/abacus-development-kit:${{ matrix.tag }}
+    timeout-minutes: 2880
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Requirements
+        run: |
+          apt install -y time
+      - name: Build
+        run: |
+          cmake -B build -DENABLE_LIBXC=ON
+          cmake --build build -j16
+          cmake --install build
+      - name: Test
+        run: |
+          test -e /opt/intel/oneapi/setvars.sh && . /opt/intel/oneapi/setvars.sh
+          cd examples/performance/
+          bash run.sh
+      - name: Show Result
+        run: |
+          cat examples/performance/sumall.dat

.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
           ref: "refs/pull/${{ github.event.number }}/merge"
       - name: Build
         run: |
-          cmake -B build -DBUILD_TESTING=ON -DENABLE_DEEPKS=ON
+          cmake -B build -DBUILD_TESTING=ON -DENABLE_DEEPKS=ON -DENABLE_LIBXC=ON
           cmake --build build -j8
           cmake --install build
       - name: Test

CMakeLists.txt

@@ -1,34 +1,37 @@
 ########################################
 # CMake build system
 # This file is part of ABACUS
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.16)
 ########################################
 
 project(ABACUS
-    VERSION 2.2.0
+    VERSION 2.2.3
     DESCRIPTION "ABACUS is an electronic structure package based on DFT."
     HOMEPAGE_URL "https://github.com/deepmodeling/abacus-develop"
     LANGUAGES CXX
 )
 
 option(ENABLE_DEEPKS "Enable DeePKS functionality" OFF)
 option(ENABLE_LIBXC "Enable LibXC functionality" OFF)
-option(USE_CUDA "Enable support to CUDA." OFF)
+option(USE_CUDA "Enable support to CUDA for PW." OFF)
+option(USE_CUSOLVER_LCAO "Enable support to CUSOLVER for LCAO." OFF)
 option(USE_ROCM "Enable support to ROCm." OFF)
 option(USE_OPENMP " Enable OpenMP in abacus." ON)
 option(ENABLE_ASAN "Enable AddressSanitizer" OFF)
 option(BUILD_TESTING "Build ABACUS unit tests" OFF)
 option(GENERATE_TEST_REPORTS "Enable test report generation" OFF)
 
 set(ABACUS_BIN_NAME abacus)
-set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/modules)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/modules)
 set(ABACUS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/source)
 set(ABACUS_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tests)
 set(ABACUS_BIN_PATH ${CMAKE_CURRENT_BINARY_DIR}/${ABACUS_BIN_NAME})
 include_directories(${ABACUS_SOURCE_DIR})
 add_executable(${ABACUS_BIN_NAME} source/main.cpp)
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 add_compile_options(-O2 -g)
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-write-strings" )
+add_compile_options(-Wno-write-strings)
 
 find_package(Cereal REQUIRED)
 include_directories(${Cereal_INCLUDE_DIR})
@@ -51,32 +54,41 @@ if(USE_OPENMP)
   find_package(OpenMP REQUIRED)
   target_link_libraries(${ABACUS_BIN_NAME} OpenMP::OpenMP_CXX)
   add_compile_options(${OpenMP_CXX_FLAGS})
-  #add_compile_definitions(_OPENMP)
 endif()
 
-set(CMAKE_CXX_STANDARD 11)
 include(CheckLanguage)
 check_language(CUDA)
 if(CMAKE_CUDA_COMPILER)
-  if(NOT DEFINED USE_CUDA)
-    message("CUDA components detected. \nWill build the CUDA version of ABACUS.")
-    set(USE_CUDA ON)
+  if(NOT DEFINED USE_CUDA OR NOT DEFINED USE_CUSOLVER_LCAO)
+    if (NOT DEFINED USE_CUDA AND NOT DEFINED USE_CUSOLVER_LCAO)
+      message("CUDA components detected. \nWill build the CUDA for PW version of ABACUS by default.")
+      set(USE_CUDA ON)
+      set(USE_CUSOLVER_LCAO OFF)
+    elseif (NOT DEFINED USE_CUDA)
+      set(USE_CUDA OFF)
+    else()
+      set(USE_CUSOLVER_LCAO OFF)
+    endif()
   else()
-    if(NOT USE_CUDA)
-      message(WARNING "CUDA components detected, but USE_CUDA set to OFF. \nNOT building CUDA version of ABACUS.")
+    if(NOT USE_CUDA AND NOT USE_CUSOLVER_LCAO)
+      message(STATUS "CUDA components detected, but both USE_CUDA and USE_CUSOLVER_LCAO set to OFF. NOT building CUDA version of ABACUS.")
+    elseif (USE_CUDA AND USE_CUSOLVER_LCAO)
+      message(FATAL_ERROR "USE_CUDA and USE_CUSOLVER_LCAO set, but now they not allowed to coexist.")
     endif()
   endif()
 else() # CUDA not found
-  if (USE_CUDA)
-    message(FATAL_ERROR "USE_CUDA set but no CUDA components found.")
+  if (USE_CUDA OR USE_CUSOLVER_LCAO)
+    message(FATAL_ERROR "USE_CUDA or USE_CUSOLVER_LCAO set but no CUDA components found.")
     set(USE_CUDA OFF)
+    set(USE_CUSOLVER_LCAO OFF)
   endif()
 endif()
-if(USE_CUDA)
+
+if(USE_CUDA OR USE_CUSOLVER_LCAO)
+  cmake_minimum_required(VERSION 3.18) # required by `CUDA_ARCHITECTURES` below
   set(CMAKE_CXX_STANDARD 14)
   set(CMAKE_CXX_EXTENSIONS ON)
-  set(CMAKE_CXX_STANDARD_REQUIRED ON)
-  set(CMAKE_CUDA_STANDARD 14)
+  set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
   set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
   enable_language(CUDA)
@@ -92,17 +104,22 @@ if(USE_CUDA)
   60 # P100
   70 # V100
   75 # T4
+  80 # A100
   )
   include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-  add_compile_definitions(__CUDA)
+  if (USE_CUDA)
+    add_compile_definitions(__CUDA)
+  endif()
+  if (USE_CUSOLVER_LCAO)
+    add_compile_definitions(__CUSOLVER_LCAO)
+  endif()
 endif()
 
+
 # Warning: CMake add support to HIP in version 3.21. This is rather a new version.
 # Use cmake with AMD-ROCm: https://rocmdocs.amd.com/en/latest/Installation_Guide/Using-CMake-with-AMD-ROCm.html
 if(USE_ROCM)
-  set(CMAKE_CXX_STANDARD 11)
-  SET(CMAKE_HIP_STANDARD 11)
-  # set(CMAKE_CXX_STANDARD_REQUIRED True)
+  SET(CMAKE_HIP_STANDARD ${CMAKE_CXX_STANDARD})
   list(APPEND CMAKE_PREFIX_PATH /opt/rocm/hip /opt/rocm)
   list(APPEND CMAKE_SYSTEM_PREFIX_PATH "/opt/rocm/hip")
 
@@ -125,6 +142,9 @@ if(USE_ROCM)
 endif()
 
 if(ENABLE_ASAN)
+  if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+    message(FATAL_ERROR "Address Sanitizer is not supported on Intel compiler.")
+  endif()
   add_compile_options(
     -fsanitize=address
     -fno-omit-frame-pointer
@@ -186,13 +206,16 @@ if(ENABLE_DEEPKS)
     FetchContent_Declare(
       libnpy
       GIT_REPOSITORY https://github.com/llohse/libnpy.git
+      GIT_SHALLOW TRUE
+      GIT_PROGRESS TRUE
     )
     FetchContent_MakeAvailable(libnpy)
   endif()
   include_directories(${libnpy_SOURCE_DIR}/include)
   add_compile_definitions(__DEEPKS)
 endif()
 
+list(APPEND math_libs m)
 target_link_libraries(${ABACUS_BIN_NAME} ${math_libs})
 
 if(DEFINED Libxc_DIR)
@@ -224,17 +247,18 @@ add_compile_definitions(
 )
 
 IF (BUILD_TESTING)
+  set(CMAKE_CXX_STANDARD 14) # Required in orbital
   include(CTest)
   enable_testing()
-  find_package(Threads)
-  set(CMAKE_CXX_STANDARD_REQUIRED ON)
   find_package(GTest HINTS /usr/local/lib/ ${GTEST_DIR})
   if(NOT ${GTest_FOUND})
     include(FetchContent)
     FetchContent_Declare(
       googletest
       GIT_REPOSITORY https://github.com/google/googletest.git
       GIT_TAG "origin/main"
+      GIT_SHALLOW TRUE
+      GIT_PROGRESS TRUE
     )
     FetchContent_MakeAvailable(googletest)
   endif()
@@ -247,8 +271,11 @@ IF (BUILD_TESTING)
     add_executable(${UT_TARGET} ${UT_SOURCES})
     #dependencies & link library
     target_link_libraries(${UT_TARGET} ${UT_LIBS}
-      OpenMP::OpenMP_CXX pthread GTest::gtest_main GTest::gmock_main)
-    install(TARGETS ${UT_TARGET} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/../../../tests )
+      Threads::Threads GTest::gtest_main GTest::gmock_main)
+    if(USE_OPENMP)
+      target_link_libraries(${UT_TARGET} OpenMP::OpenMP_CXX)
+    endif()
+    install(TARGETS ${UT_TARGET} DESTINATION ${CMAKE_BINARY_DIR}/tests )
     add_test(NAME ${UT_TARGET}
       COMMAND ${UT_TARGET}
       WORKING_DIRECTORY $<TARGET_FILE_DIR:${UT_TARGET}>
@@ -263,6 +290,7 @@ target_link_libraries(${ABACUS_BIN_NAME}
     cell
     symmetry
     md
+    planewave
     surchem
     neighbor
     orb
@@ -276,8 +304,11 @@ target_link_libraries(${ABACUS_BIN_NAME}
     ri
     driver
     xc
+    hsolver
+    elecstate
+    hamilt
+    psi
     esolver
-    -lm
 )
 
 install(PROGRAMS ${ABACUS_BIN_PATH}