Merge branch 'develop' of https://github.com/mohanchen/abacus-mc into develop

Author: mohanchen

CMakeLists.txt

@@ -286,10 +286,6 @@ if (USE_SW)
   include_directories(${SW_FFT}/include)
 
   target_link_libraries(${ABACUS_BIN_NAME} ${SW_FFT}/lib/libfftw3.a)
-  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswfft.a)
-  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswscalapack.a)
-  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswlapack.a)
-  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswblas.a)
 endif()
 
 find_package(Threads REQUIRED)
@@ -470,10 +466,7 @@ if(MKLROOT)
   if(CMAKE_CXX_COMPILER_ID MATCHES Intel)
     list(APPEND math_libs ifcore)
   endif()
-elseif(USE_SW)
-   list(APPEND math_libs gfortran)
-  # SW architecture can only use its own math library
-else()
+elseif(NOT USE_SW)
   find_package(FFTW3 REQUIRED)
   find_package(Lapack REQUIRED)
   include_directories(${FFTW3_INCLUDE_DIRS})
@@ -796,11 +789,22 @@ if(ENABLE_RAPIDJSON)
   target_link_libraries(${ABACUS_BIN_NAME} json_output)
 endif()
 
+if (USE_SW)
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswfft.a)
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswscalapack.a)
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswlapack.a)
+  target_link_libraries(${ABACUS_BIN_NAME} ${SW_MATH}/libswblas.a)
+  list(APPEND math_libs gfortran)
+endif()
+
+list(APPEND math_libs m)
+target_link_libraries(${ABACUS_BIN_NAME} ${math_libs})
+
 install(PROGRAMS ${ABACUS_BIN_PATH}
         TYPE BIN
         # DESTINATION ${CMAKE_INSTALL_BINDIR}
 )
 
 if(ENABLE_COVERAGE)
   coverage_evaluate()
-endif()
+endif()

Dockerfile.intel

@@ -2,7 +2,8 @@ FROM ubuntu:22.04
 
 RUN apt-get update && apt-get install -y \
     bc cmake git gnupg gcc g++ python3-numpy sudo wget vim unzip \
-    libcereal-dev libxc-dev libgtest-dev libgmock-dev libbenchmark-dev
+    libcereal-dev libxc-dev libgtest-dev libgmock-dev libbenchmark-dev \
+    pkg-config build-essential autoconf automake libtool
 
 # Following steps by https://software.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/apt.html .
 RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
@@ -16,21 +17,44 @@ RUN apt-get update && \
         intel-oneapi-compiler-dpcpp-cpp \
         intel-oneapi-compiler-fortran \
         intel-oneapi-mkl-devel \
-        intel-oneapi-mpi-devel="2021.11.*" \
+        intel-oneapi-mkl-sycl-devel \
+        intel-oneapi-mkl-sycl-distributed-dft-devel \
+        intel-oneapi-mpi-devel \
         intel-oneapi-vtune && \
-        rm /opt/intel/oneapi/mpi/latest && ln -s /opt/intel/oneapi/mpi/2021.11 /opt/intel/oneapi/mpi/latest
-ENV I_MPI_ROOT=/opt/intel/oneapi/mpi/latest \
-    LIBRARY_PATH=/opt/intel/oneapi/tbb/latest/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/mkl/latest/lib/:/opt/intel/oneapi/ippcp/latest/lib/:/opt/intel/oneapi/ipp/latest/lib:/opt/intel/oneapi/dpl/latest/lib:/opt/intel/oneapi/dnnl/latest/lib:/opt/intel/oneapi/dal/latest/lib:/opt/intel/oneapi/compiler/latest/lib:/opt/intel/oneapi/ccl/latest/lib/ \
-    LD_LIBRARY_PATH=/opt/intel/oneapi/tbb/latest/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib:/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/mkl/latest/lib:/opt/intel/oneapi/itac/latest/slib:/opt/intel/oneapi/ippcp/latest/lib/:/opt/intel/oneapi/ipp/latest/lib:/opt/intel/oneapi/dpl/latest/lib:/opt/intel/oneapi/dnnl/latest/lib:/opt/intel/oneapi/debugger/latest/opt/debugger/lib:/opt/intel/oneapi/dal/latest/lib:/opt/intel/oneapi/compiler/latest/opt/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/latest/opt/compiler/lib:/opt/intel/oneapi/compiler/latest/lib:/opt/intel/oneapi/ccl/latest/lib/ \
-    PATH=/opt/intel/oneapi/vtune/latest/bin64:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/bin:/opt/intel/oneapi/mpi/latest/bin:/opt/intel/oneapi/mkl/latest/bin/:/opt/intel/oneapi/itac/latest/bin:/opt/intel/oneapi/inspector/latest/bin64:/opt/intel/oneapi/dpcpp-ct/latest/bin:/opt/intel/oneapi/dev-utilities/latest/bin:/opt/intel/oneapi/debugger/latest/opt/debugger/bin:/opt/intel/oneapi/compiler/latest/opt/oclfpga/bin:/opt/intel/oneapi/compiler/latest/bin:/opt/intel/oneapi/advisor/latest/bin64:/opt/mamba/bin:/opt/mamba/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
-    MKLROOT=/opt/intel/oneapi/mkl/latest \
-    FI_PROVIDER_PATH=/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib/prov:/usr/lib/x86_64-linux-gnu/libfabric \
-    CMAKE_PREFIX_PATH=/opt/intel/oneapi/tbb/latest/env/..:/opt/intel/oneapi/mkl/latest/lib/cmake:/opt/intel/oneapi/ipp/latest/lib/cmake/ipp:/opt/intel/oneapi/dpl/latest/lib/cmake/oneDPL:/opt/intel/oneapi/dnnl/latest/lib/cmake:/opt/intel/oneapi/dal/latest:/opt/intel/oneapi/compiler/latest \
-    CMPLR_ROOT=/opt/intel/oneapi/compiler/latest
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ls -la /opt/intel/oneapi/mkl/latest/lib/intel64/*sycl*dft*
+RUN ls -la /opt/intel/oneapi/mkl/latest/lib/intel64/*dis*
+
+# Set oneAPI environment variables
+ENV ONEAPI_ROOT=/opt/intel/oneapi
+ENV I_MPI_ROOT=${ONEAPI_ROOT}/mpi/latest
+ENV MKLROOT=${ONEAPI_ROOT}/mkl/latest
+ENV CMPLR_ROOT=${ONEAPI_ROOT}/compiler/latest
+
+# Set library paths and include paths
+ENV LIBRARY_PATH=${ONEAPI_ROOT}/tbb/latest/lib/intel64/gcc4.8:${ONEAPI_ROOT}/mpi/latest/lib:${MKLROOT}/lib/intel64:${ONEAPI_ROOT}/compiler/latest/lib/:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=${ONEAPI_ROOT}/tbb/latest/lib/intel64/gcc4.8:${ONEAPI_ROOT}/mpi/latest/lib:${MKLROOT}/lib/intel64:${ONEAPI_ROOT}/compiler/latest/lib/:${LD_LIBRARY_PATH}
+ENV PATH=${ONEAPI_ROOT}/vtune/latest/bin64:${ONEAPI_ROOT}/mpi/latest/bin:${MKLROOT}/bin/intel64:${ONEAPI_ROOT}/compiler/latest/bin:${PATH}
+ENV CPATH=${MKLROOT}/include:${ONEAPI_ROOT}/mpi/latest/include:${CPATH}
+ENV PKG_CONFIG_PATH=${MKLROOT}/lib/pkgconfig:${ONEAPI_ROOT}/mpi/latest/lib/pkgconfig:${PKG_CONFIG_PATH}
+
+# Set CMAKE related paths
+ENV CMAKE_PREFIX_PATH=${ONEAPI_ROOT}/tbb/latest:${MKLROOT}/lib/cmake:${ONEAPI_ROOT}/dpl/latest/lib/cmake:${ONEAPI_ROOT}/dnnl/latest/lib/cmake:${ONEAPI_ROOT}/dal/latest:${ONEAPI_ROOT}/compiler/latest:${CMAKE_PREFIX_PATH}
 
 SHELL ["/bin/bash", "-c"]
 ENV CC=mpiicx CXX=mpiicpx FC=mpiifx
 
+# Verify oneAPI installation
+RUN source ${ONEAPI_ROOT}/setvars.sh && \
+    echo "=== Verify compiler ===" && \
+    which mpiicx && mpiicx --version && \
+    echo "=== Verify MKL ===" && \
+    ls ${MKLROOT}/lib/intel64/ && \
+    echo "=== Verify MPI ===" && \
+    which mpirun && mpirun --version
+
 # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html
 RUN source /opt/intel/oneapi/setvars.sh && \
     cd /tmp && \
@@ -39,28 +63,34 @@ RUN source /opt/intel/oneapi/setvars.sh && \
     tar xzf elpa-$ELPA_VER.tar.gz  && rm elpa-$ELPA_VER.tar.gz && \
     cd elpa-$ELPA_VER && mkdir build && cd build && \
     ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \
-    make -j`nproc` && \
+    make -j$(nproc) && \
     make PREFIX=/usr/local install && \
     ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \
     cd /tmp && rm -rf elpa-$ELPA_VER
 
-RUN cd /tmp && git clone https://github.com/Tencent/rapidjson.git && cp -r rapidjson/include/rapidjson /usr/include/ \
-    && rm -rf rapidjson
-
-RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip \
-        --no-check-certificate --quiet -O libtorch.zip && \
-    unzip -q libtorch.zip -d /opt  && rm libtorch.zip
-
-ENV CMAKE_PREFIX_PATH=/opt/libtorch/share/cmake
+# rapidjson and libtorch
+RUN cd /tmp && git clone --depth 1 https://github.com/Tencent/rapidjson.git && cp -r rapidjson/include/rapidjson /usr/include/ && rm -rf rapidjson
+RUN wget -q https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip -O /tmp/libtorch.zip && \
+    unzip -q /tmp/libtorch.zip -d /opt && rm -f /tmp/libtorch.zip
+ENV CMAKE_PREFIX_PATH=/opt/libtorch/share/cmake:${CMAKE_PREFIX_PATH}
 
+# Clone and build abacus (optional during image build; keep for CI image)
 ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/develop /dev/null
-
-RUN source /opt/intel/oneapi/setvars.sh && \
-    git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \
+RUN export LD_LIBRARY_PATH=${GKLIB_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_ROOT}/lib:${SUPERLU32_DIST_ROOT}/lib:${PEXSI32_ROOT}/lib:${LD_LIBRARY_PATH} && \
+    export CPATH=${GKLIB_ROOT}/include:${METIS32_ROOT}/include:${PARMETIS32_ROOT}/include:${SUPERLU32_DIST_ROOT}/include:${PEXSI32_ROOT}/include:${CPATH} && \
+    cd /tmp && git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \
     cd abacus-develop && \
-    cmake -B build -DENABLE_MLALGO=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON -DENABLE_RAPIDJSON=ON && \
-    cmake --build build -j`nproc` && \
+    cmake -B build \
+        -DENABLE_MLALGO=ON \
+        -DENABLE_LIBXC=ON \
+        -DENABLE_LIBRI=ON \
+        -DENABLE_RAPIDJSON=ON \
+        -DCMAKE_BUILD_TYPE=Release && \
+    cmake --build build -j"$(nproc)" && \
     cmake --install build && \
-    rm -rf build && \
-    abacus --version
-    #&& rm -rf abacus-develop
+    (/usr/local/bin/abacus --version || echo "ABACUS installed but version check failed") && \
+    rm -rf /tmp/abacus-develop
+        # -DMKL_SYCL=OFF \
+        # -DMKL_SYCL_DISTRIBUTED_DFT:BOOL=OFF \
+# Default entry
+CMD ["/bin/bash"]

cmake/FindMKL.cmake

@@ -4,7 +4,7 @@
 #  MKL_INCLUDE - where to find mkl.h, etc.
 #  MKL_FOUND        - True if mkl found.
 
-find_package(MKL NO_MODULE) # try using official module first
+# find_package(MKL NO_MODULE) # try using official module first
 if(NOT TARGET MKL::MKL)
 
 find_path(MKL_INCLUDE mkl_service.h HINTS ${MKLROOT}/include)
@@ -70,7 +70,9 @@ if(MKL_FOUND)
     set_property(TARGET MKL::MKL PROPERTY
     INTERFACE_LINK_LIBRARIES
     "-Wl,--start-group"
-    MKL::INTEL MKL::INTEL_THREAD MKL::CORE)
+    MKL::INTEL MKL::INTEL_THREAD MKL::CORE
+    "-Wl,--end-group"
+    )
   endif()
 endif()
 

source/source_base/test_parallel/CMakeLists.txt

@@ -67,7 +67,7 @@ add_test(NAME MODULE_BASE_parallel_2d_test_para
 
   # figure out the lib that provides BLACS
   if(MKLROOT)
-    list(APPEND BLACS_LIB MKL::MKL MKL::MKL_SCALAPACK)
+    list(APPEND BLACS_LIB MKL::MKL)
   else()
     set(BLACS_LIB ScaLAPACK::ScaLAPACK)
   endif()