[Toolchain] pack-run and package url (#6051)

* implement pack-run function

* add dry-run and pack-run readme

* give back testing option

* make added global var uppercase

* update downloading url

* add pack-run in intel/amd

* fix link of github packages

Author: QuantumMisaka

toolchain/README.md

@@ -148,7 +148,11 @@ by running these scripts after loading `gcc` or `intel-mkl-mpi`
 environment. 
 
 The toolchain installation process can be interrupted at anytime.
-just re-run *toolchain_\*.sh*, toolchain itself may fix it
+just re-run *toolchain_\*.sh*, toolchain itself may fix it. If you encouter some problem, you can always remove some package in the interrupted points and re-run the toolchain.
+
+Some useful options:
+- `--dry-run`: just run the main install scripts for environment setting, without any package downloading or installation.
+- `--pack-run`: just run the install scripts without any package building, which helps user to download and check the packages, paticularly for offline installation to a server.
 
 If compliation is successful, a message will be shown like this:
 

toolchain/install_abacus_toolchain.sh

@@ -115,13 +115,14 @@ OPTIONS:
 --target-cpu              Compile for the specified target CPU (e.g. haswell or generic), i.e.
                           do not optimize for the actual host system which is the default (native)
 --dry-run                 Write only config files, but don't actually build packages.
+--pack-run                Only check and install required packages without actually unpack and build packages 
 
 The --enable-FEATURE options follow the rules:
   --enable-FEATURE=yes    Enable this particular feature
   --enable-FEATURE=no     Disable this particular feature
   --enable-FEATURE        The option keyword alone is equivalent to
                           --enable-FEATURE=yes
-  ===== NOTICE: THESE FEATURE AER NOT INCLUDED IN ABACUS =====
+  ===== NOTICE: THESE GPU FEATURE IS ON TESTING =====
   --enable-cuda           Turn on GPU (CUDA) support (can be combined
                           with --enable-opencl).
                           Default = no
@@ -329,10 +330,11 @@ export intel_classic="no"
 # but icx is recommended by intel compiler
 # option: --with-intel-classic can change it to yes/no
 # QuantumMisaka by 2023.08
-export intelmpi_classic="no"
-export with_ifx="yes" # whether ifx is used in oneapi
-export with_flang="no" # whether flang is used in aocc
-export openmpi_4th="no" # whether openmpi downgrade
+export PACK_RUN="__FALSE__"
+export INTELMPI_CLASSIC="no"
+export WITH_IFX="yes" # whether ifx is used in oneapi
+export WITH_FLANG="no" # whether flang is used in aocc
+export OPENMPI_4TH="no" # whether openmpi downgrade
 export GPUVER="no"
 export MPICH_DEVICE="ch4"
 export TARGET_CPU="native"
@@ -461,6 +463,9 @@ while [ $# -ge 1 ]; do
     --dry-run)
       dry_run="__TRUE__"
       ;;
+    --pack-run)
+      PACK_RUN="__TRUE__"
+      ;;
     --enable-tsan*)
       enable_tsan=$(read_enable $1)
       if [ "${enable_tsan}" = "__INVALID__" ]; then
@@ -514,7 +519,7 @@ while [ $# -ge 1 ]; do
       fi
       ;;
     --with-4th-openmpi*)
-      openmpi_4th=$(read_with "${1}" "no") # default new openmpi
+      OPENMPI_4TH=$(read_with "${1}" "no") # default new openmpi
       ;;
     --with-openmpi*)
       with_openmpi=$(read_with "${1}")
@@ -532,19 +537,19 @@ while [ $# -ge 1 ]; do
       intel_classic=$(read_with "${1}" "no") # default new intel compiler
       ;;
     --with-intel-mpi-clas*)
-      intelmpi_classic=$(read_with "${1}" "no") # default new intel mpi compiler
+      INTELMPI_CLASSIC=$(read_with "${1}" "no") # default new intel mpi compiler
       ;;
     --with-intel*)  # must be read after items above
       with_intel=$(read_with "${1}" "__SYSTEM__")
       ;;
     --with-ifx*)
-      with_ifx=$(read_with "${1}" "yes") # default yes
+      WITH_IFX=$(read_with "${1}" "yes") # default yes
       ;;
     --with-amd*)
       with_amd=$(read_with "${1}" "__SYSTEM__")
       ;;
     --with-flang*)
-      with_flang=$(read_with "${1}" "no")
+      WITH_FLANG=$(read_with "${1}" "no")
       ;;
     --with-aocl*)
       with_aocl=$(read_with "${1}" "__SYSTEM__")
@@ -840,7 +845,6 @@ else
   ./scripts/stage2/install_stage2.sh
   ./scripts/stage3/install_stage3.sh
   ./scripts/stage4/install_stage4.sh
-fi
 
 cat << EOF
 ========================== usage =========================
@@ -858,5 +862,6 @@ or you can modify the builder scripts to suit your needs.
 """
 EOF
 
+fi
 
 #EOF

toolchain/scripts/stage0/install_amd.sh

@@ -27,18 +27,24 @@ case "${with_amd}" in
   __INSTALL__)
     echo "==================== Installing the AMD compiler ======================"
     echo "__INSTALL__ is not supported; please install the AMD compiler manually"
-    exit 1
+    if [ "${PACK_RUN}" != "__TRUE__" ]; then
+        exit 1
+    fi
     ;;
   __SYSTEM__)
     echo "==================== Finding AMD compiler from system paths ===================="
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+        echo "--pack-run mode specified, skip system check"
+    else
     check_command clang "amd" && CC="$(realpath $(command -v clang))" || exit 1
     check_command clang++ "amd" && CXX="$(realpath $(command -v clang++))" || exit 1
-    if [ "${with_flang}" = "yes" ]; then
+    if [ "${WITH_FLANG}" = "yes" ]; then
         check_command flang "amd" && FC="$(realpath $(command -v flang))" || exit 1
     else
         check_command gfortran "gcc" && FC="gfortran" || exit 1
         add_lib_from_paths GCC_LDFLAGS "libgfortran.*" ${LIB_PATHS}
     fi
+    fi
     F90="${FC}"
     F77="${FC}"
     ;;
@@ -53,7 +59,7 @@ case "${with_amd}" in
     check_dir "${pkg_install_dir}/include"
     check_command ${pkg_install_dir}/bin/clang "amd" && CC="${pkg_install_dir}/bin/clang" || exit 1
     check_command ${pkg_install_dir}/bin/clang++ "amd" && CXX="${pkg_install_dir}/bin/clang++" || exit 1
-    if [ "${with_flang}" = "yes" ]; then
+    if [ "${WITH_FLANG}" = "yes" ]; then
         check_command ${pkg_install_dir}/bin/flang "amd" && FC="${pkg_install_dir}/bin/flang" || exit 1
     else
         check_command gfortran "gcc" && FC="$(command -v gfortran)" || exit 1
@@ -70,7 +76,7 @@ if [ "${with_amd}" != "__DONTUSE__" ]; then
   echo "CXX is ${CXX}"
   [ $(realpath $(command -v clang++) | grep -e aocc-compiler) ] || echo "Check the AMD C++ compiler path"
   echo "FC  is ${FC}"
-  if [ "${with_flang}" = "yes" ]; then
+  if [ "${WITH_FLANG}" = "yes" ]; then
     [ $(realpath $(command -v flang) | grep -e aocc-compiler) ] || echo "Check the AMD Fortran compiler path"
   else
     [ $(realpath $(command -v gfortran) | grep -e aocc-compiler) ] || echo "Check the GNU Fortran compiler path"

toolchain/scripts/stage0/install_cmake.sh

@@ -43,12 +43,18 @@ case "${with_cmake}" in
       if [ -f $cmake_pkg ]; then
         echo "$cmake_pkg is found"
       else
-        download_pkg_from_ABACUS_org "${cmake_sha256}" "$cmake_pkg"
+        #download_pkg_from_ABACUS_org "${cmake_sha256}" "$cmake_pkg"
+        url="https://cmake.org/files/v${cmake_ver%.*}/${cmake_pkg}"
+        download_pkg_from_url "${cmake_sha256}" "${cmake_pkg}" "${url}"
+      fi
+      if [ "${PACK_RUN}" = "__TRUE__" ]; then
+        echo "--pack-run mode specified, skip installation"
+      else
+        echo "Installing from scratch into ${pkg_install_dir}"
+        mkdir -p ${pkg_install_dir}
+        /bin/sh $cmake_pkg --prefix=${pkg_install_dir} --skip-license > install.log 2>&1 || tail -n ${LOG_LINES} install.log
+        write_checksums "${install_lock_file}" "${SCRIPT_DIR}/stage0/$(basename ${SCRIPT_NAME})"
       fi
-      echo "Installing from scratch into ${pkg_install_dir}"
-      mkdir -p ${pkg_install_dir}
-      /bin/sh $cmake_pkg --prefix=${pkg_install_dir} --skip-license > install.log 2>&1 || tail -n ${LOG_LINES} install.log
-      write_checksums "${install_lock_file}" "${SCRIPT_DIR}/stage0/$(basename ${SCRIPT_NAME})"
     fi
     ;;
   __SYSTEM__)

toolchain/scripts/stage0/install_gcc.sh

@@ -40,8 +40,13 @@ case "${with_gcc}" in
       if [ -f gcc-${gcc_ver}.tar.gz ]; then
         echo "gcc-${gcc_ver}.tar.gz is found"
       else
-        download_pkg_from_ABACUS_org "${gcc_sha256}" "gcc-${gcc_ver}.tar.gz"
+        #download_pkg_from_ABACUS_org "${gcc_sha256}" "gcc-${gcc_ver}.tar.gz"
+        url=https://mirrors.tuna.tsinghua.edu.cn/gnu/gcc/gcc-${gcc_ver}/gcc-${gcc_ver}.tar.gz
+        download_pkg_from_url "${gcc_sha256}" "gcc-${gcc_ver}.tar.gz" "${url}"
       fi
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+      echo "--pack-run mode specified, skip installation"
+    else
       [ -d gcc-${gcc_ver} ] && rm -rf gcc-${gcc_ver}
       tar -xzf gcc-${gcc_ver}.tar.gz
 
@@ -115,13 +120,18 @@ case "${with_gcc}" in
       cd ../..
       write_checksums "${install_lock_file}" "${SCRIPT_DIR}/stage0/$(basename ${SCRIPT_NAME})"
     fi
-    check_install ${pkg_install_dir}/bin/gcc "gcc" && CC="${pkg_install_dir}/bin/gcc" || exit 1
-    check_install ${pkg_install_dir}/bin/g++ "gcc" && CXX="${pkg_install_dir}/bin/g++" || exit 1
-    check_install ${pkg_install_dir}/bin/gfortran "gcc" && FC="${pkg_install_dir}/bin/gfortran" || exit 1
-    F90="${FC}"
-    F77="${FC}"
-    GCC_CFLAGS="-I'${pkg_install_dir}/include'"
-    GCC_LDFLAGS="-L'${pkg_install_dir}/lib64' -L'${pkg_install_dir}/lib' -Wl,-rpath,'${pkg_install_dir}/lib64' -Wl,-rpath,'${pkg_install_dir}/lib64'"
+    fi
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+        echo "--pack-run mode specified, skip system check"
+    else
+        check_install ${pkg_install_dir}/bin/gcc "gcc" && CC="${pkg_install_dir}/bin/gcc" || exit 1
+        check_install ${pkg_install_dir}/bin/g++ "gcc" && CXX="${pkg_install_dir}/bin/g++" || exit 1
+        check_install ${pkg_install_dir}/bin/gfortran "gcc" && FC="${pkg_install_dir}/bin/gfortran" || exit 1
+        F90="${FC}"
+        F77="${FC}"
+        GCC_CFLAGS="-I'${pkg_install_dir}/include'"
+        GCC_LDFLAGS="-L'${pkg_install_dir}/lib64' -L'${pkg_install_dir}/lib' -Wl,-rpath,'${pkg_install_dir}/lib64' -Wl,-rpath,'${pkg_install_dir}/lib64'"
+    fi
     ;;
   __SYSTEM__)
     echo "==================== Finding GCC from system paths ===================="

toolchain/scripts/stage0/install_intel.sh

@@ -26,23 +26,30 @@ case "${with_intel}" in
   __INSTALL__)
     echo "==================== Installing the Intel compiler ===================="
     echo "__INSTALL__ is not supported; please install the Intel compiler manually"
+    if [ "${PACK_RUN}" != "__TRUE__" ]; then
+        exit 1
+    fi
     exit 1
     ;;
   __SYSTEM__)
     echo "==================== Finding Intel compiler from system paths ===================="
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+        echo "--pack-run mode specified, skip system check"
+    else
     if [ "${intel_classic}" = "yes" ]; then
       check_command icc "intel" && CC="$(realpath $(command -v icc))" || exit 1
       check_command icpc "intel" && CXX="$(realpath $(command -v icpc))" || exit 1
       check_command ifort "intel" && FC="$(realpath $(command -v ifort))" || exit 1
     else
       check_command icx "intel" && CC="$(realpath $(command -v icx))" || exit 1
       check_command icpx "intel" && CXX="$(realpath $(command -v icpx))" || exit 1
-      if [ "${with_ifx}" = "yes" ]; then
+      if [ "${WITH_IFX}" = "yes" ]; then
         check_command ifx "intel" && FC="$(realpath $(command -v ifx))" || exit 1
       else
         check_command ifort "intel" && FC="$(realpath $(command -v ifort))" || exit 1
       fi
     fi
+    fi
     F90="${FC}"
     F77="${FC}"
     ;;
@@ -62,7 +69,7 @@ case "${with_intel}" in
     else
       check_command ${pkg_install_dir}/bin/icx "intel" && CC="${pkg_install_dir}/bin/icx" || exit 1
       check_command ${pkg_install_dir}/bin/icpx "intel" && CXX="${pkg_install_dir}/bin/icpx" || exit 1
-      if [ "${with_ifx}" = "yes" ]; then
+      if [ "${WITH_IFX}" = "yes" ]; then
         check_command ${pkg_install_dir}/bin/ifx "intel" && FC="${pkg_install_dir}/bin/ifx" || exit 1
       else
         check_command ${pkg_install_dir}/bin/ifort "intel" && FC="${pkg_install_dir}/bin/ifort" || exit 1

toolchain/scripts/stage1/install_intelmpi.sh

@@ -27,24 +27,29 @@ case "${with_intelmpi}" in
   __INSTALL__)
     echo "==================== Installing Intel MPI ===================="
     echo '__INSTALL__ is not supported; please manually install Intel MPI'
-    exit 1
+    if [ "${PACK_RUN}" != "__TRUE__" ]; then
+        exit 1
+    fi
     ;;
   __SYSTEM__)
     echo "==================== Finding Intel MPI from system paths ===================="
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+        echo "--pack-run mode specified, skip system check"
+    else
     check_command mpiexec "intelmpi" && MPIRUN="$(realpath $(command -v mpiexec))"
     if [ "${intel_classic}" = "yes" ]; then
     # if intel compiler used as classic, so as intelmpi
-        export intelmpi_classic="yes"
+        export INTELMPI_CLASSIC="yes"
     fi
     if [ "${with_intel}" != "__DONTUSE__" ]; then
-        if [ "${intelmpi_classic}" = "yes" ]; then
+        if [ "${INTELMPI_CLASSIC}" = "yes" ]; then
             check_command mpiicc "intelmpi" && MPICC="$(realpath $(command -v mpiicc))" || exit 1
             check_command mpiicpc "intelmpi" && MPICXX="$(realpath $(command -v mpiicpc))" || exit 1
             check_command mpiifort "intelmpi" && MPIFC="$(realpath $(command -v mpiifort))" || exit 1
         else
             check_command mpiicx "intelmpi" && MPICC="$(realpath $(command -v mpiicx))" || exit 1
             check_command mpiicpx "intelmpi" && MPICXX="$(realpath $(command -v mpiicpx))" || exit 1
-            if [ "${with_ifx}" == "yes" ]; then
+            if [ "${WITH_IFX}" == "yes" ]; then
                 check_command mpiifx "intelmpi" && MPIFC="$(realpath $(command -v mpiifx))" || exit 1
             else
                 check_command mpiifort "intelmpi" && MPIFC="$(realpath $(command -v mpiifort))" || exit 1
@@ -61,6 +66,7 @@ case "${with_intelmpi}" in
     add_lib_from_paths INTELMPI_LDFLAGS "libmpi.*" $LIB_PATHS
     check_lib -lmpi "intelmpi"
     check_lib -lmpicxx "intelmpi"
+    fi
     ;;
   __DONTUSE__)
     # Nothing to do
@@ -73,14 +79,14 @@ case "${with_intelmpi}" in
     check_dir "${pkg_install_dir}/include"
     check_command ${pkg_install_dir}/bin/mpiexec "intel" && MPIRUN="${pkg_install_dir}/bin/mpiexec" || exit 1
     if [ "${with_intel}" != "__DONTUSE__" ]; then
-        if [ "${intelmpi_classic}" = "yes" ]; then
+        if [ "${INTELMPI_CLASSIC}" = "yes" ]; then
             check_command ${pkg_install_dir}/bin/mpiicc "intel" && MPICC="${pkg_install_dir}/bin/mpiicc" || exit 1
             check_command ${pkg_install_dir}/bin/mpiicpc "intel" && MPICXX="${pkg_install_dir}/bin/mpiicpc" || exit 1
             check_command ${pkg_install_dir}/bin/mpiifort "intel" && MPIFC="${pkg_install_dir}/bin/mpiifort" || exit 1
         else
             check_command ${pkg_install_dir}/bin/mpiicx "intel" && MPICC="${pkg_install_dir}/bin/mpiicx" || exit 1
             check_command ${pkg_install_dir}/bin/mpiicpx "intel" && MPICXX="${pkg_install_dir}/bin/mpiicpx" || exit 1
-            if [ "${with_ifx}" = "yes" ]; then
+            if [ "${WITH_IFX}" = "yes" ]; then
                 check_command ${pkg_install_dir}/bin/mpiifx "intel" && MPIFC="${pkg_install_dir}/bin/mpiifx" || exit 1
             else
                 check_command ${pkg_install_dir}/bin/mpiifort "intel" && MPIFC="${pkg_install_dir}/bin/mpiifort" || exit 1
@@ -105,7 +111,7 @@ if [ "${with_intelmpi}" != "__DONTUSE__" ]; then
   else
     I_MPI_CXX="icpx"
     I_MPI_CC="icx"
-    if [ "${with_ifx}" = "yes" ]; then
+    if [ "${WITH_IFX}" = "yes" ]; then
       I_MPI_FC="ifx"
     else
       I_MPI_FC="ifort"

toolchain/scripts/stage1/install_mpich.sh

@@ -47,6 +47,9 @@ case "${with_mpich}" in
         #download_pkg_from_ABACUS_org "${mpich_sha256}" "${mpich_pkg}"
         download_pkg_from_url "${mpich_sha256}" "${mpich_pkg}" "${url}"
       fi
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+      echo "--pack-run mode specified, skip installation"
+    else
       echo "Installing from scratch into ${pkg_install_dir} for MPICH device ${MPICH_DEVICE}"
       [ -d mpich-${mpich_ver} ] && rm -rf mpich-${mpich_ver}
       tar -xzf ${mpich_pkg}
@@ -72,6 +75,10 @@ case "${with_mpich}" in
       cd ..
       write_checksums "${install_lock_file}" "${SCRIPT_DIR}/stage1/$(basename ${SCRIPT_NAME})"
     fi
+    fi
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+        echo "--pack-run mode specified, skip system check"
+    else
     check_dir "${pkg_install_dir}/bin"
     check_dir "${pkg_install_dir}/lib"
     check_dir "${pkg_install_dir}/include"
@@ -83,6 +90,7 @@ case "${with_mpich}" in
     MPIF77="${MPIFC}"
     MPICH_CFLAGS="-I'${pkg_install_dir}/include'"
     MPICH_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath,'${pkg_install_dir}/lib'"
+    fi
     ;;
   __SYSTEM__)
     echo "==================== Finding MPICH from system paths ===================="

toolchain/scripts/stage1/install_openmpi.sh

@@ -10,7 +10,7 @@
 [ "${BASH_SOURCE[0]}" ] && SCRIPT_NAME="${BASH_SOURCE[0]}" || SCRIPT_NAME=$0
 SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_NAME")/.." && pwd -P)"
 
-if [ "${openmpi_4th}" = "yes" ]; then
+if [ "${OPENMPI_4TH}" = "yes" ]; then
     openmpi_ver="4.1.6"
     openmpi_sha256="f740994485516deb63b5311af122c265179f5328a0d857a567b85db00b11e415"
 else
@@ -40,7 +40,7 @@ case "${with_openmpi}" in
     pkg_install_dir="${INSTALLDIR}/openmpi-${openmpi_ver}"
     #pkg_install_dir="${HOME}/apps/openmpi/${openmpi_ver}-gcc8"
     install_lock_file="$pkg_install_dir/install_successful"
-    url="https://download.open-mpi.org/release/open-mpi/v${openmpi_ver:0:3}/${openmpi_pkg}"
+    url="https://download.open-mpi.org/release/open-mpi/v${openmpi_ver%.*}/${openmpi_pkg}"
     if verify_checksums "${install_lock_file}"; then
       echo "openmpi-${openmpi_ver} is already installed, skipping it."
     else
@@ -49,6 +49,9 @@ case "${with_openmpi}" in
       else
         download_pkg_from_url "${openmpi_sha256}" "${openmpi_pkg}" "${url}"
       fi
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+      echo "--pack-run mode specified, skip installation"
+    else
       echo "Installing from scratch into ${pkg_install_dir}"
       [ -d openmpi-${openmpi_ver} ] && rm -rf openmpi-${openmpi_ver}
       tar -xjf ${openmpi_pkg}
@@ -87,6 +90,10 @@ case "${with_openmpi}" in
       cd ..
       write_checksums "${install_lock_file}" "${SCRIPT_DIR}/stage1/$(basename ${SCRIPT_NAME})"
     fi
+    fi
+    if [ "${PACK_RUN}" = "__TRUE__" ]; then
+        echo "--pack-run mode specified, skip system check"
+    else
     check_dir "${pkg_install_dir}/bin"
     check_dir "${pkg_install_dir}/lib"
     check_dir "${pkg_install_dir}/include"
@@ -98,6 +105,7 @@ case "${with_openmpi}" in
     MPIF77="${MPIFC}"
     OPENMPI_CFLAGS="-I'${pkg_install_dir}/include'"
     OPENMPI_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath,'${pkg_install_dir}/lib'"
+    fi
     ;;
   __SYSTEM__)
     echo "==================== Finding OpenMPI from system paths ===================="