CMake support

Author: Mathieu Taillefumier

.gitignore

@@ -18,7 +18,6 @@ tags*
 hmc_tm
 invert
 offline_measurement
-lib/
 benchmark
 *.data
 *.para

CMakeLists.txt

@@ -0,0 +1,329 @@
+cmake_minimum_required(VERSION 3.24)
+
+project(
+  tmlqcd
+  DESCRIPTION "tmlQCD"
+  HOMEPAGE_URL "http://www.itkp.uni-bonn.de/~urbach/software.html"
+  VERSION "6.0.0"
+  LANGUAGES C CXX)
+
+# include our cmake snippets
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+
+# =================================================================================================
+# REQUIRE OUT-OF-SOURCE BUILDS
+file(TO_CMAKE_PATH "${PROJECT_BINARY_DIR}/CMakeLists.txt" LOC_PATH)
+if(EXISTS "${LOC_PATH}")
+  message(
+    FATAL_ERROR
+      "You cannot build in a source directory (or any directory with a CMakeLists.txt file). Please make a build subdirectory."
+  )
+endif()
+
+# =================================================================================================
+# PROJECT AND VERSION
+include(CMakeDependentOption)
+include(CheckSymbolExists)
+include(CheckLibraryExists)
+include(CheckFunctionExists)
+include(GNUInstallDirs)
+
+cmake_policy(SET CMP0048 NEW)
+
+if(POLICY CMP0144)
+  cmake_policy(SET CMP0144 NEW)
+endif()
+
+if(NOT DEFINED CMAKE_CUDA_STANDARD)
+  set(CMAKE_CUDA_STANDARD 14)
+  set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+endif()
+
+if(NOT DEFINED CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 17)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+endif()
+
+if(NOT DEFINED CMAKE_C_STANDARD)
+  set(CMAKE_C_STANDARD 11)
+  set(CMAKE_C_STANDARD_REQUIRED ON)
+endif()
+
+if(NOT DEFINED CMAKE_HIP_STANDARD)
+  set(CMAKE_HIP_STANDARD 14)
+  set(CMAKE_HIP_STANDARD_REQUIRED ON)
+endif()
+
+find_package(PkgConfig)
+
+# ##############################################################################
+# Define the paths for static libraries and executables
+# ##############################################################################
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY
+    ${cp2k_BINARY_DIR}/lib
+    CACHE PATH "Single output directory for building all libraries.")
+
+# Search for rocm in common locations
+foreach(__var ROCM_ROOT CRAY_ROCM_ROOT ORNL_ROCM_ROOT CRAY_ROCM_PREFIX
+              ROCM_PREFIX CRAY_ROCM_DIR)
+  if($ENV{${__var}})
+    list(APPEND CMAKE_PREFIX_PATH $ENV{__var})
+    set(ROCM_PATH
+        $ENV{__var}
+        CACHE PATH "Path to ROCm installation")
+  endif()
+endforeach()
+
+option(CMAKE_POSITION_INDEPENDENT_CODE "Enable position independent code" ON)
+
+option(TMLQCD_USE_FFTW "Enable fftw support" OFF)
+option(TMLQCD_USE_MPI "Enable MPI support" OFF)
+option(TMLQCD_USE_CUDA "Enable QUDA support" OFF)
+option(TMLQCD_USE_HIP "Enable HIP support" OFF)
+option(TMLQCD_USE_DDALPHAAMG "Enable DDalphaAMG support" OFF)
+option(TMLCDQ_USE_OPENMP "Enable openMP" ON)
+option(TMLQCD_FIXED_VOLUME "fix volume at compile time" OFF)
+option(
+  TMLQCD_ENABLE_ALIGNMENT
+  "Automatically or expliclty align arrays to byte number. auto, none, 16, 32, 64"
+  "auto")
+option(TMLQCD_BGL_DRAM "use BGL dram window (BGL only!)" ON)
+option(TMLQCD_USE_OPTIMIZATION "enable optimisation" ON)
+option(TMLQCD_USE_GAUGE_COPY "Enable use of a copy of the gauge field" ON)
+option(TMLQCD_USE_HALFSPINOR "Use a Dirac Op. with halfspinor exchange" ON)
+option(TMLQCD_USE_TSPLITPAR "Enable timeslice-splitted communications" ON)
+option(TMLQCD_USE_QPHIX "enable QPhiX" OFF)
+option(TMLQCD_USE_SHMEM "Use shmem API" OFF)
+option(TMLQCD_USE_QUDA "Enable QUDA support" OFF)
+# MPI dependent options
+cmake_dependent_option(
+  TMLQCD_PERSISTENT_MPI "Use persistent MPI calls for halfspinor [default=no]"
+  OFF "TMLQCD_USE_MPI" OFF)
+cmake_dependent_option(
+  TMLQCD_NONBLOCKING_MPI "Use non-blocking MPI calls for spinor and gaug" ON
+  "TMLQCD_USE_MPI" OFF)
+
+# need to do it properly. Just a place holder
+cmake_dependent_option(
+  TMLQCD_MPI_DIMENSION "Use n dimensional parallelisation [default=4]" 4
+  "TMLQCD_USE_MPI" OFF)
+
+# HIP dependent options
+cmake_dependent_option(TMLQCD_USE_CUDA_HIP "Enable CUDA support in HIP" OFF
+                       "TMLQCD_USE_HIP" OFF)
+
+# clime and lemon depend on MPI
+cmake_dependent_option(TMLQCD_USE_LEMON "Use the lemon io library" OFF
+                       "TMLQCD_USE_MPI" ON)
+
+# GPU dependent options
+cmake_dependent_option(TMLQCD_USE_QUDA_EXPERIMENTAL "Enable QUDA support" ON
+                       "TMLQCD_USE_QUDA" OFF)
+cmake_dependent_option(
+  TMLQCD_QUDA_FERMIONIC_FORCES "Enable support for fermionic forces using QUDA"
+  ON "TMLQCD_USE_QUDA" OFF)
+
+cmake_dependent_option(TMLQCD_USE_NVHPC "Enable Nvidia HPC toolkit" OFF
+                       "TMLQCD_USE_CUDA" OFF)
+
+# search for blas and lapack
+find_package(BLAS REQUIRED)
+#
+find_package(LAPACK REQUIRED)
+set(HAVE_LAPACK ON)
+find_package(FLEX REQUIRED)
+# do we need bison ?
+find_package(BISON REQUIRED)
+
+set(PACKAGE_NAME ${PROJECT_DESCRIPTION})
+set(PACKAGE_VERSION ${PROJECT_VERSION})
+set(PACKAGE_TARNAME "tmlqcd")
+set(PACKAGE_BUGREPORT "curbach@gmx.de")
+set(PACKAGE_STRING "${PROJECT_DESCRIPTION} ${PROJECT_VERSION}")
+
+unset(TM_USE_MPI)
+unset(TM_USE_OMP)
+unset(HAVE_LIBLEMON)
+unset(HAVE_LIBLIME)
+unset(FIXEDVOLUME)
+unset(_PERSISTENT)
+unset(_NON_BLOCKING)
+unset(HAVE_LIBQUDA)
+unset(TM_USE_QUDA)
+unset(TM_QUDA_EXPERIMENTAL)
+unset(TM_QUDA_FERMIONIC_FORCES)
+unset(DDalphaAMG)
+unset(TM_USE_QPHIX)
+unset(QPHIX_SOALEN)
+unset(_NEW_GEOMETRY)
+unset(_NON_BLOCKING)
+unset(_USE_SHMEM)
+unset(_USE_HALFSPINOR)
+set(ALIGN " ")
+set(ALIGN_BASE "0")
+set(ALIGN_BASE32 "0")
+set(ALIGN32 " ")
+
+if(TMLQCD_USE_HALFSPINOR)
+  set(_USE_HALFSPINOR ON)
+endif()
+
+if(TMLQCD_FIXED_VOLUME)
+  set(FIXEDVOLUME ON)
+endif()
+
+if(TMLQCD_PERSISTENT_MPI)
+  set(_PERSISTENT ON)
+endif()
+
+if(TMLQCD_USE_MPI)
+  find_package(MPI REQUIRED)
+  set(TM_USE_MPI ON)
+  if(TMLQCD_NONBLOCKING_MPI)
+    set(_NON_BLOCKING ON)
+  endif()
+endif()
+
+if(TMLQCD_USE_OPENMP)
+  find_package(OpenMP REQUIRED COMPONENTS C CXX)
+  set(TM_USE_OMP ON)
+endif()
+
+if(TMLQCD_USE_HDF5)
+  find_package(HDF5 REQUIRED COMPONENTS C)
+endif()
+
+if(TMLQCD_USE_LEMON)
+  find_package(Clemon REQUIRED)
+  set(HAVE_LIBLEMON ON)
+endif()
+
+find_package(CLime REQUIRED)
+set(HAVE_LIBLIME ON)
+
+if(TMLQCD_USE_QUDA)
+  find_package(QUDA REQUIRED config)
+  set(HAVE_LIBQUDA ON)
+  if(TMLQCD_USE_QUDA_EXPERIMENTAL)
+    set(TM_QUDA_EXPERIMENTAL ON)
+  endif()
+  if(TMLQCD_QUDA_FERMIONIC_FORCES)
+    set(TM_QUDA_FERMIONIC_FORCES ON)
+  endif()
+  if(TMLQCD_USE_CUDA OR TMLQCD_USE_HIP)
+    set(TM_USE_QUDA ON)
+  endif()
+endif()
+
+if(TMLQCD_USE_CUDA AND TMLQCD_USE_HIP)
+  message(
+    ERROR
+    "HIP and CUDA are mutually exclusive. Please choose one GPU support only")
+endif()
+
+if(TMLQCD_USE_CUDA OR QUDA_TARGET_CUDA)
+  enable_language(CUDA)
+  if(TMLQCD_USE_NVHPC)
+    find_package(NVHPC REQUIRED COMPONENTS CUDA MATH HOSTUTILS NCCL)
+  else()
+    find_package(CUDAToolkit REQUIRED)
+  endif()
+endif()
+
+message("QUDA_TARGET: ${QUDA_TARGET_CUDA}")
+if(TMLQCD_USE_HIP OR QUDA_TARGET_HIP)
+  enable_language(hip)
+
+  # we may want to use hip-cuda for development or debugging purposes especially
+  # if AMD GPU access is not possible. So allow it
+  if(TMLQCD_USE_CUDA_HIP)
+    find_package(CUDA)
+  endif()
+
+  if(CMAKE_HIP_PLATFORM MATCHES "amd")
+    set(TMLQCD_GPU_PLATFORM_DFLAGS "__HIP_PLATFORM_AMD__")
+  else()
+    set(TMLQCD_GPU_PLATFORM_DFLAGS "__HIP_PLATFORM_NVIDIA__")
+  endif()
+endif()
+
+if(TMLQCD_USE_SHMEM)
+  set(_USE_SHMEM ON)
+endif()
+
+if(TMLQCD_USE_QPIHX)
+  find_package(QphiX REQUIRED)
+  set(TM_USE_QPHIX ON)
+endif()
+
+# check for fftw3 (rely on pkgconfig).
+if(TMLQCD_USE_FFTW)
+  pkg_search_module(tmlqcd_fftw3 IMPORTED_TARGET GLOBAL fftw3)
+  if(tmlqcd_fftw3_FOUND)
+    add_library(tmlqcd::fftw3 ALIAS PkgConfig::tmlqcd_fftw3)
+  endif()
+endif()
+
+# check for the presence of clock_gettime in libc or librt
+check_symbol_exists(clock_gettime "time.h" HAVE_CLOCK_GETTIME)
+check_library_exists(rt clock_gettime "" HAVE_CLOCK_GETTIME_IN_RT)
+check_function_exists(fseeko HAVE_FSEEKO)
+
+# set the parallelization
+
+if(TMLQCD_USE_MPI)
+  if(TMLQCD_MPI_DIMENSION EQUAL "1")
+    # T parallelisation
+    set(PARALLELT ON)
+  elseif(TMLQCD_MPI_DIMENSION EQUAL "2")
+    # XT parallelisation
+    set(PARALLELXT ON)
+  elseif(TMLQCD_MPI_DIMENSION EQUAL "3")
+    set(PARALLELXYT ON)
+    # XYZ parallelisation
+  elseif(TMLQCD_MPI_DIMENSION EQUAL "4")
+    # timeslice-splitted communications
+    set(PARALLELXYZT ON)
+  elseif(TMLQCD_MPI_DIMENSION EQUAL "X")
+    set(PARALLELX ON)
+  elseif(TMLQCD_MPI_DIMENSION EQUAL "XY")
+    set(PARALLELXY ON)
+  elseif(TMLQCD_MPI_DIMENSION EQUAL "XYZ")
+    set(PARALLELXYZ ON)
+  else()
+    set(PARALLELXYZT ON)
+  endif()
+endif()
+
+# keep the autotool config.h header.
+configure_file("${PROJECT_SOURCE_DIR}/cmake/tmlqcd_config_internal.h.in"
+               "${PROJECT_BINARY_DIR}/tmlqcd_config_internal.h" @ONLY)
+configure_file("${PROJECT_SOURCE_DIR}/fixed_volume.h.in"
+               "${PROJECT_BINARY_DIR}/fixed_volume.h" @ONLY)
+# check if git command exists
+find_program(GIT_EXE NAMES git)
+
+# generate version header
+string(TIMESTAMP TMLQCD_TIMESTAMP "%Y-%m-%d %H:%M:%S")
+if(DEFINED GIT_EXE AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
+  execute_process(
+    COMMAND git rev-parse HEAD
+    OUTPUT_VARIABLE TMLQCD_SHA
+    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+  execute_process(
+    COMMAND git describe --all
+    OUTPUT_VARIABLE TMLQCD_GIT_BRANCH
+    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+  message(STATUS "git hash ${TMLQCD_SHA}")
+else()
+  # set(TMLQCD_GIT_BRANCH "release v${SIRIUS_VERSION}")
+  set(TMLQCD_SHA
+      "https://github.com/etmc/tmLQCD/releases/tag/rel-${TMLQCD_VERSION_MAJOR}-${TMLQCD_VERSION_MINOR}"
+  )
+endif()
+
+configure_file(cmake/git_hash.h.in git_hash.h @ONLY)
+add_subdirectory(src/lib)
+add_subdirectory(src/bin)

cmake/FindCLime.cmake

@@ -0,0 +1,27 @@
+include(FindPackageHandleStandardArgs)
+
+find_library(
+  TMLQCD_CLIME_LIBRARIES
+  NAMES lime
+  PATH_SUFFIXES "lib" "lib64")
+
+find_path(
+  TMLQCD_CLIME_INCLUDE_DIRS
+  NAMES lime.h
+  PATH_SUFFIXES "include" "include/${_pacakge_name}" "${_package_name}")
+
+message("${TMLQCD_CLIME_INCLUDE_DIRS}")
+find_package_handle_standard_args(CLime DEFAULT_MSG TMLQCD_CLIME_LIBRARIES
+                                  TMLQCD_CLIME_INCLUDE_DIRS)
+
+if(NOT TARGET tmlqcd::clime)
+  add_library(tmlqcd::clime INTERFACE IMPORTED)
+  set_target_properties(tmlqcd::clime PROPERTIES INTERFACE_LINK_LIBRARIES
+                                                 "${TMLQCD_CLIME_LIBRARIES}")
+  set_target_properties(tmlqcd::clime PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                                 "${TMLQCD_CLIME_INCLUDE_DIRS}")
+endif()
+
+set(TMLQCD_CLIME_FOUND ON)
+mark_as_advanced(TMLQCD_CLIME_FOUND TMLQCD_CLIME_LIBRARIES
+                 TMLQCD_CLIME_INCLUDE_DIRS)

cmake/FindLemon.cmake

@@ -0,0 +1,25 @@
+include(FindPackageHandleStandardArgs)
+
+find_library(
+  TMLQCD_LEMON_LIBRARIES
+  NAMES lemon
+  PATH_SUFFIXES "lib" "lib64")
+
+find_path(
+  TMLQCD_LEMON_INCLUDE_DIRS
+  NAMES lemon.h
+  PATH_SUFFIXES "include" "include/${_pacakge_name}" "${_package_name}")
+
+find_package_handle_standard_args(Lemon DEFAULT_MSG TMLQCD_LEMON_LIBRARIES
+                                  TMLQCD_LEMON_INCLUDE_DIRS)
+
+if(NOT TARGET tmlqcd::lemon)
+  add_library(tmlqcd::lemon INTERFACE IMPORTED)
+  set_target_properties(tmlqcd::lemon PROPERTIES INTERFACE_LINK_LIBRARIES
+                                                 "${TMLQCD_LEMON_LIBRARIES}")
+  set_target_properties(tmlqcd::lemon PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                                 "${TMLQCD_LEMON_INCLUDE_DIRS}")
+endif()
+
+set(TMLQCD_LEMON_FOUND ON)
+mark_as_advanced(TMLQCD_LEMON_LIBRARIES TMLQCD_LEMON_INCLUDE_DIRS)

cmake/git_hash.h.in

@@ -0,0 +1,6 @@
+#ifndef _GIT_HASH_H
+#define _GIT_HASH_H
+
+const char git_hash[] = "@TMLQCD_SHA@";
+
+#endif /* _GIT_HASH_H */