CMakeLists.txt

##############################################################################################
# CMAKE Configurations for UppASD
# (C) Anders Bergman 2019
#
# Acknowledgements: 
# https://github.com/SethMMorton/cmake_fortran_template  (for starting template)
# https://github.com/openmeeg/findmkl_cmake/blob/master/cmake/FindMKL.cmake (for FindMKL)
# https://github.com/libigl/eigen/blob/master/cmake/FindFFTW.cmake (for FindFFTW)
# 
##############################################################################################


##############################################################################################
# Define the project and the depencies that it has
##############################################################################################
#CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12)
CMAKE_MINIMUM_REQUIRED(VERSION 3.13)
CMAKE_POLICY(VERSION 3.13)

PROJECT(uppasd
   LANGUAGES Fortran)
# Set the project version
SET(VERSION 5.1.0)
##############################################################################################

##############################################################################################
# Check OS and compiler status
##############################################################################################
message(STATUS "Operating system= ${CMAKE_SYSTEM_NAME}")
include(CMakeDetermineFortranCompiler)
message(STATUS "CMAKE_Fortran_COMPILER_ID ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER}.")
message(STATUS "CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}.")
message(STATUS "CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES ${CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES}.")
include(CMakeDetermineCCompiler)
message(STATUS "CMAKE_C_COMPILER_ID ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER}.")
enable_language(C)
##############################################################################################


##############################################################################################
# Add our local modlues to the module path
##############################################################################################
message(STATUS "Initial module path: ${CMAKE_MODULE_PATH}")
SET(LOCAL_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/")
SET(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${LOCAL_MODULE_PATH}")
message(STATUS "Final module path  : ${CMAKE_MODULE_PATH}")
##############################################################################################

##############################################################################################
# Create lib directory if missing
##############################################################################################
set(LIB ${CMAKE_SOURCE_DIR}/lib)
message(STATUS "The lib is ${LIB}")
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${LIB})
message(STATUS "The lib is ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
file(MAKE_DIRECTORY ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
##############################################################################################


##############################################################################################
# Defines the directories for source, modules and object filed
##############################################################################################
SET(SRC ${CMAKE_SOURCE_DIR}/source)
#SET(MOD ${CMAKE_SOURCE_DIR}/source/mod)
SET(MOD ${CMAKE_BINARY_DIR}/mod)
SET(BIN ${CMAKE_SOURCE_DIR}/bin)
# Have the .mod files placed in the lib folder
SET(CMAKE_Fortran_MODULE_DIRECTORY ${MOD})
# Have the .o files placed in the bin folder
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/bin)
##############################################################################################


##############################################################################################
# Uncomment if it is required that Fortran 90 is supported
##############################################################################################
IF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
   MESSAGE(FATAL_ERROR "Fortran compiler does not support F90")
ENDIF(NOT CMAKE_Fortran_COMPILER_SUPPORTS_F90)
##############################################################################################

##############################################################################################
# There is an error in CMAKE with this flag for pgf90.  Unset it
##############################################################################################
GET_FILENAME_COMPONENT(FCNAME ${CMAKE_Fortran_COMPILER} NAME)
set(CMAKE_Fortran_COMPILE_OPTIONS_PIE "")
IF(FCNAME STREQUAL "pgf90")
   UNSET(CMAKE_SHARED_LIBRARY_LINK_Fortran_FLAGS)
ENDIF(FCNAME STREQUAL "pgf90")
##############################################################################################


##############################################################################################
# Define the executable name
##############################################################################################
set(ASD_binary sd)
#if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
#   string(APPEND CMAKE_Fortran_COMPILER ".exe")
#   #message(STATUS "---> ${CMAKE_Fortran_COMPILER}")
#endif()
# Find the vernacular compiler name
string(REGEX MATCH gfortran|g95|ifort|pgf90|f95|f77|nv current_compiler  ${CMAKE_Fortran_COMPILER})
message(STATUS "Matched compiler: ${current_compiler}.  ${CMAKE_Fortran_COMPILER} ${FCNAME}")
message(STATUS "Matched compiler: ${current_compiler}.  ${CMAKE_Fortran_COMPILER} ${FCNAME}")
if(current_compiler)
   string(APPEND ASD_binary ".${current_compiler}")
endif()
# add a `cuda` tag to the binary if used.
if(USE_CUDA)
   string(APPEND ASD_binary ".cuda")
endif()
SET(UppASD_EXE ${ASD_binary})
message(STATUS "ASD_binary ${ASD_binary}.")
##############################################################################################


##############################################################################################
# Define the executable in terms of the source files
##############################################################################################
ADD_EXECUTABLE(${UppASD_EXE} ${SRC}/0sd.f90 )
##############################################################################################


##############################################################################################
# Define the library name
##############################################################################################
SET(UppASD_LIB asdlib)
#SET(UppASD_pyLIB pyasdlib)
##############################################################################################


##############################################################################################
# Define the executable in terms of the source files
##############################################################################################
set(lib_type STATIC)
if (BUILD_PYTHON)
   set(CMAKE_POSITION_INDEPENDENT_CODE ON)
   #   ADD_LIBRARY(${UppASD_pyLIB} SHARED)
endif(BUILD_PYTHON)
message(STATUS " LIB INFO: ${UppASD_LIB} ${lib_type} ${SRC} ${CMAKE_POSITION_INDEPENDENT_CODE}")
ADD_LIBRARY(${UppASD_LIB} ${lib_type})
##############################################################################################


##############################################################################################
# MKL / FFTW actions
##############################################################################################
set(MKL_LINK static)
set(MKL_INTERFACE lp64)
set(MKL_THREADING sequential)
find_package(MKL CONFIG)
message(STATUS "${MKL_IMPORTED_TARGETS}") 

set(FFTW_USE_STATIC_LIBS ON)
find_package(FFTW)
#include(${CMAKE_MODULE_PATH}/FindFFTW.cmake)
if (FFTW_FOUND)
	message(STATUS "FFTW found       : ${FFTW_FOUND}, ${FFTW_INCLUDES}, ${FFTW_INCLUDE_DIRS}")
	message(STATUS "FFTW_LIBRARIES   : ${FFTW_LIBRARIES}")
	message(STATUS "FFTW_INCLUDE_DIRS: ${FFTW_INCLUDE_DIRS}")
	message(STATUS "FFTW_DOUBLE_LIB  : ${FFTW_DOUBLE_LIB}")
	message(STATUS "FFTW_DOUBLE_OPENMP_LIB  : ${FFTW_DOUBLE_OPENMP_LIB}")
   set(FFTW_LIB ${FFTW_LIBRARIES})
   #set(FFTW_LIB "${FFTW_DOUBLE_LIB};${FFTW_DOUBLE_OPENMP_LIB}")
   # It sets the following variables:
#   FFTW_FOUND                  ... true if fftw is found on the system
#   FFTW_[component]_LIB_FOUND  ... true if the component is found on the system (see components below)
#   FFTW_LIBRARIES              ... full paths to all found fftw libraries
#   FFTW_[component]_LIB        ... full path to one of the components (see below)
#   FFTW_INCLUDE_DIRS           ... fftw include directory paths

else (FFTW_FOUND)
   set(FFTW_LIB "")
   set(FFTW_LIBRARIES "")
endif (FFTW_FOUND)
##############################################################################################

##############################################################################################
# Set relevant user-controlled options for the compilation
##############################################################################################
include(CMakeDependentOption)
OPTION(USE_OPENMP "Use OpenMP for parallelization" ON)
OPTION(USE_CUDA "Compile with CUDA support" OFF)
# Uncomment below to make gnu defaults without MKL
if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
   CMAKE_DEPENDENT_OPTION(USE_MKL "Use Intel MKL for BLAS/LAPACK" OFF "MKL_FOUND" OFF)
   CMAKE_DEPENDENT_OPTION(USE_VSL "Use Intel VSL (needs MKL)" OFF "MKL_FOUND" OFF)
   CMAKE_DEPENDENT_OPTION(USE_MKL_FFT "Use Intel MKL for FFT" OFF "MKL_FOUND"  OFF)
   CMAKE_DEPENDENT_OPTION(USE_FFTW "Use FFTW for FFT" OFF "FFTW_FOUND" OFF)
else()
   CMAKE_DEPENDENT_OPTION(USE_MKL "Use Intel MKL for BLAS/LAPACK" ON "MKL_FOUND" OFF)
   CMAKE_DEPENDENT_OPTION(USE_VSL "Use Intel VSL (needs MKL)" ON "MKL_FOUND" OFF)
   CMAKE_DEPENDENT_OPTION(USE_MKL_FFT "Use Intel MKL for FFT" ON "MKL_FOUND"  OFF)
   CMAKE_DEPENDENT_OPTION(USE_FFTW "Use FFTW for FFT" OFF "FFTW_FOUND" OFF)
endif()
CMAKE_DEPENDENT_OPTION(BUILD_CUDA_DOCS "Build LaTeX notes of the CUDA implementation" OFF "BUILD_MANUAL" OFF) 
CMAKE_DEPENDENT_OPTION(BUILD_GNEB_DOCS "Build LaTeX notes of the GNEB implementation" OFF "BUILD_MANUAL" OFF) 
OPTION(USE_OVF "Use OVF format" OFF)
OPTION(BUILD_MANUAL "Build manual"  OFF)
OPTION(BUILD_DOXYGEN "Build doxygen"  OFF)
OPTION(BUILD_PYTHON "Build Python interface"  OFF)
CMAKE_DEPENDENT_OPTION(RUN_ASD_TESTS "Run asd-tests"  ON "BUILD_TESTING" OFF)
CMAKE_DEPENDENT_OPTION(RUN_SLD_TESTS "Run sld-tests"  OFF "BUILD_TESTING" OFF)
CMAKE_DEPENDENT_OPTION(RUN_GNEB_TESTS "Run gneb-tests"  OFF "BUILD_TESTING" OFF)
CMAKE_DEPENDENT_OPTION(RUN_REG_TESTS "Run regression-test"  ON "BUILD_TESTING" OFF)
#OPTION(RUN_ASD_TESTS "Run asd-tests"  OFF)
##############################################################################################

##############################################################################################
# Determine and set compiler-dependent flags
##############################################################################################
INCLUDE(${LOCAL_MODULE_PATH}/SetFortranFlags.cmake) 
##############################################################################################

##############################################################################################
# Add OpenMP compiler/linker flags
##############################################################################################
IF(USE_OPENMP)
   FIND_PACKAGE(OpenMP)
   message(STATUS "Did we find OpenMP? ${OpenMP_Fortran_FOUND}")

   if(OpenMP_Fortran_FOUND)
      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${OpenMP_Fortran_FLAGS}")
   endif()

ENDIF(USE_OPENMP)
##############################################################################################


##############################################################################################
# Build LaTeX manual (and extra notes if requested)
##############################################################################################
if(BUILD_MANUAL)
   include(${CMAKE_MODULE_PATH}/UseLATEX.cmake)
   add_subdirectory(docs/Manual)
   if(BUILD_GNEB_DOCS)
      add_subdirectory(docs/GNEB)
   endif()
   if(BUILD_CUDA_DOCS)
      add_subdirectory(docs/CUDA)
   endif()
endif()
##############################################################################################

##############################################################################################
# Build Doxygen documentation
##############################################################################################
if(BUILD_DOXYGEN)
   # check if Doxygen is installed
   find_package(Doxygen)
   if (DOXYGEN_FOUND)
      ## set input and output files
      set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile)
      #set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)

      ## request to configure the file
      #configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
      message(STATUS "Doxygen build started")

      # note the option ALL which allows to build the docs together with the application
      add_custom_target( doc_doxygen ALL
         COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_IN}
         WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docs
         COMMENT "Generating API documentation with Doxygen"
         VERBATIM )
   else (DOXYGEN_FOUND)
      message("Doxygen need to be installed to generate the doxygen documentation")
   endif (DOXYGEN_FOUND)
endif()
##############################################################################################



##############################################################################################
# If MKL/VSL is requested. First checks here to set the linear algebra packages
##############################################################################################
set(BLA_STATIC ON)
message( STATUS "USING::::: ${USE_VSL} ${USE_MKL} ${USE_MKL_FFT}")
if(USE_VSL OR USE_MKL OR USE_MKL_FFT)

   set(BLA_DEFINITIONS)

   if (BLA_VENDOR MATCHES "_seq")
      set(MKL_USE_parallel OFF)
   else()
      set(MKL_USE_parallel ON)
   endif()

   if (MKL_FOUND)
      message( STATUS "MKL_FOUND condition true")
      set(BLA_INCLUDE_DIR ${MKL_INCLUDE_DIR})
      set(BLAS_LIBRARIES ${MKL_LIBRARIES})
      set(LAPACK_LIBRARIES ${MKL_LIBRARIES})
      set(BLA_DEFINITIONS USE_MKL)
      #set(MKL_INTERFACE_FULL "lp64")
   else()
      #include(FindBLAS)
      #include(FindLAPACK)
      find_package(BLAS REQUIRED)
      find_package(LAPACK REQUIRED)
   endif()
   message(STATUS "mkl-location: ${MKL_LIBRARY_DIR}")
   message(STATUS "mkl-includes: ${MKL_INCLUDE}")
   #link_directories(${MKL_LIBRARY_DIR})

   target_compile_options(${UppASD_LIB} PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
   target_include_directories(${UppASD_LIB} PUBLIC $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)

else()
   find_package(BLAS REQUIRED)
   find_package(LAPACK REQUIRED)
endif()
#find_package(BLAS REQUIRED)
#find_package(LAPACK REQUIRED)
message(STATUS "Math libs: 
BLAS:${BLAS_FOUND} ${BLAS_LIBRARIES}  ${BLAS_LINKER_FLAGS}
LAPACK:${LAPACK_FOUND} ${LAPACK_LIBRARIES}  ${LAPACK_LINKER_FLAGS}
MKL:${MKL_FOUND} ${MKL_LIBRARIES} ${MKL_LINKER_FLAGS}"
)
##############################################################################################


##############################################################################################
# assume built-in pthreads on MacOS (Is this needed?)
##############################################################################################
IF(APPLE)
   set(CMAKE_THREAD_LIBS_INIT "-lpthread")
   set(CMAKE_HAVE_THREADS_LIBRARY 1)
   set(CMAKE_USE_WIN32_THREADS_INIT 0)
   set(CMAKE_USE_PTHREADS_INIT 1)
   set(THREADS_PREFER_PTHREAD_FLAG ON)
ENDIF()
##############################################################################################


##############################################################################################
# Extract git-stamp
##############################################################################################
INCLUDE(${LOCAL_MODULE_PATH}/git-watcher.cmake)
##############################################################################################

##############################################################################################
# Add CUDA flags and preprocessor commands
##############################################################################################
if (USE_CUDA) 

   ENABLE_LANGUAGE(C CUDA CXX)
   find_package(CUDAToolkit)
   target_sources(${UppASD_LIB} PRIVATE ${SRC}/chelper.f90)
   add_subdirectory(${SRC}/gpu_files)
   add_compile_definitions(CUDA)
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -I${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${CUDAToolkit_MATH_INCLUDE_DIR}")

   # Set preprocessing flags indicating compiler
   string(TOUPPER ${current_compiler} PP_CUDA_COMPILER_FLAG)
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D__${PP_CUDA_COMPILER_FLAG}__")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__${PP_CUDA_COMPILER_FLAG}__")

   # Find CUDA arch on local machine
   include(FindCUDA/select_compute_arch)
   CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
   string(STRIP "${INSTALLED_GPU_CCS_1}" INSTALLED_GPU_CCS_2)
   string(REPLACE " " ";" INSTALLED_GPU_CCS_3 "${INSTALLED_GPU_CCS_2}")
   string(REPLACE "." "" CUDA_ARCH_LIST "${INSTALLED_GPU_CCS_3}")
   SET(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_LIST})
   SET_TARGET_PROPERTIES(${UppASD_LIB} PROPERTIES CUDA_ARCHITECTURES "${CUDA_ARCH_LIST}")

else ()
   target_sources(${UppASD_LIB} PRIVATE ${SRC}/Tools/nocuda.f90)
endif (USE_CUDA)
##############################################################################################


##############################################################################################
# Link with OVF library if available 
##############################################################################################
IF(USE_OVF)
   find_library(OVF_LIB ovf /lib/python3.7/site-packages/ovf/ /usr/lib64/)
   message(STATUS "Looking for OVF library, found in ${OVF_LIB}.")
   add_compile_definitions(USE_OVF)
   target_sources(${UppASD_LIB} PRIVATE ${SRC}/Third_party/ovf/fortran/ovf.f90)
ELSE()
   unset(OVF_LIB)
ENDIF(USE_OVF)
##############################################################################################

##############################################################################################
# MKL edits in case of GNU compiler
##############################################################################################
if(USE_MKL)
   if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
      target_include_directories(${UppASD_LIB} PRIVATE ${MKL_INCLUDE})
   endif()
endif()
##############################################################################################


##############################################################################################
# Include VSL preprocessor flag (recommended)
##############################################################################################
if(USE_VSL AND MKL_FOUND)
   add_compile_definitions(VSL)
   #add_compile_definitions(USE_VSL)
endif()
##############################################################################################


##############################################################################################
# Check for FFT treatment (MKL default, FFTW available)
##############################################################################################
if(USE_MKL_FFT AND MKL_FOUND)
   target_sources(${UppASD_LIB} PRIVATE ${SRC}/Hamiltonian/fftdipole_mkl.f90)
   add_compile_definitions(USE_MKL_FFT)
   include_directories(${BLA_INCLUDE_DIR})
   #message(STATUS "FFT-> ${USE_MKL_FFT} ")
elseif(USE_FFTW)
   target_sources(${UppASD_LIB} PRIVATE ${SRC}/Hamiltonian/fftdipole_fftw.f90)
   add_compile_definitions(USE_FFTW)
   message(STATUS "FFT-> ${USE_FFTW} ")
   #include(${CMAKE_MODULE_PATH}/FindFFTW.cmake)
   include_directories(${FFTW_INCLUDE_DIRS})
   message(STATUS "FFTW: -I${FFTW_INCLUDE_DIRS} ${FFTW_LIB}")
endif()
##############################################################################################


##############################################################################################
# Add a distclean target to the Makefile
##############################################################################################
ADD_CUSTOM_TARGET(distclean 
   COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/distclean.cmake
   )
##############################################################################################


##############################################################################################
# Build Python interface
###############################################################################################
if(BUILD_PYTHON)
   # check if Python is installed
   #set(Python_VERSION $ENV{PYTHON_VERSION})
   #message(STATUS "Python target version: ${Python_VERSION}")
   message(STATUS "Python target version: $ENV{PYTHON_VERSION}")
   #include(FindPython)

   # Fix since FindPython is a bit flawed
   # https://github.com/scikit-build/scikit-build/issues/506#issuecomment-706791791
   find_package(Python3 COMPONENTS Interpreter Development.Module NumPy)
   #set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
   #set(Python_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}")
   #set(Python_LIBRARY "${PYTHON_LIBRARY}")
   #find_package(Python3 COMPONENTS Interpreter Development.Module NumPy)
   #find_package(Python ${Python_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development.Module NumPy)
   #find_package(Python)
   message(STATUS "============================================")
   message(STATUS "Python3_EXECUTABLE         ${Python3_EXECUTABLE}")
   message(STATUS "Python3_STDLIB             ${Python3_STDLIB}.")
   message(STATUS "Python3_SITELIB            ${Python3_SITELIB}")
   message(STATUS "Python3_LIBRARIES          ${Python3_LIBRARIES}.")
   message(STATUS "Python3_LIBRARY_DIRS       ${Python3_LIBRARY_DIRS}.")
   message(STATUS "Python3_NumPy_INCLUDE_DIRS ${Python3_NumPy_INCLUDE_DIRS}")
   message(STATUS "============================================")

   if (Python3_FOUND)
      # Grab the variables from a local Python installation
      # F2PY headers
      execute_process(
         COMMAND "${Python3_EXECUTABLE}" -c "import numpy.f2py; print(numpy.f2py.get_include())"
         OUTPUT_VARIABLE F2PY_INCLUDE_DIR
         OUTPUT_STRIP_TRAILING_WHITESPACE
         )

      execute_process(
         COMMAND "${Python3_EXECUTABLE}" -c "import numpy; print(numpy.version.version)"
         OUTPUT_VARIABLE F2PY_version
         OUTPUT_STRIP_TRAILING_WHITESPACE
         )

      # Common variables
      set(f2py_module_name "uppasd")
      set(f2py_module_c "_${f2py_module_name}module.c")
      set(f2py_wrapper_f "_${f2py_module_name}-f2pywrappers2.f90")
      set(f2py_wrapper_o "_${f2py_module_name}-f2pywrappers2.o")
      set(UppASD_F90w f90_wraps)
      set(F90wrap_kind
         ${CMAKE_SOURCE_DIR}/cmake/.f2py_f2cmap
         )

      # Uncomment if f90wrap needs to be invoked
      set(F90wrap_sources 
         ${SRC}/pyasd.f90 
         ${SRC}/uppasd.f90 
         ${SRC}/Input/inputdatatype.f90
         ${SRC}/Input/inputdata.f90
         ${SRC}/Input/inputhandler.f90
         ${SRC}/Input/inputhandler_ext.f90
         ${SRC}/System/momentdata.f90 
         ${SRC}/System/simulationdata.f90
         )
      set(F2PY_sources 
         ${SRC}/pyasd.f90
         #${SRC}/f90wrap/f90wrap_pyasd.f90
         #${SRC}/f90wrap/f90wrap_uppasd.f90
         #${SRC}/f90wrap/f90wrap_inputdatatype.f90
         #${SRC}/f90wrap/f90wrap_inputdata.f90
         #${SRC}/f90wrap/f90wrap_inputhandler.f90
         #${SRC}/f90wrap/f90wrap_inputhandler_ext.f90
         #${SRC}/f90wrap/f90wrap_simulationdata.f90
         #${SRC}/f90wrap/f90wrap_momentdata.f90
         )

      # Bild library list for f2py
      foreach ( elem ${OpenMP_Fortran_LIBRARIES} ${OVF_LIB} ${FFTW_LIB} ${CUDA_curand_LIBRARY} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES})
         #message(STATUS "Element ${elem}")
         string(SUBSTRING ${elem} 0 2 head)
         # Check for directories on linux/mac ('/') or windows ('C:')
         if(${head} MATCHES "/" OR ${head} MATCHES "C")
            if(NOT ${elem} MATCHES "\\.dylib" AND NOT ${elem} MATCHES "\\.so" AND NOT ${elem} MATCHES "\\.a")
               #if(NOT ${elem} MATCHES ".dylib" )
               string(PREPEND elem "-L")
            endif()
            if(${elem} MATCHES "libfftw3.dylib")
               string(LENGTH ${elem} str_len)
               string(LENGTH "libfftw3.dylib" str_lib)
               math(EXPR str_sub "${str_len}-${str_lib}")
               string(SUBSTRING ${elem} 0  ${str_sub}  leader)

               list(APPEND F2PY_libs -L${leader})
               set(elem "-lfftw3")
            endif()
            if(${elem} MATCHES "libgomp.dylib" OR ${elem} MATCHES "libgomp.a")
               set(elem "-lgomp")
            endif()
         endif()
         list(APPEND F2PY_libs ${elem})
      endforeach()
      list(APPEND F2PY_libs "-lgfortran")
      if(OpenMP_Fortran_FOUND AND ${current_compiler} MATCHES "gfortran")
         list(APPEND F2PY_libs "-lgomp")
      endif()
      list(REMOVE_DUPLICATES F2PY_libs)
      #foreach (elem ${F2PY_libs})
      #   message(STATUS "------------------ ${elem}")
      #endforeach()

      string(STRIP ${CMAKE_Fortran_FLAGS} fflags)
      if (current_compiler MATCHES "ifort")
         #set(F2PY_compiler --fcompiler=intelem)
         string(STRIP ${CMAKE_Fortran_FLAGS} fflags)
         set(F2PY_f90flags '--f90flags="${fflags}"')
      elseif (current_compiler MATCHES "nv")
         #set(F2PY_compiler --fcompiler=nv)
         set(F2PY_f90flags "--f90flags='${fflags}'")
      else()
         # Assuming GNU compiler if not intel/nvidia
         #if(current_compiler MATCHES "gfortran" OR current_compiler MATCHES "g95" OR current_compiler MATCHES "g95" OR current_compiler MATCHES "gfortran.exe")
         #set(F2PY_compiler --fcompiler=gnu95)
         set(F2PY_f90flags "--f90flags='${fflags}'")
      endif()
      set(F2PY_compiler --f77exec=${CMAKE_Fortran_COMPILER} --f90exec=${CMAKE_Fortran_COMPILER})
      
      message(STATUS "===========================================")
      message(STATUS "FORTRAN FLAGS= ${CMAKE_Fortran_FLAGS}")
      message(STATUS "Numpy version= ${F2PY_version}")
      message(STATUS "F2PY_compiler= ${F2PY_compiler}")
      message(STATUS "F2PY_libs    = ${F2PY_libs}")
      message(STATUS "F2PY_flags   = ${F2PY_f90flags}")
      message(STATUS "F2PY_flags   = --f90flags='${F2PY_f90flags} -O2'")
      message(STATUS "UppASD_LIB   = ${UppASD_LIB}")
      message(STATUS "===========================================")

##############################################################################################
# Shenanigans for f2py on Windows
##############################################################################################
message(STATUS "LDFLAGS pre: $ENV{LDFLAGS}")
if($ENV{NPY_WINDOWS_GIT_ACTION_FIX})
   message(STATUS "=========NPY_WINDOWS_GIT_ACTION_FIX=====")
   set(myLDFLAGS "$ENV{LDFLAGS} -LD:/a/UppASD/UppASD/build -lasdlib -LC:/rtools43/x86_64-w64-mingw32.static.posix/lib -lopenblas -ID:/a/UppASD/UppASD/build/mod --fcompiler=gnu95")
   #set(myLDFLAGS "$ENV{LDFLAGS} -LD:/a/UppASD/UppASD/build -lasdlib -LC:/rtools43/x86_64-w64-mingw32.static.posix/lib -lopenblas -Imod")
else()
   set(myLDFLAGS="$ENV{LDFLAGS}")
endif()
message(STATUS "LDFLAGS aft: $ENV{LDFLAGS}")
message(STATUS "myLDFLAGS: $ENV{myLDFLAGS}")




##############################################################################################
# Transform F2PY include-string to acceptable slash convention (thanks ChatGPT..)
##############################################################################################
# Input file name string (with double backslashes)
if (F2PY_INCLUDE_DIR)
   message(STATUS "Original F2PY-include: ${F2PY_INCLUDE_DIR}")
   ## Convert backslashes to slashes and replace drive letter (case-insensitive)
   #string(REGEX REPLACE "([Cc])(:)" "/\\1" t_string "${F2PY_INCLUDE_DIR}")
   ## Use string(SUBSTRING ...) and string(TOLOWER ...) to convert the first character of the drive letter to lowercase
   #string(SUBSTRING "${t_string}" 0 3 t_string_a)
   #string(SUBSTRING "${t_string}" 3 -1 t_string_b)
   #string(TOLOWER "${t_string_a}" t_string_a)
   #set(t_string "${t_string_a}${t_string_b}")
   ## Replace remaining backslashes with slashes
   #string(REPLACE "\\" "/" ${F2PY_INCLUDE_DIR}  "${t_string}")
   ## Output the transformed string
   file(TO_CMAKE_PATH ${F2PY_INCLUDE_DIR} F2PY_INCLUDE_DIR)
   message(STATUS "Transformed F2PY-include: ${F2PY_INCLUDE_DIR}")
endif ()

##############################################################################################


##############################################################################################
# Old Python generation
##############################################################################################
#       add_custom_target ( ${UppASD_F90w}  ALL
#          DEPENDS ${UppASD_LIB}
#          # Uncomment if f90wrap needs to be invoked
#          #DEPENDS ${UppASD_LIB} ${F2PY_sources}
#          WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
#          COMMAND ${CMAKE_COMMAND} -E env "LDFLAGS=${myLDFLAGS}" f2py -c ${F2PY_compiler}  ${F2PY_sources} -m _${f2py_module_name} -I${CMAKE_Fortran_MODULE_DIRECTORY} -L${CMAKE_BINARY_DIR} -lasdlib ${F2PY_libs} -I${Python3_INCLUDE_DIRS}  --link-blas_opt --link-lapack_opt --verbose
#          ### Old versions
#          ### COMMAND f2py -c ${F2PY_compiler}  ${F2PY_sources} -m _${f2py_module_name} -I${CMAKE_Fortran_MODULE_DIRECTORY} -L${CMAKE_BINARY_DIR} -l${UppASD_LIB} ${F2PY_libs} -I${Python3_INCLUDE_DIRS} --verbose
#          ### COMMAND f2py -c ${F2PY_compiler}  ${F2PY_sources} -m _${f2py_module_name} -I${CMAKE_Fortran_MODULE_DIRECTORY} -L${CMAKE_BINARY_DIR} -L. -l${UppASD_LIB} ${F2PY_libs} -I${Python3_INCLUDE_DIRS} --link-blas_opt --link-lapack_opt --verbose
#          ### COMMAND ${Python3_EXECUTABLE}  -m "numpy.f2py" -c ${F2PY_compiler}  ${F2PY_sources} -m _${f2py_module_name} -I${CMAKE_Fortran_MODULE_DIRECTORY} -L${CMAKE_BINARY_DIR} -l${UppASD_LIB} ${F2PY_libs} --link-blas_opt --link-lapack_opt --lower
#          COMMENT "Generating Python modules f2py -c ${F2PY_compiler}  ${F2PY_sources} -m _${f2py_module_name} -I${CMAKE_Fortran_MODULE_DIRECTORY} -L${CMAKE_BINARY_DIR} -l${UppASD_LIB} ${F2PY_libs} -I${Python3_INCLUDE_DIRS}"
#          VERBATIM
#          )
##############################################################################################
# New Python generation (not stable)
##############################################################################################
      add_custom_target ( ${UppASD_F90w}
         DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}"
         VERBATIM
         )

      #add_custom_command( 
      #   OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${f2py_wrapper_o}"
      #   COMMAND ${CMAKE_Fortran_COMPILER} -c  ${CMAKE_CURRENT_BINARY_DIR}/${f2py_wrapper_f}  -I${MOD}
      #   COMMENT "Writing F90wrappers for F2PY"
      #   DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${f2py_wrapper_f}"
      #   )

      add_custom_command(
         OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}" "${CMAKE_CURRENT_BINARY_DIR}/${f2py_wrapper_f}"
         COMMAND ${Python3_EXECUTABLE}  -m "numpy.f2py"  ${F2PY_sources} -m _${f2py_module_name}  --lower --quiet
         #COMMAND f2py   ${F2PY_sources} -m _${f2py_module_name}  --lower --quiet
         COMMENT "Generating Python modules"
         DEPENDS ${UppASD_LIB}  ${F2PY_sources}
         VERBATIM
         )

      message(STATUS "Python3_INCLUDE_DIRS" ${Python3_INCLUDE_DIRS})
      message(STATUS "F2PY_INCLUDE_DIR" ${F2PY_INCLUDE_DIR})
      message(STATUS "Python3_NumPy_INCLUDE_DIRS" ${Python3_NumPy_INCLUDE_DIRS})

      # Set up target
      message(STATUS "CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}")
      message(STATUS "F2PY_INCLUDE_DIR:        ${F2PY_INCLUDE_DIR}")
      message(STATUS "=============>  ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}")

      Python3_add_library(_${CMAKE_PROJECT_NAME} MODULE WITH_SOABI
         "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}"
         "${F2PY_sources}"
         "${CMAKE_CURRENT_BINARY_DIR}/${f2py_wrapper_f}"
         "${F2PY_INCLUDE_DIR}/fortranobject.c" # From NumPy
         )

      #target_link_libraries(_${CMAKE_PROJECT_NAME}  PUBLIC Python3::NumPy ${UppASD_LIB})
      target_link_libraries(_${CMAKE_PROJECT_NAME}  PUBLIC Python3::NumPy ${UppASD_LIB}    ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${OVF_LIB} ${FFTW_LIB} ${CUDA_curand_LIBRARY} ${OpenMP_Fortran_LIBRARIES} )
      add_dependencies(_${CMAKE_PROJECT_NAME} ${UppASD_F90w})
      target_include_directories(_${CMAKE_PROJECT_NAME} PRIVATE "${F2PY_INCLUDE_DIR}" )
##############################################################################################

      # Not used by default (invoke only if new header files need to be generated)
      #add_custom_command( 
      #   OUTPUT ${F2PY_sources}
      #   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
      #   COMMAND f90wrap  -P -k ${F90wrap_kind} -m ${f2py_module_name}  ${F90wrap_sources}
      #   COMMENT "Writing F90wrappers for F2PY"
      #   DEPENDS ${F90wrap_sources}
      #   )

      #add_dependencies(${UppASD_F90w} ${UppASD_LIB})

      ###add_custom_command(
      ###   TARGET ${UppASD_F90w} POST_BUILD
      ###   COMMAND ${CMAKE_COMMAND} -E 
      ###   #${CMAKE_BINARY_DIR}/_*.so
      ###   copy -t ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
      ###   ${CMAKE_BINARY_DIR}/_*.*
      ###   )
      message(STATUS "COPY:::" ${CMAKE_BINARY_DIR} )

   else (Python3_FOUND)
      message("A proper Python environment needs to be installed to generate the python libraries")
   endif (Python3_FOUND)
endif(BUILD_PYTHON)
##############################################################################################



##############################################################################################
mark_as_advanced( ENABLE_BLACS ENABLE_BLAS95 ENABLE_CDFT ENABLE_CPARDISO ENABLE_LAPACK95 ENABLE_OMP_OFFLOAD ENABLE_SCALAPACK MKL_DIR MKL_H MKL_LINK MKL_MPI mkl_gf_ilp64_file mkl_intel_thread_file mkl_core_file mkl_intel_ilp64_file mkl_gnu_thread_file mkl_intel_lp64_file mkl_gf_lp64_file OMP_LIBRARY)

##############################################################################################
# Add the needed libraries and special compiler flags
##############################################################################################
TARGET_LINK_LIBRARIES(${UppASD_LIB} 
   -lgfortran
   ${BLAS_LIBRARIES}
   ${LAPACK_LIBRARIES}
   ${CMAKE_THREAD_LIBS_INIT} 
   ${OVF_LIB} 
   ${FFTW_LIB}  
   ${CUDA_curand_LIBRARY}
   ${OpenMP_Fortran_LIBRARIES}
   )

#TARGET_LINK_LIBRARIES(${UppASD_pyLIB} 
#   -lgfortran
#   ${UppASD_LIB}
#   ${BLAS_LIBRARIES}
#   ${LAPACK_LIBRARIES}
#   ${CMAKE_THREAD_LIBS_INIT} 
#   ${OVF_LIB} 
#   ${FFTW_LIB}  
#   ${CUDA_curand_LIBRARY}
#   ${OpenMP_Fortran_LIBRARIES}
#   )

TARGET_LINK_LIBRARIES(${UppASD_EXE}
   -lgfortran
   ${UppASD_LIB}
   ${BLAS_LIBRARIES}
   ${LAPACK_LIBRARIES}
   ${CMAKE_THREAD_LIBS_INIT} 
   ${OVF_LIB} 
   ${FFTW_LIB}  
   ${CUDA_curand_LIBRARY}
   ${OpenMP_Fortran_LIBRARIES}
   )


##############################################################################################
# Tell where to install this executable
##############################################################################################
IF(WIN32)
   SET(CMAKE_INSTALL_PREFIX "C:\\Program Files")
ELSE()
   SET(CMAKE_INSTALL_PREFIX /usr/local)
ENDIF(WIN32)


SET_TARGET_PROPERTIES(${UppASD_EXE} PROPERTIES LINKER_LANGUAGE "Fortran")
if (NOT BUILD_PYTHON)
   INSTALL(TARGETS ${UppASD_EXE} RUNTIME DESTINATION bin)
   INSTALL(TARGETS ${UppASD_LIB} RUNTIME DESTINATION lib)
   #INSTALL(TARGETS UppASD_F90w RUNTIME DESTINATION bin)
else()
   #   INSTALL(TARGETS ${UppASD_pyLIB} RUNTIME DESTINATION lib)
   INSTALL(TARGETS ${UppASD_LIB} RUNTIME DESTINATION lib)
endif(NOT BUILD_PYTHON)
##############################################################################################

##############################################################################################
# Copy static library to ./lib
##############################################################################################
add_custom_command(
   TARGET ${UppASD_LIB} POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E copy
   ${CMAKE_BINARY_DIR}/lib${UppASD_LIB}.a
   ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
   #COMMAND ${CMAKE_COMMAND} -E copy
   #${CMAKE_BINARY_DIR}/_uppasd*.so
   #${CMAKE_SOURCE_DIR}/python/uppasd/
   )


###############################################################################################
## Add Sources in sub-directories
###############################################################################################
add_subdirectory(${SRC})
##############################################################################################

###############################################################################################
## Add tests directory
###############################################################################################
include(CTest)
set(TestBinary ${CMAKE_SOURCE_DIR}/bin/${UppASD_EXE})
set(TestDIR ${CMAKE_SOURCE_DIR}/tests)
if (RUN_REG_TESTS)
   add_test(NAME regression-test  COMMAND python3 -u bergtest.py --file regressionHisingen.yaml --binary ${TestBinary}  WORKING_DIRECTORY ${TestDIR})
endif (RUN_REG_TESTS)
if (RUN_ASD_TESTS)
   add_test(NAME asd-tests  COMMAND python3 -u bergtest.py --file regulartests.yaml --binary ${TestBinary}  WORKING_DIRECTORY ${TestDIR})
endif (RUN_ASD_TESTS)
if (RUN_SLD_TESTS)
   add_test(NAME sld-tests  COMMAND python3 -u ./bergtest.py --file sldtests.yaml --binary ${TestBinary}  WORKING_DIRECTORY ${TestDIR})
endif (RUN_SLD_TESTS)
if (RUN_GNEB_TESTS)
   add_test(NAME gneb-tests  COMMAND python3 -u ./bergtest.py --file regressionGNEB_15d7169b.yaml --binary ${TestBinary}  WORKING_DIRECTORY ${TestDIR})
endif (RUN_GNEB_TESTS)
message(STATUS "Output binary:  ${TestBinary}")
##############################################################################################