Build omp_lib.mod on device

Author: Meinersbur

flang-rt/CMakeLists.txt

@@ -43,7 +43,7 @@ include(CMakePushCheckState)
 ############################
 
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
-set(LLVM_TOOLS_DIR "${LLVM_TOOLS_BINARY_DIR}")
+set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
 # Fortran compiler not optional for building Flang-RT
 enable_language(Fortran)
@@ -113,14 +113,8 @@ elseif (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "CUDA")
   option(FLANG_RT_CUDA_RUNTIME_PTX_WITHOUT_GLOBAL_VARS "Do not compile global variables' definitions when producing PTX library" OFF)
 elseif (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP")
   # Support for OpenMP offloading
-else ()
-  message(FATAL_ERROR "Invalid value '${FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT}' for FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT; must be empty, 'CUDA', or 'OpenMP'")
-endif ()
-
-
-if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP" OR LLVM_TARGET_TRIPLE MATCHES "^nvptx")
-  set(FLANG_RT_DEVICE_ARCHITECTURES "all" CACHE STRING
-      "List of offload device architectures to be used to compile the Fortran runtime (e.g. 'gfx1103;sm_90')"
+  set(FLANG_RT_DEVICE_ARCHITECTURES "${RUNTIMES_DEVICE_ARCHITECTURES}" CACHE STRING
+      "List of OpenMP device architectures to be used to compile the Fortran runtime (e.g. 'gfx1103;sm_90')"
     )
 
   if (FLANG_RT_DEVICE_ARCHITECTURES STREQUAL "all")
@@ -139,6 +133,8 @@ if (FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT STREQUAL "OpenMP" OR LLVM_TARGET_TRIPL
       set(FLANG_RT_DEVICE_ARCHITECTURES ${all_gpu_architectures})
   endif()
   list(REMOVE_DUPLICATES FLANG_RT_DEVICE_ARCHITECTURES)
+else ()
+  message(FATAL_ERROR "Invalid value '${FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT}' for FLANG_RT_EXPERIMENTAL_OFFLOAD_SUPPORT; must be empty, 'CUDA', or 'OpenMP'")
 endif ()
 
 

flang-rt/cmake/modules/AddFlangRT.cmake

@@ -234,14 +234,11 @@ function (add_flangrt_library name)
     target_compile_features(${tgtname} PRIVATE cxx_std_17)
 
     target_compile_options(${tgtname} PRIVATE
-      # Always enable preprocessor regardless of file extention
+      # Always enable preprocessor regardless of file extension
       "$<$<COMPILE_LANGUAGE:Fortran>:-cpp>"
 
-      # Missing type descriptors are expected for intrinsic modules 
+      # Missing type descriptors are expected for intrinsic modules
       "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
-
-      # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
-      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
     )
 
     # When building the flang runtime if LTO is enabled the archive file
@@ -300,11 +297,6 @@ function (add_flangrt_library name)
       target_compile_options(${tgtname} PRIVATE
           $<$<COMPILE_LANGUAGE:CXX>:-nogpulib -flto -fvisibility=hidden -Wno-unknown-cuda-version --cuda-feature=+ptx63>
         )
-      foreach (_arch IN LISTS FLANG_RT_DEVICE_ARCHITECTURES)
-        target_compile_options(${tgtname} PRIVATE
-         -march=${_arch}
-        )
-       endforeach()
     elseif (APPLE)
       # Clang on Darwin enables non-POSIX extensions by default.
       # This causes some macros to leak, such as HUGE from <math.h>, which

flang-rt/cmake/modules/AddFlangRTOffload.cmake

@@ -106,7 +106,8 @@ macro(enable_omp_offload_compilation name files)
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
 
-     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     # If building flang-rt together with libomp, ensure that libomp is built
+     # first and found because -fopenmp will try to link it.
      if (TARGET omp)
        add_dependencies(${name} omp)
        target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")

openmp/CMakeLists.txt

@@ -95,6 +95,24 @@ endif()
 # Check and set up common compiler flags.
 include(config-ix)
 include(HandleOpenMPOptions)
+include(LibompUtils)
+
+
+# Set libomp version
+set(LIBOMP_VERSION_MAJOR 5)
+set(LIBOMP_VERSION_MINOR 0)
+
+# Set the OpenMP Year and Month associated with version
+set(LIBOMP_OMP_YEAR_MONTH 201611)
+
+# Get the build number from kmp_version.cpp
+libomp_get_build_number("${CMAKE_CURRENT_SOURCE_DIR}/runtime" LIBOMP_VERSION_BUILD)
+math(EXPR LIBOMP_VERSION_BUILD_YEAR "${LIBOMP_VERSION_BUILD}/10000")
+math(EXPR LIBOMP_VERSION_BUILD_MONTH_DAY "${LIBOMP_VERSION_BUILD}%10000")
+
+# Currently don't record any timestamps
+set(LIBOMP_BUILD_DATE "No_Timestamp")
+
 
 # Check for flang
 set(OPENMP_TEST_Fortran_COMPILER_default "")
@@ -104,6 +122,13 @@ endif ()
 set(OPENMP_TEST_Fortran_COMPILER "${OPENMP_TEST_Fortran_COMPILER_default}" CACHE STRING
   "Fortran compiler to use for testing OpenMP runtime libraries.")
 
+if (LLVM_RUNTIMES_BUILD)
+  set(LIBOMP_FORTRAN_MODULES "${RUNTIMES_FLANG_MODULES_ENABLED}" CACHE BOOL
+    "Create Fortran module files? (requires fortran compiler)")
+else ()
+  set(LIBOMP_FORTRAN_MODULES OFF)
+endif ()
+
 # Set up testing infrastructure.
 include(OpenMPTesting)
 
@@ -136,6 +161,10 @@ else()
   get_clang_resource_dir(LIBOMP_HEADERS_INSTALL_PATH SUBDIR include)
 endif()
 
+if(LIBOMP_FORTRAN_MODULES)
+  add_subdirectory(module)
+endif()
+
 # Use the current compiler target to determine the appropriate runtime to build.
 if("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx" OR
    "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^amdgcn|^nvptx")

openmp/runtime/cmake/LibompUtils.cmake renamed to openmp/cmake/LibompUtils.cmake

@@ -0,0 +1,30 @@
+#//===----------------------------------------------------------------------===//
+#//
+#// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#// See https://llvm.org/LICENSE.txt for license information.
+#// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#//
+#//===----------------------------------------------------------------------===//
+
+# Build the module files if a Fortran compiler is available.
+# Only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp
+# has been deprecated.
+
+configure_file(omp_lib.F90.var "{CMAKE_CURRENT_BINARY_DIR}/omp_lib.F90" @ONLY)
+
+# One compilation step creates both omp_lib.mod and omp_lib_kinds.mod. Only
+# these files are used, the object file itself can be discarded.
+# FIXME: Adding it to libomp.so would allow implementing Fortran API in Fortran
+add_library(libomp-mod OBJECT
+  "{CMAKE_CURRENT_BINARY_DIR}/omp_lib.F90"
+)
+set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+ target_compile_options(libomp-mod PRIVATE -fno-range-check)
+endif()
+
+flang_module_target(libomp-mod PUBLIC)
+if (FORTRAN_MODULE_DEPS)
+  add_dependencies(libomp-mod ${FORTRAN_MODULE_DEPS})
+endif ()

openmp/module/CMakeLists.txt

@@ -15,10 +15,6 @@ endif()
 # Add cmake directory to search for custom cmake functions
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
 
-# Set libomp version
-set(LIBOMP_VERSION_MAJOR 5)
-set(LIBOMP_VERSION_MINOR 0)
-
 # These include files are in the cmake/ subdirectory
 include(LibompUtils)
 include(LibompGetArchitecture)
@@ -102,21 +98,12 @@ libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc ppc64 ppc64le aarch
 set(LIBOMP_LIB_TYPE normal CACHE STRING
   "Performance,Profiling,Stubs library (normal/profile/stubs)")
 libomp_check_variable(LIBOMP_LIB_TYPE normal profile stubs)
-# Set the OpenMP Year and Month associated with version
-set(LIBOMP_OMP_YEAR_MONTH 201611)
 set(LIBOMP_MIC_ARCH knc CACHE STRING
   "Intel(R) Many Integrated Core Architecture (Intel(R) MIC Architecture) (knf/knc).  Ignored if not Intel(R) MIC Architecture build.")
 if("${LIBOMP_ARCH}" STREQUAL "mic")
   libomp_check_variable(LIBOMP_MIC_ARCH knf knc)
 endif()
 
-if (LLVM_RUNTIMES_BUILD)
-  set(LIBOMP_FORTRAN_MODULES "${RUNTIMES_FLANG_MODULES_ENABLED}" CACHE BOOL
-    "Create Fortran module files? (requires fortran compiler)")
-else ()
-  set(LIBOMP_FORTRAN_MODULES OFF)
-endif ()
-
 
 # - Support for universal fat binary builds on Mac
 # - Having this extra variable allows people to build this library as a universal library
@@ -167,14 +154,6 @@ set(LIBOMP_USE_HWLOC FALSE CACHE BOOL
 set(LIBOMP_HWLOC_INSTALL_DIR /usr/local CACHE PATH
   "Install path for hwloc library")
 
-# Get the build number from kmp_version.cpp
-libomp_get_build_number("${CMAKE_CURRENT_SOURCE_DIR}" LIBOMP_VERSION_BUILD)
-math(EXPR LIBOMP_VERSION_BUILD_YEAR "${LIBOMP_VERSION_BUILD}/10000")
-math(EXPR LIBOMP_VERSION_BUILD_MONTH_DAY "${LIBOMP_VERSION_BUILD}%10000")
-
-# Currently don't record any timestamps
-set(LIBOMP_BUILD_DATE "No_Timestamp")
-
 # Architecture
 set(IA32 FALSE)
 set(INTEL64 FALSE)
@@ -477,3 +456,5 @@ set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${LIBOMP_OMP_TOOLS_INCLUDE_DIR} PARENT_SCOPE)
 # make these variables available for tools/libompd:
 set(LIBOMP_SRC_DIR ${LIBOMP_SRC_DIR} PARENT_SCOPE)
 set(LIBOMP_OMPD_SUPPORT ${LIBOMP_OMPD_SUPPORT} PARENT_SCOPE)
+
+

openmp/runtime/src/include/omp_lib.F90.var renamed to openmp/module/omp_lib.F90.var

@@ -65,12 +65,14 @@ if(${LIBOMP_OMPT_SUPPORT})
   )
 endif()
 if(${LIBOMP_FORTRAN_MODULES})
-  add_custom_command(TARGET libomp-mod POST_BUILD
+  # We cannot attach a POST_BUILD command to libomp-mod, so instead attach it
+  # to omp and ensure that libomp-mod is built befoe by adding a dependency
+  add_custom_command(TARGET omp POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E make_directory ${LIBOMP_EXPORTS_MOD_DIR}
     COMMAND ${CMAKE_COMMAND} -E copy ${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.mod ${LIBOMP_EXPORTS_MOD_DIR}
     COMMAND ${CMAKE_COMMAND} -E copy ${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib_kinds.mod ${LIBOMP_EXPORTS_MOD_DIR}
   )
-
+  add_dependencies(omp libomp-mod)
   add_custom_command(TARGET omp POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy omp_lib.h ${LIBOMP_EXPORTS_CMN_DIR}
   )

openmp/runtime/CMakeLists.txt

@@ -377,6 +377,13 @@ if(WIN32)
   endif()
 endif()
 
+configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
+
+# Move files to exports/ directory if requested
+if(${LIBOMP_COPY_EXPORTS})
+  include(LibompExports)
+endif()
+
 # Micro test rules for after library has been built (cmake/LibompMicroTests.cmake)
 include(LibompMicroTests)
 add_custom_target(libomp-micro-tests)
@@ -452,35 +459,7 @@ if(${LIBOMP_OMPT_SUPPORT})
   install(FILES ${LIBOMP_HEADERS_INTDIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
   set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 endif()
-
-
-# Build the module files if a Fortran compiler is available.
-# Only LLVM_ENABLE_RUNTIMES=openmp is supported, LLVM_ENABLE_PROJECTS=openmp
-# has been deprecated.
-# TODO: openmp/runtimes/CMakeLists.txt is only included for host
-#       (non-amdgpu/nvptx) targets, but device side code needs a module as well.
 if(LIBOMP_FORTRAN_MODULES)
-  configure_file(${LIBOMP_INC_DIR}/omp_lib.h.var omp_lib.h @ONLY)
-  configure_file(${LIBOMP_INC_DIR}/omp_lib.F90.var omp_lib.F90 @ONLY)
-
-  # One compilation step creates both omp_lib.mod and omp_lib_kinds.mod. Only
-  # these files are used, the library itself can be discarded. Cannot use an
-  # OBJECT library to avoid creating the library because LibompExports.cmake
-  # attaches a POST_BUILD command to libomp-mod.
-  add_library(libomp-mod STATIC
-    omp_lib.F90
-  )
-  set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
-
-  if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-   target_compile_options(libomp-mod PRIVATE -fno-range-check)
-  endif()
-
-  flang_module_target(libomp-mod PUBLIC)
-  if (FORTRAN_MODULE_DEPS)
-    add_dependencies(libomp-mod ${FORTRAN_MODULE_DEPS})
-  endif ()
-
   set (destination ${LIBOMP_HEADERS_INSTALL_PATH})
   if (NOT ${LIBOMP_MODULES_INSTALL_PATH} STREQUAL "")
     set (destination ${LIBOMP_MODULES_INSTALL_PATH})
@@ -490,8 +469,3 @@ if(LIBOMP_FORTRAN_MODULES)
     DESTINATION ${destination}
   )
 endif()
-
-# Move files to exports/ directory if requested
-if(LIBOMP_COPY_EXPORTS)
-  include(LibompExports)
-endif()