[Flang] Move builtin .mod generation into runtimes (#137828)

Move building the .mod files from openmp/flang to openmp/flang-rt using
a shared mechanism. Motivations to do so are:

1. Most modules are target-dependent and need to be re-compiled for each
target separately, which is something the LLVM_ENABLE_RUNTIMES system
already does. Prime example is `iso_c_binding.mod` which encodes the
target's ABI. Most other modules have `#ifdef`-enclosed code as well.

2. CMake has support for Fortran that we should use. Among other things,
it automatically determines module dependencies so there is no need to
hardcode them in the CMakeLists.txt.

3. It allows using Fortran itself to implement Flang-RT. Currently, only
`iso_fortran_env_impl.f90` emits object files that are needed by Fortran
applications (#89403). The workaround of #95388 could be reverted.


Some new dependencies come into play:
* openmp depends on flang-rt for building `lib_omp.mod` and
`lib_omp_kinds.mod`. Currently, if flang-rt is not found then the
modules are not built.
* check-flang depends on flang-rt: If not found, the majority of tests
are disabled. If not building in a bootstrpping build, the location of
the module files can be pointed to using
`-DFLANG_INTRINSIC_MODULES_DIR=<path>`, e.g. in a flang-standalone
build. Alternatively, the test needing any of the intrinsic modules
could be marked with `REQUIRES: flangrt-modules`.
* check-flang depends on openmp: Not a change; tests requiring
`lib_omp.mod` and `lib_omp_kinds.mod` those are already marked with
`openmp_runtime`.

As intrinsic are now specific to the target, their location is moved
from `include/flang` to `<resource-dir>/finclude/flang/<triple>`. The
mechnism to compute the location have been moved from flang-rt
(previously used to compute the location of `libflang_rt.*.a`) to common
locations in `cmake/GetToolchainDirs.cmake` and
`runtimes/CMakeLists.txt` so they can be used by both, openmp and
flang-rt. Potentially the mechnism could also be shared by other
libraries such as compiler-rt.

`finclude` was chosen because `gfortran` uses it as well and avoids
misuse such as `#include <flang/iso_c_binding.mod>`. The search location
is now determined by `ToolChain` in the driver, instead of by the
frontend. Now the driver adds `-fintrinsic-module-path` for that
location to the frontend call (Just like gfortran does).
`-fintrinsic-module-path` had to be fixed for this because ironically it
was only added to `searchDirectories`, but not
`intrinsicModuleDirectories_`. Since the driver determines the location,
tests invoking `flang -fc1` and `bbc` must also be passed the location
by llvm-lit. This works like llvm-lit does for finding the include dirs
for Clang using `-print-file-name=...`.

Author: Meinersbur

clang/include/clang/Driver/ToolChain.h

@@ -538,6 +538,10 @@ class ToolChain {
   // Returns Triple without the OSs version.
   llvm::Triple getTripleWithoutOSVersion() const;
 
+  /// Returns the target-specific path for Flang's intrinsic modules in the
+  /// resource directory if it exists.
+  std::optional<std::string> getDefaultIntrinsicModuleDir() const;
+
   // Returns the target specific runtime path if it exists.
   std::optional<std::string> getRuntimePath() const;
 

clang/include/clang/Options/Options.td

@@ -6082,7 +6082,7 @@ def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
   HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
-  Visibility<[ClangOption, CLOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,

clang/lib/Driver/Driver.cpp

@@ -6602,6 +6602,17 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   if (llvm::sys::fs::exists(Twine(P)))
     return std::string(P);
 
+  // With Flang, also look for instrinsic modules
+  if (IsFlangMode()) {
+    if (std::optional<std::string> IntrPath =
+            TC.getDefaultIntrinsicModuleDir()) {
+      SmallString<128> P(*IntrPath);
+      llvm::sys::path::append(P, Name);
+      if (llvm::sys::fs::exists(Twine(P)))
+        return std::string(P);
+    }
+  }
+
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))

clang/lib/Driver/ToolChain.cpp

@@ -1020,6 +1020,12 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
   return {};
 }
 
+std::optional<std::string> ToolChain::getDefaultIntrinsicModuleDir() const {
+  SmallString<128> P(D.ResourceDir);
+  llvm::sys::path::append(P, "finclude", "flang");
+  return getTargetSubDirPath(P);
+}
+
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");

clang/lib/Driver/ToolChains/Flang.cpp

@@ -1060,6 +1060,14 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  // Default intrinsic module dirs must be added after any user-provided
+  // -fintrinsic-modules-path to have lower precedence
+  if (std::optional<std::string> IntrModPath =
+          TC.getDefaultIntrinsicModuleDir()) {
+    CmdArgs.push_back("-fintrinsic-modules-path");
+    CmdArgs.push_back(Args.MakeArgString(*IntrModPath));
+  }
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 

flang-rt/cmake/modules/GetToolchainDirs.cmake renamed to cmake/Modules/GetToolchainDirs.cmake

@@ -47,6 +47,17 @@ function (get_toolchain_library_subdir outvar)
 endfunction ()
 
 
+# Corresponds to Flang's ToolChain::getDefaultIntrinsicModuleDir().
+function (get_toolchain_module_subdir outvar)
+  set(outval "finclude/flang")
+
+  get_toolchain_arch_dirname(arch_dirname)
+  set(outval "${outval}/${arch_dirname}")
+
+  set(${outvar} "${outval}" PARENT_SCOPE)
+endfunction ()
+
+
 # Corresponds to Clang's ToolChain::getOSLibName(). Adapted from Compiler-RT.
 function (get_toolchain_os_dirname outvar)
   if (ANDROID)

flang-rt/CMakeLists.txt

@@ -23,40 +23,6 @@ set(FLANG_RT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FLANG_RT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(FLANG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../flang")
 
-# CMake 3.24 is the first version of CMake that directly recognizes Flang.
-# LLVM's requirement is only CMake 3.20, teach CMake 3.20-3.23 how to use Flang.
-if (CMAKE_VERSION VERSION_LESS "3.24")
-  cmake_path(GET CMAKE_Fortran_COMPILER STEM _Fortran_COMPILER_STEM)
-  if (_Fortran_COMPILER_STEM STREQUAL "flang-new" OR _Fortran_COMPILER_STEM STREQUAL "flang")
-    include(CMakeForceCompiler)
-    CMAKE_FORCE_Fortran_COMPILER("${CMAKE_Fortran_COMPILER}" "LLVMFlang")
-
-    set(CMAKE_Fortran_COMPILER_ID "LLVMFlang")
-    set(CMAKE_Fortran_COMPILER_VERSION "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}")
-
-    set(CMAKE_Fortran_SUBMODULE_SEP "-")
-    set(CMAKE_Fortran_SUBMODULE_EXT ".mod")
-
-    set(CMAKE_Fortran_PREPROCESS_SOURCE
-      "<CMAKE_Fortran_COMPILER> -cpp <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
-
-    set(CMAKE_Fortran_FORMAT_FIXED_FLAG "-ffixed-form")
-    set(CMAKE_Fortran_FORMAT_FREE_FLAG "-ffree-form")
-
-    set(CMAKE_Fortran_MODDIR_FLAG "-module-dir")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_ON "-cpp")
-    set(CMAKE_Fortran_COMPILE_OPTIONS_PREPROCESS_OFF "-nocpp")
-    set(CMAKE_Fortran_POSTPROCESS_FLAG "-ffixed-line-length-72")
-
-    set(CMAKE_Fortran_COMPILE_OPTIONS_TARGET "--target=")
-
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG "-Wl,")
-    set(CMAKE_Fortran_LINKER_WRAPPER_FLAG_SEP ",")
-  endif ()
-endif ()
-enable_language(Fortran)
-
 
 list(APPEND CMAKE_MODULE_PATH
     "${FLANG_RT_SOURCE_DIR}/cmake/modules"
@@ -65,69 +31,24 @@ list(APPEND CMAKE_MODULE_PATH
 include(AddFlangRT)
 include(GetToolchainDirs)
 include(FlangCommon)
+include(FlangRTIntrospection)
 include(HandleCompilerRT)
 include(ExtendPath)
+include(CheckFortranSourceCompiles)
+include(CMakePushCheckState)
 
 
 ############################
 # Build Mode Introspection #
 ############################
 
-# Determine whether we are in the runtimes/runtimes-bins directory of a
-# bootstrap build.
-set(LLVM_TREE_AVAILABLE OFF)
-if (LLVM_LIBRARY_OUTPUT_INTDIR AND LLVM_RUNTIME_OUTPUT_INTDIR AND PACKAGE_VERSION)
-  set(LLVM_TREE_AVAILABLE ON)
-endif()
-
 # Path to LLVM development tools (FileCheck, llvm-lit, not, ...)
 set(LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}/bin")
 
-# Determine build and install paths.
-# The build path is absolute, but the install dir is relative, CMake's install
-# command has to apply CMAKE_INSTALL_PREFIX itself.
-get_toolchain_library_subdir(toolchain_lib_subdir)
-if (LLVM_TREE_AVAILABLE)
-  # In a bootstrap build emit the libraries into a default search path in the
-  # build directory of the just-built compiler. This allows using the
-  # just-built compiler without specifying paths to runtime libraries.
-  #
-  # Despite Clang in the name, get_clang_resource_dir does not depend on Clang
-  # being added to the build. Flang uses the same resource dir as clang.
-  include(GetClangResourceDir)
-  get_clang_resource_dir(FLANG_RT_OUTPUT_RESOURCE_DIR PREFIX "${LLVM_LIBRARY_OUTPUT_INTDIR}/..")
-  get_clang_resource_dir(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT)
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "${toolchain_lib_subdir}")
-else ()
-  # In a standalone runtimes build, do not write into LLVM_BINARY_DIR. It may be
-  # read-only and/or shared by multiple runtimes with different build
-  # configurations (e.g. Debug/Release). Use the runtime's own lib dir like any
-  # non-toolchain library.
-  # For the install prefix, still use the resource dir assuming that Flang will
-  # be installed there using the same prefix. This is to not have a difference
-  # between bootstrap and standalone runtimes builds.
-  set(FLANG_RT_OUTPUT_RESOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
-  set(FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT "lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}")
-
-  extend_path(FLANG_RT_OUTPUT_RESOURCE_LIB_DIR "${FLANG_RT_OUTPUT_RESOURCE_DIR}" "lib${LLVM_LIBDIR_SUFFIX}")
-endif ()
-set(FLANG_RT_INSTALL_RESOURCE_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH_DEFAULT}"
-    CACHE PATH "Path to install runtime libraries to (default: clang resource dir)")
-extend_path(FLANG_RT_INSTALL_RESOURCE_LIB_PATH "${FLANG_RT_INSTALL_RESOURCE_PATH}" "${toolchain_lib_subdir}")
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_PATH)
-# FIXME: For the libflang_rt.so, the toolchain resource lib dir is not a good
-#        destination because it is not a ld.so default search path.
-#        The machine where the executable is eventually executed may not be the
-#        machine where the Flang compiler and its resource dir is installed, so
-#        setting RPath by the driver is not an solution. It should belong into
-#        /usr/lib/<triple>/libflang_rt.so, like e.g. libgcc_s.so.
-#        But the linker as invoked by the Flang driver also requires
-#        libflang_rt.so to be found when linking and the resource lib dir is
-#        the only reliable location.
-cmake_path(NORMAL_PATH FLANG_RT_OUTPUT_RESOURCE_LIB_DIR)
-cmake_path(NORMAL_PATH FLANG_RT_INSTALL_RESOURCE_LIB_PATH)
+# Fortran compiler not optional for building Flang-RT
+enable_language(Fortran)
+
+flang_module_fortran_enable()
 
 
 #################
@@ -234,6 +155,10 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+# Look for support of REAL(16), if not already defined via command
+# line via -DFORTRAN_SUPPORTS_REAL16=YES/NO
+check_fortran_quadmath_support()
+
 # Search for clang_rt.builtins library. Need in addition to msvcrt.
 if (WIN32)
   find_compiler_rt_library(builtins FLANG_RT_BUILTINS_LIBRARY)

flang-rt/cmake/modules/AddFlangRT.cmake

@@ -190,6 +190,12 @@ function (add_flangrt_library name)
     endif ()
   endif ()
 
+  if (build_object)
+    add_library(${name}.compile ALIAS "${name_object}")
+  else ()
+    add_library(${name}.compile ALIAS "${default_target}")
+  endif ()
+
   foreach (tgtname IN LISTS libtargets)
     if (NOT WIN32)
       # Use same stem name for .a and .so. Common in UNIX environments.
@@ -219,13 +225,28 @@ function (add_flangrt_library name)
     # Minimum required C++ version for Flang-RT, even if CMAKE_CXX_STANDARD is defined to something else.
     target_compile_features(${tgtname} PRIVATE cxx_std_17)
 
+    target_compile_options(${tgtname} PRIVATE
+      # Always enable preprocessor regardless of file extention
+      "$<$<COMPILE_LANGUAGE:Fortran>:-cpp>"
+
+      # Missing type descriptors are expected for intrinsic modules 
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-mmlir;SHELL:-ignore-missing-type-desc>"
+
+      # Flang bug workaround: Reformating of cooked token buffer causes identifier to be split between lines
+      "$<$<COMPILE_LANGUAGE:Fortran>:SHELL:-Xflang;SHELL:-fno-reformat>"
+    )
+
     # When building the flang runtime if LTO is enabled the archive file
     # contains LLVM IR rather than object code. Currently flang is not
     # LTO aware so cannot link this file to compiled Fortran code.
     if (FLANG_RT_HAS_FNO_LTO_FLAG)
       target_compile_options(${tgtname} PRIVATE -fno-lto)
     endif ()
 
+    if (FORTRAN_SUPPORTS_REAL16)
+      target_compile_definitions(${tgtname} PRIVATE FLANG_SUPPORT_R16=1)
+    endif ()
+
     # Use compiler-specific options to disable exceptions and RTTI.
     if (LLVM_COMPILER_IS_GCC_COMPATIBLE)
       target_compile_options(${tgtname} PRIVATE
@@ -344,13 +365,13 @@ function (add_flangrt_library name)
     if (ARG_INSTALL_WITH_TOOLCHAIN)
       set_target_properties(${tgtname}
         PROPERTIES
-          ARCHIVE_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
-          LIBRARY_OUTPUT_DIRECTORY "${FLANG_RT_OUTPUT_RESOURCE_LIB_DIR}"
+          ARCHIVE_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
+          LIBRARY_OUTPUT_DIRECTORY "${RUNTIMES_OUTPUT_RESOURCE_LIB_DIR}"
         )
 
       install(TARGETS ${tgtname}
-          ARCHIVE DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
-          LIBRARY DESTINATION "${FLANG_RT_INSTALL_RESOURCE_LIB_PATH}"
+          ARCHIVE DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
+          LIBRARY DESTINATION "${RUNTIMES_INSTALL_RESOURCE_LIB_PATH}"
         )
     endif ()
 

flang-rt/cmake/modules/AddFlangRTOffload.cmake

@@ -88,23 +88,29 @@ macro(enable_omp_offload_compilation name files)
         "${FLANG_RT_DEVICE_ARCHITECTURES}"
         )
 
-      set(OMP_COMPILE_OPTIONS
+      set(OMP_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:C,CXX>:
         -fopenmp
         -fvisibility=hidden
         -fopenmp-cuda-mode
         --offload-arch=${compile_for_architectures}
         # Force LTO for the device part.
         -foffload-lto
-        )
-      set_source_files_properties(${files} PROPERTIES COMPILE_OPTIONS
-        "${OMP_COMPILE_OPTIONS}"
+        >)
+      set_property(SOURCE ${files} APPEND
+        PROPERTY COMPILE_DEFINITIONS ${OMP_COMPILE_OPTIONS}
         )
       target_link_options(${name}.static PUBLIC ${OMP_COMPILE_OPTIONS})
 
       # Enable "declare target" in the source code.
       set_source_files_properties(${files}
         PROPERTIES COMPILE_DEFINITIONS OMP_OFFLOAD_BUILD
         )
+
+     # If building flang-rt together with libomp, ensure that libomp is built first and found because -fopenmp will try to link it.
+     if (TARGET omp)
+       add_dependencies(${name} omp)
+       target_link_options(${name}.static PUBLIC "-L$<TARGET_FILE_DIR:omp>")
+     endif ()
     else()
       message(FATAL_ERROR
         "Flang-rt build with OpenMP offload is not supported for these compilers:\n"

flang-rt/cmake/modules/FlangRTIntrospection.cmake

@@ -0,0 +1,37 @@
+#===-- cmake/modules/FlangRTIntrospection.cmake ----------------------------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===------------------------------------------------------------------------===#
+
+
+# Check whether the Fortran compiler supports real(16)/quadmath types
+#
+# Implementation notes:
+#  * FORTRAN_SUPPORTS_REAL16 can be set externally in a bootstrapping-runtimes
+#    build to ensure consistency of real(16) support between compiler and
+#    runtime.
+#
+#  * Does not work with Flang and CMake < 3.24
+#
+#  * This is intentionally wrapped in a function to get its own namespace for
+#    CMAKE_REQUIRED_FLAGS and CMAKE_TRY_COMPILE_TARGET_TYPE. In particular,
+#    cmake_pop_check_state() does not reset CMAKE_TRY_COMPILE_TARGET_TYPE,
+#    causing later try_compile invocations to fail. If you see
+#    enable_language(CUDA) failing because CMAKE_RANLIB is empty, this is the
+#    reason.
+function (check_fortran_quadmath_support)
+  cmake_push_check_state(RESET)
+  set(CMAKE_REQUIRED_FLAGS "-ffree-form")
+  set(CMAKE_TRY_COMPILE_TARGET_TYPE "STATIC_LIBRARY") # Skip link step
+  check_fortran_source_compiles([[
+      subroutine test_quadmath
+        real(16) :: var1
+      end
+    ]]
+    FORTRAN_SUPPORTS_REAL16
+  )
+  cmake_pop_check_state()
+endfunction ()