[Offload] Enable unified ROCr in external project build

With ROCm 6.3 the ROCr runtime includes the ROCt, with some changes to
the directory structure and where build artifacts are placed.

The CMake variable OFFLOAD_EXTERNAL_PROJECT_UNIFIED_ROCR is introduced
to enable the search paths for this unified ROCr when enabled as an
external project.
This builds ROCr as part of the compilder build and resolves the
dependency issue offload otherwise hits.

The ExternalProject mechanism is what AOMP uses today. However, ROCm 6.2
does still have separate ROCr and ROCt.

Change-Id: Ia0d8241a77ccb1048d546f0a57f43ed29e2c48ca

Author: jplehr

llvm/runtimes/CMakeLists.txt

@@ -503,6 +503,39 @@ if(build_runtimes)
   endif()
 
   if("offload" IN_LIST LLVM_ENABLE_RUNTIMES)
+    # With ROCm 6.3 the ROCr runtime and the thunk layer share a single repository.
+    # No need to provide a separate path for ROCt.
+    if (OFFLOAD_EXTERNAL_PROJECT_UNIFIED_ROCR)
+      if(NOT DEFINED LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH)
+        message(SEND_ERROR "External ROCr requires setting LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH")
+      endif()
+
+      message(STATUS "Add external unified ROCr: ${LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH}")
+      ExternalProject_Add(rocr-runtime
+        SOURCE_DIR ${LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH}
+        DEPENDS clang llvm-link lld opt llvm-objcopy
+        INSTALL_COMMAND ""
+        CMAKE_ARGS -DBUILD_SHARED_LIBS=ON
+                   -DCMAKE_PREFIX_PATH=${CMAKE_BINARY_DIR}/lib/cmake
+                   -DIMAGE_SUPPORT=OFF
+                   -DLLVM_RUNTIME_OPENMP=ON)
+      set(HSA_DEP rocr-runtime)
+
+      # omptarget device RTL depends on device libs, leading to circular dependency in build scripts.
+      # Providing path to the sources enables to build them as part of compiler build, which
+      # removes the ciruclar dependency on the script-side.
+      if (DEFINED LIBOMPTARGET_EXTERNAL_PROJECT_ROCM_DEVICE_LIBS_PATH)
+        message(STATUS "Add external AMD device-libs: ${LIBOMPTARGET_EXTERNAL_PROJECT_ROCM_DEVICE_LIBS_PATH}")
+        ExternalProject_Add(rocm-device-libs
+          SOURCE_DIR ${LIBOMPTARGET_EXTERNAL_PROJECT_ROCM_DEVICE_LIBS_PATH}
+          DEPENDS clang llvm-link lld opt llvm-objcopy
+          CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${ROCM_DEVICE_LIBS_INSTALL_PREFIX_PATH}
+                     -DCMAKE_PREFIX_PATH=${CMAKE_BINARY_DIR}/lib/cmake
+                     -DROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_NEW=${ROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC}
+                     -DROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_OLD=amdgcn)
+      endif()
+    endif()
+
     if(DEFINED LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH AND
        DEFINED LIBOMPTARGET_EXTERNAL_PROJECT_THUNK_PATH AND
        DEFINED LIBOMPTARGET_EXTERNAL_PROJECT_ROCM_DEVICE_LIBS_PATH)

offload/hostexec/CMakeLists.txt

@@ -18,20 +18,41 @@ endif()
 
 # If we are bootstrapping hsa via external project we need to use find_library
 # as it will not be installed.
-if(DEFINED LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH)
-  find_library(HSA_RUNTIME hsa-runtime64 HINTS ${CMAKE_BINARY_DIR}/../../runtimes/rocr-runtime-prefix/src/rocr-runtime-build ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)
+if(OFFLOAD_EXTERNAL_PROJECT_UNIFIED_ROCR)
+  # With ROCm 6.3 ROCr and ROCt are in one repo and we need to account for the different paths
+  find_library(HSA_RUNTIME hsa-runtime64
+    HINTS
+    ${CMAKE_BINARY_DIR}/../../runtimes/rocr-runtime-prefix/src/rocr-runtime-build/rocr/lib
+    ${CMAKE_INSTALL_PREFIX}
+    PATHS
+    /opt/rocm)
 else()
- # As of rocm-3.7, hsa is installed with cmake packages and kmt is found via hsa
-  find_package(hsa-runtime64 QUIET 1.2.0 HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)
+  if(DEFINED LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH)
+    find_library(HSA_RUNTIME hsa-runtime64
+      HINTS
+      ${CMAKE_BINARY_DIR}/../../runtimes/rocr-runtime-prefix/src/rocr-runtime-build
+      ${CMAKE_INSTALL_PREFIX}
+      PATHS
+      /opt/rocm)
+  else()
+    # As of rocm-3.7, hsa is installed with cmake packages and kmt is found via hsa
+    find_package(hsa-runtime64 QUIET 1.2.0 HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)
+  endif()
 endif()
 
 if(DEFINED HSA_RUNTIME)
   message(STATUS "Building hostexec for AMDGCN linked against libhsa")
-  set(HSA_INCLUDE ${LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH}/inc)
+  if(OFFLOAD_EXTERNAL_PROJECT_UNIFIED_ROCR)
+    set(HSA_INCLUDE ${LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH}/runtime/hsa-runtime/inc)
+  else()
+    set(HSA_INCLUDE ${LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH}/inc)
+  endif()
   set(HOSTEXEC_EXTRA_SOURCE)
   set(HOSTEXEC_DEP_LIBRARIES ${HSA_RUNTIME})
+  message(STATUS "HSA Runtime found: ${HSA_RUNTIME}")
+  message(STATUS "HSA Runtime include: ${HSA_INCLUDE}")
 elseif(${hsa-runtime64_FOUND})
-  message(STATUS "Building hostexec for AMDGCN linked against libhsa")
+  message(STATUS "Building hostexec for AMDGCN linked against libhsa64")
   set(HSA_INCLUDE ${hsa-runtime64_DIR}/../../../include/hsa)
   get_target_property(HSA_INCLUDE hsa-runtime64::hsa-runtime64 INTERFACE_INCLUDE_DIRECTORIES)
   set(HOSTEXEC_EXTRA_SOURCE)

offload/plugins-nextgen/amdgpu/CMakeLists.txt

@@ -9,13 +9,21 @@ endif()
 # If we are bootstrapping hsa via external project we need to use find_library
 # as it will not be installed.
 if(DEFINED LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH)
-  find_library(HSA_RUNTIME hsa-runtime64
-    HINTS
-    ${CMAKE_BINARY_DIR}/../../runtimes/rocr-runtime-prefix/src/rocr-runtime-build
-    ${CMAKE_INSTALL_PREFIX}
-    PATHS
-    ${LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH}
-  )
+  if(OFFLOAD_EXTERNAL_PROJECT_UNIFIED_ROCR)
+    find_library(HSA_RUNTIME hsa-runtime64
+      HINTS
+      ${CMAKE_BINARY_DIR}/../../runtimes/rocr-runtime-prefix/src/rocr-runtime-build/rocr/lib
+      ${CMAKE_INSTALL_PREFIX}
+      PATHS
+      /opt/rocm)
+  else()
+    find_library(HSA_RUNTIME hsa-runtime64
+      HINTS
+      ${CMAKE_BINARY_DIR}/../../runtimes/rocr-runtime-prefix/src/rocr-runtime-build
+      ${CMAKE_INSTALL_PREFIX}
+      PATHS
+      ${LIBOMPTARGET_EXTERNAL_PROJECT_HSA_PATH})
+  endif()
 else()
   # As of rocm-3.7, hsa is installed with cmake packages and kmt is found via hsa
   find_package(hsa-runtime64 QUIET 1.2.0 HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)