diff --git a/buildbot/configure.py b/buildbot/configure.py index 4dcbd85a355c6..b2f9a9805976f 100644 --- a/buildbot/configure.py +++ b/buildbot/configure.py @@ -82,6 +82,11 @@ def do_configure(args, passthrough_args): if libclc_enabled: llvm_enable_projects += ";libclc" + # DeviceRTL uses -fuse-ld=lld, so enable lld. + if args.offload: + llvm_enable_projects += ";lld" + sycl_enabled_backends.append("offload") + if args.cuda: llvm_targets_to_build += ";NVPTX" libclc_targets_to_build = libclc_nvidia_target_names @@ -210,6 +215,12 @@ def do_configure(args, passthrough_args): "-DSYCL_ENABLE_MAJOR_RELEASE_PREVIEW_LIB={}".format(sycl_preview_lib), "-DBUG_REPORT_URL=https://github.com/intel/llvm/issues", ] + if args.offload: + cmake_cmd.extend( + [ + "-DUR_BUILD_ADAPTER_OFFLOAD=ON", + ] + ) if libclc_enabled: cmake_cmd.extend( @@ -340,6 +351,11 @@ def main(): default="AMD", help="choose hardware platform for HIP backend", ) + parser.add_argument( + "--offload", + action="store_true", + help="Enable UR liboffload adapter (experimental)", + ) parser.add_argument( "--level_zero_adapter_version", type=str, diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt index f722f35ad1be3..25d437f46f201 100644 --- a/sycl/CMakeLists.txt +++ b/sycl/CMakeLists.txt @@ -532,6 +532,15 @@ if("hip" IN_LIST SYCL_ENABLE_BACKENDS) list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS ur_adapter_hip) endif() +if("offload" IN_LIST SYCL_ENABLE_BACKENDS) + if(NOT TARGET lld) + message(FATAL_ERROR + "Offload support requires adding \"lld\" to the CMake argument \"LLVM_ENABLE_PROJECTS\"") + endif() + add_dependencies(sycl-toolchain ur_adapter_offload) + list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS ur_adapter_offload) +endif() + # Use it as fake dependency in order to force another command(s) to execute. add_custom_command(OUTPUT __force_it COMMAND "${CMAKE_COMMAND}" -E echo diff --git a/sycl/cmake/modules/BuildUnifiedRuntime.cmake b/sycl/cmake/modules/BuildUnifiedRuntime.cmake index e90eb7887d14d..e0fce1bf48ad0 100644 --- a/sycl/cmake/modules/BuildUnifiedRuntime.cmake +++ b/sycl/cmake/modules/BuildUnifiedRuntime.cmake @@ -175,6 +175,10 @@ if("native_cpu" IN_LIST SYCL_ENABLE_BACKENDS) endif() endif() +if("offload" IN_LIST SYCL_ENABLE_BACKENDS) + add_sycl_ur_adapter(offload) +endif() + if(CMAKE_SYSTEM_NAME STREQUAL Windows) # On Windows, also build/install debug libraries with the d suffix that are # compiled with /MDd so users can link against these in debug builds. diff --git a/unified-runtime/source/adapters/offload/CMakeLists.txt b/unified-runtime/source/adapters/offload/CMakeLists.txt index 9411139a3e2d4..f148930b59294 100644 --- a/unified-runtime/source/adapters/offload/CMakeLists.txt +++ b/unified-runtime/source/adapters/offload/CMakeLists.txt @@ -5,14 +5,103 @@ set(TARGET_NAME ur_adapter_offload) -set(UR_OFFLOAD_INSTALL_DIR "" CACHE PATH "Path to the directory containing libomptarget.so etc") -if (UR_OFFLOAD_INSTALL_DIR STREQUAL "") - message(FATAL_ERROR "UR_OFFLOAD_INSTALL_DIR must be defined for the Offload adapter") -endif() +add_ur_adapter(${TARGET_NAME} + SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/context.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/enqueue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/event.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ur2offload.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp +) +set(UR_OFFLOAD_INSTALL_DIR "" CACHE PATH "Path to the directory containing libomptarget.so etc") set(UR_OFFLOAD_INCLUDE_DIR "" CACHE PATH "Path to the directory containing LLVM headers") -if (UR_OFFLOAD_INCLUDE_DIR STREQUAL "") - message(FATAL_ERROR "UR_OFFLOAD_INCLUDE_DIR must be defined for the Offload adapter") +if (UR_OFFLOAD_INSTALL_DIR STREQUAL "" OR UR_OFFLOAD_INCLUDE_DIR STREQUAL "") + include(ExternalProject) + set(LLVM_PROJECT_SOURCE_DIR ${CMAKE_BINARY_DIR}/llvm-src-offload) + set(LLVM_PROJECT_TAG 32beea0605f37ea7a6429375d41b19ee78ddfe7d) + set(OPENMP_INSTALL_DIR ${CMAKE_BINARY_DIR}/openmp-install) + set(UR_OFFLOAD_INSTALL_DIR ${CMAKE_BINARY_DIR}/offload-install) + set(UR_OFFLOAD_INCLUDE_DIR ${UR_OFFLOAD_INSTALL_DIR}/include) + + execute_process(COMMAND git -C "${CMAKE_SOURCE_DIR}" worktree prune) + + if(NOT IS_DIRECTORY "${LLVM_PROJECT_SOURCE_DIR}") + execute_process( + COMMAND git -C "${CMAKE_SOURCE_DIR}" worktree add --no-checkout --detach "${LLVM_PROJECT_SOURCE_DIR}" "${LLVM_PROJECT_TAG}" + COMMAND_ERROR_IS_FATAL ANY + ) + endif() + + execute_process( + COMMAND git sparse-checkout init --cone + WORKING_DIRECTORY "${LLVM_PROJECT_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY + ) + + execute_process( + COMMAND git sparse-checkout set openmp offload cmake llvm/include libc + WORKING_DIRECTORY "${LLVM_PROJECT_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY + ) + + execute_process( + COMMAND git checkout "${LLVM_PROJECT_TAG}" + WORKING_DIRECTORY "${LLVM_PROJECT_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY + ) + + # Build OpenMP runtime (required dependency for offload's libomptarget) from the cloned source + ExternalProject_Add(openmp_ext + # DeviceRTL uses -fuse-ld=lld, so add lld to the dependencies. + DEPENDS llvm-tblgen LLVMSupport clang lld FileCheck not + SOURCE_DIR ${LLVM_PROJECT_SOURCE_DIR}/openmp + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${OPENMP_INSTALL_DIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_C_COMPILER=${CMAKE_BINARY_DIR}/bin/clang + -DCMAKE_CXX_COMPILER=${CMAKE_BINARY_DIR}/bin/clang++ + -DLIBOMP_OMPD_GDB_SUPPORT=OFF + -DOPENMP_ENABLE_OMPT_TOOLS=OFF + -DCMAKE_PREFIX_PATH=${CMAKE_BINARY_DIR}/bin + INSTALL_COMMAND ${CMAKE_COMMAND} --build . --target install + UPDATE_COMMAND "" + DOWNLOAD_COMMAND "" + ) + + # Build liboffload runtime from the same source tree + ExternalProject_Add(offload_ext + DEPENDS openmp_ext + SOURCE_DIR ${LLVM_PROJECT_SOURCE_DIR}/offload + LIST_SEPARATOR | + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${UR_OFFLOAD_INSTALL_DIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_C_COMPILER=${CMAKE_BINARY_DIR}/bin/clang + -DCMAKE_CXX_COMPILER=${CMAKE_BINARY_DIR}/bin/clang++ + -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=${LLVM_PROJECT_SOURCE_DIR}/llvm/include|${CMAKE_BINARY_DIR}/include + -DLLVM_DIR=${CMAKE_BINARY_DIR}/lib/cmake/llvm + -DLIBOMPTARGET_PLUGINS_TO_BUILD=cuda|amdgpu + -DLIBOMP_INCLUDE_DIR=${OPENMP_INSTALL_DIR}/include + -DLLVM_TABLEGEN=${CMAKE_BINARY_DIR}/bin/llvm-tblgen + -DCMAKE_PREFIX_PATH=${OPENMP_INSTALL_DIR}|${CMAKE_BINARY_DIR}/bin + INSTALL_COMMAND ${CMAKE_COMMAND} --build . --target install + UPDATE_COMMAND "" + DOWNLOAD_COMMAND "" + BUILD_BYPRODUCTS "${UR_OFFLOAD_INSTALL_DIR}/lib/libLLVMOffload.so" + ) + add_dependencies(${TARGET_NAME} offload_ext) + install(DIRECTORY "${UR_OFFLOAD_INSTALL_DIR}/" + DESTINATION "${CMAKE_INSTALL_PREFIX}" + COMPONENT ur_adapter_offload) endif() # When targetting CUDA devices, we need a workaround to avoid sending PTX to @@ -29,22 +118,6 @@ if (NOT TARGET cudadrv) ) endif() -add_ur_adapter(${TARGET_NAME} - SHARED - ${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/context.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/enqueue.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/event.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/ur2offload.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp -) install_ur_library(${TARGET_NAME}) set_target_properties(${TARGET_NAME} PROPERTIES