Skip to content

[UR][Offload] Add option to build offload adapter #19864

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Sep 2, 2025
Merged

[UR][Offload] Add option to build offload adapter #19864

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
2f8de1a
[UR][Offload] Add option to build offload adapter
againull Aug 19, 2025
750d5aa
Format
againull Aug 22, 2025
13d5408
Fix typos
againull Aug 22, 2025
d31e63c
Typo
againull Aug 22, 2025
de08c3f
Merge remote-tracking branch 'origin/sycl' into pushed
againull Aug 26, 2025
b65d042
Create worktree from existing workspace
againull Aug 25, 2025
b9a0b9f
Remove unused var
againull Aug 26, 2025
41ec405
Merge remote-tracking branch 'origin/sycl' into pushed
againull Aug 26, 2025
0410a38
Merge remote-tracking branch 'origin/sycl' into ur_offload_build_option
againull Sep 2, 2025
4a94f47
Update hash
againull Sep 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions buildbot/configure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,11 @@ def do_configure(args, passthrough_args):
if libclc_enabled:
llvm_enable_projects += ";libclc"

# DeviceRTL uses -fuse-ld=lld, so enable lld.
if args.offload:
llvm_enable_projects += ";lld"
sycl_enabled_backends.append("offload")

if args.cuda:
llvm_targets_to_build += ";NVPTX"
libclc_targets_to_build = libclc_nvidia_target_names
Expand Down Expand Up @@ -210,6 +215,12 @@ def do_configure(args, passthrough_args):
"-DSYCL_ENABLE_MAJOR_RELEASE_PREVIEW_LIB={}".format(sycl_preview_lib),
"-DBUG_REPORT_URL=https://github.com/intel/llvm/issues",
]
if args.offload:
cmake_cmd.extend(
[
"-DUR_BUILD_ADAPTER_OFFLOAD=ON",
]
)

if libclc_enabled:
cmake_cmd.extend(
Expand Down Expand Up @@ -340,6 +351,11 @@ def main():
default="AMD",
help="choose hardware platform for HIP backend",
)
parser.add_argument(
"--offload",
action="store_true",
help="Enable UR liboffload adapter (experimental)",
)
parser.add_argument(
"--level_zero_adapter_version",
type=str,
Expand Down
9 changes: 9 additions & 0 deletions sycl/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,15 @@ if("hip" IN_LIST SYCL_ENABLE_BACKENDS)
list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS ur_adapter_hip)
endif()

if("offload" IN_LIST SYCL_ENABLE_BACKENDS)
if(NOT TARGET lld)
message(FATAL_ERROR
"Offload support requires adding \"lld\" to the CMake argument \"LLVM_ENABLE_PROJECTS\"")
endif()
add_dependencies(sycl-toolchain ur_adapter_offload)
list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS ur_adapter_offload)
endif()

# Use it as fake dependency in order to force another command(s) to execute.
add_custom_command(OUTPUT __force_it
COMMAND "${CMAKE_COMMAND}" -E echo
Expand Down
4 changes: 4 additions & 0 deletions sycl/cmake/modules/BuildUnifiedRuntime.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,10 @@ if("native_cpu" IN_LIST SYCL_ENABLE_BACKENDS)
endif()
endif()

if("offload" IN_LIST SYCL_ENABLE_BACKENDS)
add_sycl_ur_adapter(offload)
endif()

if(CMAKE_SYSTEM_NAME STREQUAL Windows)
# On Windows, also build/install debug libraries with the d suffix that are
# compiled with /MDd so users can link against these in debug builds.
Expand Down
117 changes: 95 additions & 22 deletions unified-runtime/source/adapters/offload/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,103 @@

set(TARGET_NAME ur_adapter_offload)

set(UR_OFFLOAD_INSTALL_DIR "" CACHE PATH "Path to the directory containing libomptarget.so etc")
if (UR_OFFLOAD_INSTALL_DIR STREQUAL "")
message(FATAL_ERROR "UR_OFFLOAD_INSTALL_DIR must be defined for the Offload adapter")
endif()
add_ur_adapter(${TARGET_NAME}
SHARED
${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/context.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device.cpp
${CMAKE_CURRENT_SOURCE_DIR}/enqueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/event.cpp
${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp
${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp
${CMAKE_CURRENT_SOURCE_DIR}/program.cpp
${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ur2offload.hpp
${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp
)

set(UR_OFFLOAD_INSTALL_DIR "" CACHE PATH "Path to the directory containing libomptarget.so etc")
set(UR_OFFLOAD_INCLUDE_DIR "" CACHE PATH "Path to the directory containing LLVM headers")
if (UR_OFFLOAD_INCLUDE_DIR STREQUAL "")
message(FATAL_ERROR "UR_OFFLOAD_INCLUDE_DIR must be defined for the Offload adapter")
if (UR_OFFLOAD_INSTALL_DIR STREQUAL "" OR UR_OFFLOAD_INCLUDE_DIR STREQUAL "")
include(ExternalProject)
set(LLVM_PROJECT_SOURCE_DIR ${CMAKE_BINARY_DIR}/llvm-src-offload)
set(LLVM_PROJECT_TAG 4c0c295775cff0dcfc6439c3f51991ffac0345d8)
set(OPENMP_INSTALL_DIR ${CMAKE_BINARY_DIR}/openmp-install)
set(UR_OFFLOAD_INSTALL_DIR ${CMAKE_BINARY_DIR}/offload-install)
set(UR_OFFLOAD_INCLUDE_DIR ${UR_OFFLOAD_INSTALL_DIR}/include)

execute_process(COMMAND git -C "${CMAKE_SOURCE_DIR}" worktree prune)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have a helper for this:
https://github.com/intel/llvm/blob/sycl/unified-runtime/cmake/helpers.cmake#L246

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, I tried this, but the helper’s shallow checkout approach seems to work only with tags, and it becomes slow when used with commit hashes. So I am leaving this part as is for now if that's ok.


if(NOT IS_DIRECTORY "${LLVM_PROJECT_SOURCE_DIR}")
execute_process(
COMMAND git -C "${CMAKE_SOURCE_DIR}" worktree add --no-checkout --detach "${LLVM_PROJECT_SOURCE_DIR}" "${LLVM_PROJECT_TAG}"
COMMAND_ERROR_IS_FATAL ANY
)
endif()

execute_process(
COMMAND git sparse-checkout init --cone
WORKING_DIRECTORY "${LLVM_PROJECT_SOURCE_DIR}"
COMMAND_ERROR_IS_FATAL ANY
)

execute_process(
COMMAND git sparse-checkout set openmp offload cmake llvm/include libc
WORKING_DIRECTORY "${LLVM_PROJECT_SOURCE_DIR}"
COMMAND_ERROR_IS_FATAL ANY
)

execute_process(
COMMAND git checkout "${LLVM_PROJECT_TAG}"
WORKING_DIRECTORY "${LLVM_PROJECT_SOURCE_DIR}"
COMMAND_ERROR_IS_FATAL ANY
)

# Build OpenMP runtime (required dependency for offload's libomptarget) from the cloned source
ExternalProject_Add(openmp_ext
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we not just using LLVM_ENABLE_RUNTIMES="offload;openmp" here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We’re not using LLVM_ENABLE_RUNTIMES="offload;openmp" here because that mode requires configuring from the top-level llvm directory. In that flow, clang/llvm would be built as part of the runtimes build, which means clang/llvm ends up being built twice.
Instead, I build OpenMP and offload in standalone mode, reusing the existing clang/llvm build from intel/llvm.

Also, in the latest commit I improved this further: rather than cloning llvm-project again, I now create a git worktree from the already cloned intel/llvm repository, pulling in only the directories needed for the runtime build.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I looked into this a while ago and figured out how to build openmp without a dependency on clang. It required a few patches, which I should really figure out who to harass about.

rather than cloning llvm-project again, I now create a git worktree from the already cloned intel/llvm repository

What if the runtimes are using a newer version of llvm-project that hasn't been pulled down into intel/llvm yet?

Copy link
Contributor Author

@againull againull Aug 26, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry if the statement was confusing, worktree is created from fresh enough hash specified in LLVM_PROJECT_TAG, i.e. assumption is that llvm-project remote and its main branch is visible in the cloned intel/llvm repo, i.e. just trying to avoid re-cloning what's already available.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue I'm thinking of is with, for example the commit ee5022d63549032235247269903f6ccf7e70bd02 (which doesn't affect liboffload, but pretend it does). That commit exists on llvm/llvm-project, but not on intel/llvm at the time of writing (at the time of writing).

Although, there's a commit from 13 minutes ago on the main branch, so it might get mirrored frequently enough that that's not an issue. How often does intel/llvm pull down new llvm/llvm-project changes?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pulldowns approximately once per week/2weeks.
How dependent are openmp and offload from other components in llvm? I wonder why an option of standalone building mode even exists if those runtimes have to be built in-tree with entire llvm project of exact same version. If you believe openmp and offload are tightly coupled with the rest of llvm and has to be of exactly same version, then I can switch to what you propose - using LLVM_ENABLE_RUNTIMES="offload;openmp" i,e, re-building clang/llvm entirely from llvm-project.
Or we can try the current approach first for some time and switch to LLVM_ENABLE_RUNTIMES="offload;openmp" if there will be cases when build breaks because of such dependency.

# DeviceRTL uses -fuse-ld=lld, so add lld to the dependencies.
DEPENDS llvm-tblgen LLVMSupport clang lld FileCheck not
SOURCE_DIR ${LLVM_PROJECT_SOURCE_DIR}/openmp
CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX=${OPENMP_INSTALL_DIR}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_C_COMPILER=${CMAKE_BINARY_DIR}/bin/clang
-DCMAKE_CXX_COMPILER=${CMAKE_BINARY_DIR}/bin/clang++
-DLIBOMP_OMPD_GDB_SUPPORT=OFF
-DOPENMP_ENABLE_OMPT_TOOLS=OFF
-DCMAKE_PREFIX_PATH=${CMAKE_BINARY_DIR}/bin
INSTALL_COMMAND ${CMAKE_COMMAND} --build . --target install
UPDATE_COMMAND ""
DOWNLOAD_COMMAND ""
)

# Build liboffload runtime from the same source tree
ExternalProject_Add(offload_ext
DEPENDS openmp_ext
SOURCE_DIR ${LLVM_PROJECT_SOURCE_DIR}/offload
LIST_SEPARATOR |
CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX=${UR_OFFLOAD_INSTALL_DIR}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_C_COMPILER=${CMAKE_BINARY_DIR}/bin/clang
-DCMAKE_CXX_COMPILER=${CMAKE_BINARY_DIR}/bin/clang++
-DLIBOMPTARGET_LLVM_INCLUDE_DIRS=${LLVM_PROJECT_SOURCE_DIR}/llvm/include|${CMAKE_BINARY_DIR}/include
-DLLVM_DIR=${CMAKE_BINARY_DIR}/lib/cmake/llvm
-DLIBOMPTARGET_PLUGINS_TO_BUILD=cuda|amdgpu
-DLIBOMP_INCLUDE_DIR=${OPENMP_INSTALL_DIR}/include
-DLLVM_TABLEGEN=${CMAKE_BINARY_DIR}/bin/llvm-tblgen
-DCMAKE_PREFIX_PATH=${OPENMP_INSTALL_DIR}|${CMAKE_BINARY_DIR}/bin
INSTALL_COMMAND ${CMAKE_COMMAND} --build . --target install
UPDATE_COMMAND ""
DOWNLOAD_COMMAND ""
BUILD_BYPRODUCTS "${UR_OFFLOAD_INSTALL_DIR}/lib/libLLVMOffload.so"
)
add_dependencies(${TARGET_NAME} offload_ext)
install(DIRECTORY "${UR_OFFLOAD_INSTALL_DIR}/"
DESTINATION "${CMAKE_INSTALL_PREFIX}"
COMPONENT ur_adapter_offload)
endif()

# When targetting CUDA devices, we need a workaround to avoid sending PTX to
Expand All @@ -29,22 +118,6 @@ if (NOT TARGET cudadrv)
)
endif()

add_ur_adapter(${TARGET_NAME}
SHARED
${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/context.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device.cpp
${CMAKE_CURRENT_SOURCE_DIR}/enqueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/event.cpp
${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp
${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp
${CMAKE_CURRENT_SOURCE_DIR}/program.cpp
${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ur2offload.hpp
${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/usm.cpp
)
install_ur_library(${TARGET_NAME})

set_target_properties(${TARGET_NAME} PROPERTIES
Expand Down
Loading