Skip to content

Commit 87cc792

Browse files
aelovikov-intelaelovikov-intel
authored
[SYCL-RTC] Distribute SYCL toolchain header files inside libsycl-jit.so (#19924)
We're still accessing `libdevice`'s `*.bc` files from the file system, those are left for another PR.
1 parent 3c1237f commit 87cc792

File tree

5 files changed

+159
-24
lines changed

5 files changed

+159
-24
lines changed

‎clang/lib/Driver/ToolChains/SYCL.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "llvm/SYCLLowerIR/DeviceConfigFile.hpp"
1414
#include "llvm/Support/CommandLine.h"
1515
#include "llvm/Support/Path.h"
16+
#include "llvm/Support/VirtualFileSystem.h"
1617
#include <sstream>
1718

1819
using namespace clang::driver;
@@ -55,7 +56,7 @@ const char *SYCLInstallationDetector::findLibspirvPath(
5556

5657
// If -fsycl-libspirv-path= is specified, try to use that path directly.
5758
if (Arg *A = Args.getLastArg(options::OPT_fsycl_libspirv_path_EQ)) {
58-
if (llvm::sys::fs::exists(A->getValue()))
59+
if (D.getVFS().exists(A->getValue()))
5960
return A->getValue();
6061

6162
return nullptr;
@@ -68,7 +69,7 @@ const char *SYCLInstallationDetector::findLibspirvPath(
6869
SmallString<128> LibraryPath(Path);
6970
llvm::sys::path::append(LibraryPath, a, b, c, Basename);
7071

71-
if (llvm::sys::fs::exists(LibraryPath))
72+
if (D.getVFS().exists(LibraryPath))
7273
return Args.MakeArgString(LibraryPath);
7374

7475
return nullptr;

‎sycl-jit/jit-compiler/CMakeLists.txt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,50 @@
11

2+
3+
set(SYCL_JIT_RESOURCE_CPP "${CMAKE_CURRENT_BINARY_DIR}/resource.cpp")
4+
set(SYCL_JIT_RESOURCE_OBJ "${CMAKE_CURRENT_BINARY_DIR}/resource.cpp.o")
5+
6+
if (WIN32)
7+
set(SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT "c:/sycl-jit-toolchain/")
8+
else()
9+
set(SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT "/sycl-jit-toolchain/")
10+
endif()
11+
12+
add_custom_command(
13+
OUTPUT ${SYCL_JIT_RESOURCE_CPP}
14+
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py --toolchain-dir ${CMAKE_BINARY_DIR} --output ${SYCL_JIT_RESOURCE_CPP} --prefix ${SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT}
15+
DEPENDS
16+
sycl-headers # include/sycl
17+
libclc # lib/clc
18+
clang # lib/clang
19+
# TODO: libdevice
20+
${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py
21+
)
22+
23+
# We use C23/C++26's `#embed` to implement this resource creation, and "current"
24+
# CMAKE_CXX_COMPILER might not have support for it. As such, use freshly built
25+
# `clang++` instead.
26+
if (WIN32)
27+
set(clang_exe ${CMAKE_BINARY_DIR}/bin/clang-cl.exe)
28+
set(SYCL_JIT_RESOURCE_CXX_FLAGS /O2 /std:c++17 /W0)
29+
if (CMAKE_BUILD_TYPE MATCHES "Debug")
30+
list(APPEND SYCL_JIT_RESOURCE_CXX_FLAGS /MDd)
31+
else()
32+
list(APPEND SYCL_JIT_RESOURCE_CXX_FLAGS /MD)
33+
endif()
34+
else()
35+
get_host_tool_path( clang CLANG clang_exe clang_target )
36+
set(SYCL_JIT_RESOURCE_CXX_FLAGS -O2 -Wno-c23-extensions -std=c++17 -fPIC -fvisibility=hidden)
37+
endif()
38+
39+
add_custom_command(
40+
OUTPUT ${SYCL_JIT_RESOURCE_OBJ}
41+
COMMAND
42+
${clang_exe} ${SYCL_JIT_RESOURCE_CPP} -I ${CMAKE_CURRENT_SOURCE_DIR}/include -c -o ${SYCL_JIT_RESOURCE_OBJ} ${SYCL_JIT_RESOURCE_CXX_FLAGS}
43+
DEPENDS
44+
${SYCL_JIT_RESOURCE_CPP}
45+
${CMAKE_CURRENT_SOURCE_DIR}/include/Resource.h
46+
)
47+
248
add_llvm_library(sycl-jit
349
lib/translation/JITContext.cpp
450
lib/translation/SPIRVLLVMTranslation.cpp
@@ -11,6 +57,8 @@ add_llvm_library(sycl-jit
1157
lib/helper/ConfigHelper.cpp
1258
lib/helper/ErrorHelper.cpp
1359

60+
${SYCL_JIT_RESOURCE_OBJ}
61+
1462
SHARED
1563

1664
DEPENDS

‎sycl-jit/jit-compiler/include/Resource.h‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#pragma once
10+
11+
#include <iterator>
12+
#include <string_view>
13+
#include <utility>
14+
15+
namespace jit_compiler {
16+
// Defined in the auto-generated file:
17+
extern const std::pair<std::string_view, std::string_view> ToolchainFiles[];
18+
extern size_t NumToolchainFiles;
19+
extern std::string_view ToolchainPrefix;
20+
} // namespace jit_compiler

‎sycl-jit/jit-compiler/lib/rtc/DeviceCompilation.cpp‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "DeviceCompilation.h"
1010
#include "ESIMD.h"
1111
#include "JITBinaryInfo.h"
12+
#include "Resource.h"
1213
#include "translation/Translation.h"
1314

1415
#include <clang/Basic/DiagnosticDriver.h>
@@ -178,7 +179,12 @@ class HashPreprocessedAction : public PreprocessorFrontendAction {
178179
};
179180

180181
class SYCLToolchain {
181-
SYCLToolchain() {}
182+
SYCLToolchain() {
183+
for (size_t i = 0; i < NumToolchainFiles; ++i) {
184+
auto [Path, Content] = ToolchainFiles[i];
185+
ToolchainFS->addFile(Path, 0, llvm::MemoryBuffer::getMemBuffer(Content));
186+
}
187+
}
182188

183189
// Similar to FrontendActionFactory, but we don't take ownership of
184190
// `FrontendAction`, nor do we create copies of it as we only perform a single
@@ -231,6 +237,7 @@ class SYCLToolchain {
231237
DiagnosticConsumer *DiagConsumer = nullptr) {
232238
auto FS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
233239
llvm::vfs::getRealFileSystem());
240+
FS->pushOverlay(ToolchainFS);
234241
if (FSOverlay)
235242
FS->pushOverlay(FSOverlay);
236243

@@ -245,8 +252,14 @@ class SYCLToolchain {
245252
return TI.run();
246253
}
247254

255+
std::string_view getClangXXExe() const { return ClangXXExe; }
256+
248257
private:
249258
clang::IgnoringDiagConsumer IgnoreDiag;
259+
std::string ClangXXExe =
260+
(jit_compiler::ToolchainPrefix + "/bin/clang++").str();
261+
llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> ToolchainFS =
262+
llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
250263
};
251264

252265
class ClangDiagnosticWrapper {
@@ -296,14 +309,11 @@ class LLVMDiagnosticWrapper : public llvm::DiagnosticHandler {
296309
} // anonymous namespace
297310

298311
static std::vector<std::string>
299-
createCommandLine(const InputArgList &UserArgList, std::string_view DPCPPRoot,
300-
BinaryFormat Format, std::string_view SourceFilePath) {
312+
createCommandLine(const InputArgList &UserArgList, BinaryFormat Format,
313+
std::string_view SourceFilePath) {
301314
DerivedArgList DAL{UserArgList};
302315
const auto &OptTable = getDriverOptTable();
303316
DAL.AddFlagArg(nullptr, OptTable.getOption(OPT_fsycl_device_only));
304-
DAL.AddJoinedArg(
305-
nullptr, OptTable.getOption(OPT_resource_dir_EQ),
306-
(DPCPPRoot + "/lib/clang/" + Twine(CLANG_VERSION_MAJOR)).str());
307317
// User args may contain options not intended for the frontend, but we can't
308318
// claim them here to tell the driver they're used later. Hence, suppress the
309319
// unused argument warning.
@@ -327,7 +337,7 @@ createCommandLine(const InputArgList &UserArgList, std::string_view DPCPPRoot,
327337

328338
std::vector<std::string> CommandLine;
329339
CommandLine.reserve(ASL.size() + 2);
330-
CommandLine.emplace_back((DPCPPRoot + "/bin/clang++").str());
340+
CommandLine.emplace_back(SYCLToolchain::instance().getClangXXExe());
331341
transform(ASL, std::back_inserter(CommandLine),
332342
[](const char *AS) { return std::string{AS}; });
333343
CommandLine.emplace_back(SourceFilePath);
@@ -355,13 +365,8 @@ Expected<std::string> jit_compiler::calculateHash(
355365
const InputArgList &UserArgList, BinaryFormat Format) {
356366
TimeTraceScope TTS{"calculateHash"};
357367

358-
const std::string &DPCPPRoot = getDPCPPRoot();
359-
if (DPCPPRoot == InvalidDPCPPRoot) {
360-
return createStringError("Could not locate DPCPP root directory");
361-
}
362-
363368
std::vector<std::string> CommandLine =
364-
createCommandLine(UserArgList, DPCPPRoot, Format, SourceFile.Path);
369+
createCommandLine(UserArgList, Format, SourceFile.Path);
365370

366371
HashPreprocessedAction HashAction;
367372

@@ -372,8 +377,7 @@ Expected<std::string> jit_compiler::calculateHash(
372377
// unique for each query, so drop it:
373378
CommandLine.pop_back();
374379

375-
// The command line contains the DPCPP root and clang major version in
376-
// "-resource-dir=<...>" argument.
380+
// TODO: Include hash of the current libsycl-jit.so/.dll somehow...
377381
BLAKE3Result<> CommandLineHash =
378382
BLAKE3::hash(arrayRefFromStringRef(join(CommandLine, ",")));
379383

@@ -394,18 +398,13 @@ Expected<ModuleUPtr> jit_compiler::compileDeviceCode(
394398
LLVMContext &Context, BinaryFormat Format) {
395399
TimeTraceScope TTS{"compileDeviceCode"};
396400

397-
const std::string &DPCPPRoot = getDPCPPRoot();
398-
if (DPCPPRoot == InvalidDPCPPRoot) {
399-
return createStringError("Could not locate DPCPP root directory");
400-
}
401-
402401
EmitLLVMOnlyAction ELOA{&Context};
403402
DiagnosticOptions DiagOpts;
404403
ClangDiagnosticWrapper Wrapper(BuildLog, &DiagOpts);
405404

406405
if (SYCLToolchain::instance().run(
407-
createCommandLine(UserArgList, DPCPPRoot, Format, SourceFile.Path),
408-
ELOA, getInMemoryFS(SourceFile, IncludeFiles), Wrapper.consumer())) {
406+
createCommandLine(UserArgList, Format, SourceFile.Path), ELOA,
407+
getInMemoryFS(SourceFile, IncludeFiles), Wrapper.consumer())) {
409408
return ELOA.takeModule();
410409
} else {
411410
return createStringError(BuildLog);

‎sycl-jit/jit-compiler/utils/generate.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import os
2+
import argparse
3+
4+
5+
def main():
6+
parser = argparse.ArgumentParser(
7+
description="Generate SYCL Headers Resource C++ file"
8+
)
9+
parser.add_argument("-o", "--output", type=str, required=True, help="Output file")
10+
parser.add_argument(
11+
"-i",
12+
"--toolchain-dir",
13+
type=str,
14+
required=True,
15+
help="Path to toolchain root directory",
16+
)
17+
parser.add_argument(
18+
"--prefix", type=str, required=True, help="Prefix for file locations"
19+
)
20+
args = parser.parse_args()
21+
22+
# abspath also strips trailing "/"
23+
toolchain_dir = os.path.abspath(args.toolchain_dir)
24+
25+
with open(args.output, "w") as out:
26+
out.write(
27+
"""
28+
#include <Resource.h>
29+
30+
namespace jit_compiler {
31+
const std::pair<std::string_view, std::string_view> ToolchainFiles[] = {"""
32+
)
33+
34+
def process_dir(dir):
35+
for root, _, files in os.walk(dir):
36+
for file in files:
37+
file_path = os.path.join(root, file)
38+
out.write(
39+
f"""
40+
{{
41+
{{"{args.prefix}{os.path.relpath(file_path, toolchain_dir).replace(os.sep, "/")}"}} ,
42+
[]() {{
43+
static const char data[] = {{
44+
#embed "{file_path}" if_empty(0)
45+
, 0}};
46+
return std::string_view(data, std::size(data) - 1);
47+
}}()
48+
}},"""
49+
)
50+
51+
process_dir(os.path.join(args.toolchain_dir, "include/"))
52+
process_dir(os.path.join(args.toolchain_dir, "lib/clang/"))
53+
process_dir(os.path.join(args.toolchain_dir, "lib/clc/"))
54+
55+
out.write(
56+
f"""
57+
}};
58+
59+
size_t NumToolchainFiles = std::size(ToolchainFiles);
60+
std::string_view ToolchainPrefix = "{args.prefix}";
61+
}} // namespace jit_compiler
62+
"""
63+
)
64+
65+
66+
if __name__ == "__main__":
67+
main()

0 commit comments

Comments
 (0)