Skip to content

Commit 800c427

Browse files
aelovikov-intelaelovikov-intel
committed
[SYCL-RTC] Distribute SYCL toolchain header files inside libsycl-jit.so
We're still accessing `libdevice`'s `*.bc` files from the file system, those are left for another PR.
1 parent c4c086b commit 800c427

File tree

5 files changed

+126
-23
lines changed

5 files changed

+126
-23
lines changed

‎sycl-jit/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ if (NOT WIN32 AND NOT CYGWIN)
1919
endif(SYCL_JIT_ENABLE_WERROR)
2020
endif()
2121

22-
22+
add_subdirectory(jit-compiler-resource)
2323
add_subdirectory(jit-compiler)
2424
add_subdirectory(passes)
2525

‎sycl-jit/jit-compiler-resource/CMakeLists.txt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# We use C23/C++26's `#embed` to implement this resource creation, and "host"
2+
# CXX compiler might not have support for it. As such, use freshly built
3+
# `clang++` instead. That, in turn, requires dedicated directory for CMake to
4+
# allow to override `CMAKE_CXX_COMPILER`.
5+
6+
7+
set(SYCL_JIT_RESOURCE_CPP "${CMAKE_CURRENT_BINARY_DIR}/resource.cpp")
8+
add_custom_command(
9+
OUTPUT ${SYCL_JIT_RESOURCE_CPP}
10+
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate.py --toolchain-dir ${CMAKE_BINARY_DIR} --output ${SYCL_JIT_RESOURCE_CPP} --prefix /sycl-jit-toolchain/
11+
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
12+
DEPENDS sycl-headers ${CMAKE_CURRENT_SOURCE_DIR}/generate.py
13+
)
14+
15+
set_source_files_properties(${SYCL_JIT_RESOURCE_CPP} PROPERTIES
16+
COMPILE_FLAGS -Wno-c23-extensions
17+
)
18+
19+
add_llvm_library(sycl-jit-resource
20+
${SYCL_JIT_RESOURCE_CPP}
21+
OBJECT
22+
DEPENDS
23+
clang
24+
# TODO: libdevice
25+
)
26+
27+
set(CMAKE_CXX_COMPILER ${CMAKE_BINARY_DIR}/bin/clang++)

‎sycl-jit/jit-compiler-resource/generate.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import os
2+
import argparse
3+
4+
5+
def main():
6+
parser = argparse.ArgumentParser(
7+
description="Generate SYCL Headers Resource C++ file"
8+
)
9+
parser.add_argument("-o", "--output", type=str, required=True, help="Output file")
10+
parser.add_argument(
11+
"-i",
12+
"--toolchain-dir",
13+
type=str,
14+
required=True,
15+
help="Path to toolchain root directory",
16+
)
17+
parser.add_argument(
18+
"--prefix", type=str, required=True, help="Prefix for file locations"
19+
)
20+
args = parser.parse_args()
21+
22+
# abspath also strips trailing "/"
23+
toolchain_dir = os.path.abspath(args.toolchain_dir)
24+
25+
with open(args.output, "w") as out:
26+
out.write(
27+
"""
28+
#include <utility>
29+
#include <string_view>
30+
31+
namespace jit_compiler {
32+
extern const std::pair<std::string_view, std::string_view> ToolchainFiles[];
33+
const std::pair<std::string_view, std::string_view> ToolchainFiles[] = {"""
34+
)
35+
36+
def process_dir(dir):
37+
for root, _, files in os.walk(dir):
38+
for file in files:
39+
file_path = os.path.join(root, file)
40+
out.write(
41+
f"""
42+
{{
43+
{{"{args.prefix}{os.path.relpath(file_path, toolchain_dir)}"}} ,
44+
[]() {{
45+
static const char data[] = {{
46+
#embed "{file_path}"
47+
, 0}};
48+
return std::string_view(data);
49+
}}()
50+
}},"""
51+
)
52+
53+
process_dir(os.path.join(args.toolchain_dir, "include/"))
54+
process_dir(os.path.join(args.toolchain_dir, "lib/clang/"))
55+
56+
out.write(
57+
f"""
58+
}};
59+
60+
extern size_t NumToolchainFiles;
61+
size_t NumToolchainFiles = std::size(ToolchainFiles);
62+
extern std::string_view ToolchainPrefix;
63+
std::string_view ToolchainPrefix = "{args.prefix}";
64+
}} // namespace jit_compiler
65+
"""
66+
)
67+
68+
69+
if __name__ == "__main__":
70+
main()

‎sycl-jit/jit-compiler/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ add_llvm_library(sycl-jit
4545
clangCodeGen
4646
clangTooling
4747
clangSerialization
48+
sycl-jit-resource
4849
)
4950

5051
if(WIN32)

‎sycl-jit/jit-compiler/lib/rtc/DeviceCompilation.cpp‎

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@
5151
#include <array>
5252
#include <sstream>
5353

54+
namespace jit_compiler {
55+
// Defined in the auto-generated file:
56+
extern const std::pair<std::string_view, std::string_view> ToolchainFiles[];
57+
extern size_t NumToolchainFiles;
58+
extern std::string_view ToolchainPrefix;
59+
} // namespace jit_compiler
60+
5461
using namespace clang;
5562
using namespace clang::tooling;
5663
using namespace clang::driver;
@@ -178,7 +185,12 @@ class HashPreprocessedAction : public PreprocessorFrontendAction {
178185
};
179186

180187
class SYCLToolchain {
181-
SYCLToolchain() {}
188+
SYCLToolchain() {
189+
for (size_t i = 0; i < NumToolchainFiles; ++i) {
190+
auto [Path, Content] = ToolchainFiles[i];
191+
ToolchainFS->addFile(Path, 0, llvm::MemoryBuffer::getMemBuffer(Content));
192+
}
193+
}
182194

183195
// Similar to FrontendActionFactory, but we don't take ownership of
184196
// `FrontendAction`, nor do we create copies of it as we only perform a single
@@ -231,6 +243,7 @@ class SYCLToolchain {
231243
DiagnosticConsumer *DiagConsumer = nullptr) {
232244
auto FS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
233245
llvm::vfs::getRealFileSystem());
246+
FS->pushOverlay(ToolchainFS);
234247
if (FSOverlay)
235248
FS->pushOverlay(FSOverlay);
236249

@@ -245,8 +258,14 @@ class SYCLToolchain {
245258
return TI.run();
246259
}
247260

261+
std::string_view getClangXXExe() const { return ClangXXExe; }
262+
248263
private:
249264
clang::IgnoringDiagConsumer IgnoreDiag;
265+
std::string ClangXXExe =
266+
(jit_compiler::ToolchainPrefix + "/bin/clang++").str();
267+
llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> ToolchainFS =
268+
llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
250269
};
251270

252271
class ClangDiagnosticWrapper {
@@ -296,14 +315,11 @@ class LLVMDiagnosticWrapper : public llvm::DiagnosticHandler {
296315
} // anonymous namespace
297316

298317
static std::vector<std::string>
299-
createCommandLine(const InputArgList &UserArgList, std::string_view DPCPPRoot,
300-
BinaryFormat Format, std::string_view SourceFilePath) {
318+
createCommandLine(const InputArgList &UserArgList, BinaryFormat Format,
319+
std::string_view SourceFilePath) {
301320
DerivedArgList DAL{UserArgList};
302321
const auto &OptTable = getDriverOptTable();
303322
DAL.AddFlagArg(nullptr, OptTable.getOption(OPT_fsycl_device_only));
304-
DAL.AddJoinedArg(
305-
nullptr, OptTable.getOption(OPT_resource_dir_EQ),
306-
(DPCPPRoot + "/lib/clang/" + Twine(CLANG_VERSION_MAJOR)).str());
307323
// User args may contain options not intended for the frontend, but we can't
308324
// claim them here to tell the driver they're used later. Hence, suppress the
309325
// unused argument warning.
@@ -327,7 +343,7 @@ createCommandLine(const InputArgList &UserArgList, std::string_view DPCPPRoot,
327343

328344
std::vector<std::string> CommandLine;
329345
CommandLine.reserve(ASL.size() + 2);
330-
CommandLine.emplace_back((DPCPPRoot + "/bin/clang++").str());
346+
CommandLine.emplace_back(SYCLToolchain::instance().getClangXXExe());
331347
transform(ASL, std::back_inserter(CommandLine),
332348
[](const char *AS) { return std::string{AS}; });
333349
CommandLine.emplace_back(SourceFilePath);
@@ -355,13 +371,8 @@ Expected<std::string> jit_compiler::calculateHash(
355371
const InputArgList &UserArgList, BinaryFormat Format) {
356372
TimeTraceScope TTS{"calculateHash"};
357373

358-
const std::string &DPCPPRoot = getDPCPPRoot();
359-
if (DPCPPRoot == InvalidDPCPPRoot) {
360-
return createStringError("Could not locate DPCPP root directory");
361-
}
362-
363374
std::vector<std::string> CommandLine =
364-
createCommandLine(UserArgList, DPCPPRoot, Format, SourceFile.Path);
375+
createCommandLine(UserArgList, Format, SourceFile.Path);
365376

366377
HashPreprocessedAction HashAction;
367378

@@ -372,8 +383,7 @@ Expected<std::string> jit_compiler::calculateHash(
372383
// unique for each query, so drop it:
373384
CommandLine.pop_back();
374385

375-
// The command line contains the DPCPP root and clang major version in
376-
// "-resource-dir=<...>" argument.
386+
// TODO: Include hash of the current libsycl-jit.so/.dll somehow...
377387
BLAKE3Result<> CommandLineHash =
378388
BLAKE3::hash(arrayRefFromStringRef(join(CommandLine, ",")));
379389

@@ -394,18 +404,13 @@ Expected<ModuleUPtr> jit_compiler::compileDeviceCode(
394404
LLVMContext &Context, BinaryFormat Format) {
395405
TimeTraceScope TTS{"compileDeviceCode"};
396406

397-
const std::string &DPCPPRoot = getDPCPPRoot();
398-
if (DPCPPRoot == InvalidDPCPPRoot) {
399-
return createStringError("Could not locate DPCPP root directory");
400-
}
401-
402407
EmitLLVMOnlyAction ELOA{&Context};
403408
DiagnosticOptions DiagOpts;
404409
ClangDiagnosticWrapper Wrapper(BuildLog, &DiagOpts);
405410

406411
if (SYCLToolchain::instance().run(
407-
createCommandLine(UserArgList, DPCPPRoot, Format, SourceFile.Path),
408-
ELOA, getInMemoryFS(SourceFile, IncludeFiles), Wrapper.consumer())) {
412+
createCommandLine(UserArgList, Format, SourceFile.Path), ELOA,
413+
getInMemoryFS(SourceFile, IncludeFiles), Wrapper.consumer())) {
409414
return ELOA.takeModule();
410415
} else {
411416
return createStringError(BuildLog);

0 commit comments

Comments
 (0)