Skip to content

Commit 629c0d4

Browse files
cperkinsintelcperkinsintel
committed
fix build race, capture what will be included when generating resource.cpp for sycl-jit, and added a blacklist that can be used to avoid IP issues
Signed-off-by: Chris Perkins <[email protected]>
1 parent ee397f9 commit 629c0d4

File tree

3 files changed

+130
-46
lines changed

3 files changed

+130
-46
lines changed

‎sycl-jit/jit-compiler/CMakeLists.txt‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11

2+
set(SYCL_JIT_RESOURCE_BLACKLIST "" CACHE FILEPATH "Path to a blacklist file for sycl-jit resources.")
3+
set(SYCL_JIT_RESOURCE_BLACKLIST ${CMAKE_CURRENT_SOURCE_DIR}/utils/blacklist.txt)
24

35
set(SYCL_JIT_RESOURCE_CPP "${CMAKE_CURRENT_BINARY_DIR}/resource.cpp")
46
set(SYCL_JIT_RESOURCE_OBJ "${CMAKE_CURRENT_BINARY_DIR}/resource.cpp.o")
@@ -10,8 +12,8 @@ set(SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT "/sycl-jit-toolchain/")
1012
endif()
1113

1214
set(SYCL_JIT_RESOURCE_DEPS
13-
sycl-headers # include/sycl
14-
clang # lib/clang/N/include
15+
sycl-headers # include/sycl
16+
opencl-resource-headers # lib/clang/N/include
1517
${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py)
1618

1719
if ("libclc" IN_LIST LLVM_ENABLE_PROJECTS)
@@ -23,11 +25,32 @@ if ("libdevice" IN_LIST LLVM_ENABLE_PROJECTS)
2325
list(APPEND SYCL_JIT_RESOURCE_DEPS libsycldevice) # lib/*.bc
2426
endif()
2527

28+
set(GENERATE_PY ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py)
29+
30+
set(GENERATE_PY_BASE_COMMAND ${Python3_EXECUTABLE} ${GENERATE_PY}
31+
--toolchain-dir ${CMAKE_BINARY_DIR}
32+
--output ${SYCL_JIT_RESOURCE_CPP}
33+
--prefix ${SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT}
34+
)
35+
set(GENERATE_PY_DEPENDS ${SYCL_JIT_RESOURCE_DEPS})
36+
37+
38+
list(APPEND GENERATE_PY_BASE_COMMAND --manifest-output ${CMAKE_CURRENT_BINARY_DIR}/capture_manifest.txt)
39+
40+
41+
if(EXISTS "${SYCL_JIT_RESOURCE_BLACKLIST}")
42+
message(STATUS "sycl-jit: Using resource blacklist file: ${SYCL_JIT_RESOURCE_BLACKLIST}")
43+
list(APPEND GENERATE_PY_BASE_COMMAND --blacklist ${SYCL_JIT_RESOURCE_BLACKLIST})
44+
# Make the build depend on the blacklist file, so it reruns if the file changes
45+
list(APPEND GENERATE_PY_DEPENDS ${SYCL_JIT_RESOURCE_BLACKLIST})
46+
endif()
47+
2648
add_custom_command(
27-
OUTPUT ${SYCL_JIT_RESOURCE_CPP}
28-
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/utils/generate.py --toolchain-dir ${CMAKE_BINARY_DIR} --output ${SYCL_JIT_RESOURCE_CPP} --prefix ${SYCL_JIT_VIRTUAL_TOOLCHAIN_ROOT}
29-
DEPENDS
30-
${SYCL_JIT_RESOURCE_DEPS}
49+
OUTPUT ${SYCL_JIT_RESOURCE_CPP}
50+
COMMAND ${GENERATE_PY_BASE_COMMAND}
51+
DEPENDS ${GENERATE_PY_DEPENDS}
52+
COMMENT "Generating resource.cpp..."
53+
VERBATIM
3154
)
3255

3356
# We use C23/C++26's `#embed` to implement this resource creation, and "current"

‎sycl-jit/jit-compiler/utils/blacklist.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# paths in this file will NOT be bundled into sycl-jit
2+
# use it to keep unneeded headers out of resources.cpp
3+
# and to avoid IP issues.
4+
lib/libsycl-msan-cpu.bc
5+
lib/libsycl-asan-pvc.bc
6+
lib/libsycl-msan.bc
7+
lib/libsycl-asan-cpu.bc
8+
lib/libsycl-msan-pvc.bc
9+
lib/libsycl-asan-dg2.bc
10+
lib/libsycl-tsan.bc
11+
lib/libsycl-tsan-pvc.bc
12+
lib/libsycl-tsan-cpu.bc

‎sycl-jit/jit-compiler/utils/generate.py‎

Lines changed: 89 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,115 @@
11
import os
22
import argparse
3+
import sys
4+
import fnmatch
35
import glob
46

5-
67
def main():
78
parser = argparse.ArgumentParser(
8-
description="Generate SYCL Headers Resource C++ file"
9-
)
10-
parser.add_argument("-o", "--output", type=str, required=True, help="Output file")
11-
parser.add_argument(
12-
"-i",
13-
"--toolchain-dir",
14-
type=str,
15-
required=True,
16-
help="Path to toolchain root directory",
17-
)
18-
parser.add_argument(
19-
"--prefix", type=str, required=True, help="Prefix for file locations"
9+
description="Generate SYCL Headers Resource C++ file."
2010
)
11+
parser.add_argument("-o", "--output", type=str, required=True, help="Output C++ file")
12+
parser.add_argument("-i", "--toolchain-dir", type=str, required=True, help="Path to toolchain root directory.")
13+
parser.add_argument("--prefix", type=str, required=True, help="Prefix for virtual file locations")
14+
parser.add_argument("-m", "--manifest-input", type=str, help="Build from this whitelist manifest (read-only).")
15+
parser.add_argument("--manifest-output", type=str, help="Glob for files and write them to this capture manifest.")
16+
parser.add_argument("--blacklist", type=str, help="Path to a file containing glob patterns of resources to exclude.")
17+
2118
args = parser.parse_args()
2219

23-
# abspath also strips trailing "/"
20+
if args.manifest_input and args.manifest_output:
21+
print("Error: --manifest-input and --manifest-output are mutually exclusive.", file=sys.stderr)
22+
sys.exit(1)
23+
24+
blacklist_patterns = set()
25+
if args.blacklist:
26+
print(f"Loading blacklist from: {args.blacklist}")
27+
with open(args.blacklist, "r") as f:
28+
for line in f:
29+
pattern = line.strip()
30+
if pattern and not pattern.startswith('#'):
31+
blacklist_patterns.add(pattern)
32+
2433
toolchain_dir = os.path.abspath(args.toolchain_dir)
34+
35+
manifest_to_write = open(args.manifest_output, "w") if args.manifest_output else open(os.devnull, "w")
36+
37+
with manifest_to_write as manifest_out, open(args.output, "w") as out:
38+
if args.manifest_output:
39+
preamble = f"""# This manifest was auto-geneerated by the sycl-jit build process
40+
# It contains the list of all candidate resource files found when globbing.
41+
#
42+
# If any of these files should NOT be included in the final library
43+
# (e.g. for IP reasons), add their relative path to the blacklist file at:
44+
# {args.blacklist}
45+
"""
46+
manifest_out.write(preamble + '\n')
2547

26-
with open(args.output, "w") as out:
2748
out.write(
2849
"""
2950
#include <Resource.h>
30-
3151
namespace jit_compiler::resource {
3252
const resource_file ToolchainFiles[] = {"""
3353
)
3454

35-
def process_file(file_path):
55+
def generate_cpp_for_file(absolute_path):
56+
relative_path = os.path.relpath(absolute_path, toolchain_dir)
57+
portable_relative_path = relative_path.replace(os.sep, '/')
58+
59+
for pattern in blacklist_patterns:
60+
# Compare the pattern against the portable relative path
61+
if fnmatch.fnmatch(portable_relative_path, pattern):
62+
print(f" -> Skipping blacklisted file: {portable_relative_path}")
63+
return None
64+
3665
out.write(
3766
f"""
38-
{{
39-
{{"{args.prefix}{os.path.relpath(file_path, toolchain_dir).replace(os.sep, "/")}"}} ,
40-
[]() {{
41-
static const char data[] = {{
42-
#embed "{file_path}" if_empty(0)
43-
, 0}};
44-
return resource_string_view{{data}};
45-
}}()
46-
}},"""
67+
{{
68+
{{"{args.prefix}{portable_relative_path}"}} ,
69+
[]() {{
70+
static const char data[] = {{
71+
#embed "{absolute_path}" if_empty(0)
72+
, 0}};
73+
return resource_string_view{{data}};
74+
}}()
75+
}},"""
4776
)
77+
return portable_relative_path
78+
79+
if args.manifest_input:
80+
print(f"Reading resource list from whitelist manifest: {args.manifest_input}")
81+
with open(args.manifest_input, "r") as manifest_file:
82+
for line in manifest_file:
83+
relative_path = line.strip()
84+
if relative_path:
85+
absolute_path = os.path.join(toolchain_dir, relative_path)
86+
generate_cpp_for_file(absolute_path)
87+
else:
88+
if args.manifest_output:
89+
print(f"Globbing for resources and writing capture manifest to: {args.manifest_output}")
90+
else:
91+
print("Globbing for resources (no capture manifest output)...")
4892

49-
def process_dir(dir):
50-
for root, _, files in os.walk(dir):
51-
for file in files:
52-
file_path = os.path.join(root, file)
53-
process_file(file_path)
93+
def process_and_log_file(absolute_path):
94+
relative_path = generate_cpp_for_file(absolute_path)
95+
if relative_path:
96+
manifest_out.write(relative_path + '\n')
5497

55-
process_dir(os.path.join(args.toolchain_dir, "include/"))
56-
process_dir(os.path.join(args.toolchain_dir, "lib/clang/"))
57-
process_dir(os.path.join(args.toolchain_dir, "lib/clc/"))
98+
def process_dir(dir):
99+
for root, _, files in os.walk(dir):
100+
for file in files:
101+
process_and_log_file(os.path.join(root, file))
58102

59-
for file in glob.iglob(
60-
"*.bc", root_dir=os.path.join(args.toolchain_dir, "lib")
61-
):
62-
file_path = os.path.join(args.toolchain_dir, "lib", file)
63-
process_file(file_path)
103+
process_dir(os.path.join(args.toolchain_dir, "include/"))
104+
process_dir(os.path.join(args.toolchain_dir, "lib/clang/"))
105+
process_dir(os.path.join(args.toolchain_dir, "lib/clc/"))
106+
107+
print("Recursively searching for .bc files in lib/...")
108+
lib_dir = os.path.join(args.toolchain_dir, "lib")
109+
search_pattern = os.path.join(lib_dir, "**", "*.bc")
110+
111+
for file_path in glob.glob(search_pattern, recursive=True):
112+
process_and_log_file(file_path)
64113

65114
out.write(
66115
f"""
@@ -72,6 +121,6 @@ def process_dir(dir):
72121
"""
73122
)
74123

75-
76124
if __name__ == "__main__":
77125
main()
126+

0 commit comments

Comments
 (0)