Skip to content

Commit a68618a

Browse files
simpal01simpal01
committed
[ATFE] Package common multilib headers into target triple directory.
The current ATFE package redundantly duplicates identical C and C++ headers across more than 100 ibrary variants, despite the fact that these headers do not vary within a given target triple aside from few. As a result, each variant currently maintains its own include/ and include/c++/v1/ directories, leading to unnecessary duplication. This patch is to refactor the multilib layout to use a single shared header directory per target triple, eliminating redundant copies of identical headers across library variants, while keeping variant-specific headers within each variant’s own include/ directory. Changing the layout of the package structure will require corresponding updates to the ATFE build system, as it currently assumes a variant specific directory hierarchy for headers and libraries during installation and packaging. A new Python script has been introduced that runs after the runtime subproject has been executed for each variant and the non-optimised multilib directories have been generated. This script identifies and extracts common headers from the non-optimised multilibs and creates an optimized multilib directory. In this optimised directory, only the common headers are centralised, while the remaining contents are preserved as-is from the non-optimised layout. A CMake flag has been added to disable this multilib optimisation phase. When this flag is set, the optimised multilib directory will not be generated. Additionally, the optimisation step is skipped automatically if there are fewer than two multilib variants to build. To support this new layout with a centralised include directory, a new field called IncludePath has been added to multilib.yaml.This field specifies a base path for locating header files wich gets added to the header search path sequence by clang. Corresponding changes in Clang: llvm/llvm-project#146651
1 parent 10b97ee commit a68618a

File tree

3 files changed

+194
-4
lines changed

3 files changed

+194
-4
lines changed

‎arm-software/embedded/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,11 @@ option(
180180
OFF
181181
)
182182

183+
option(
184+
ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase"
185+
ON
186+
)
187+
183188
# Previously, the LLVM_TOOLCHAIN_LIBRARY_OVERLAY_INSTALL option was
184189
# called LLVM_TOOLCHAIN_NEWLIB_OVERLAY_INSTALL. Detect a setting of
185190
# that name, in case it's in an existing CMakeCache.txt or command
@@ -646,6 +651,7 @@ if(NOT PREBUILT_TARGET_LIBRARIES)
646651
-DLLVM_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/llvm
647652
-DMULTILIB_JSON=${LLVM_TOOLCHAIN_MULTILIB_JSON}
648653
-DENABLE_VARIANTS=${ENABLE_VARIANTS_PASSTHROUGH}
654+
-DENABLE_MULTILIB_HEADER_OPTIMISATION=${ENABLE_MULTILIB_HEADER_OPTIMISATION}
649655
-DENABLE_PARALLEL_LIB_CONFIG=${ENABLE_PARALLEL_LIB_CONFIG}
650656
-DENABLE_PARALLEL_LIB_BUILD=${ENABLE_PARALLEL_LIB_BUILD}
651657
-DPARALLEL_LIB_BUILD_LEVELS=${PARALLEL_LIB_BUILD_LEVELS}

‎arm-software/embedded/arm-multilib/CMakeLists.txt‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ set(
4848
fvp/get_fvps.sh"
4949
)
5050
set(FVP_CONFIG_DIR "${TOOLCHAIN_SOURCE_DIR}/fvp/config" CACHE STRING "The directory in which the FVP models are installed.")
51+
option(ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase" ON)
5152
option(
5253
ENABLE_PARALLEL_LIB_CONFIG
5354
"Run the library variant configuration steps in parallel."
@@ -185,10 +186,12 @@ foreach(lib_idx RANGE ${lib_count_dec})
185186
set(parent_dir_name arm-none-eabi)
186187
endif()
187188
set(destination_directory "${CMAKE_CURRENT_BINARY_DIR}/multilib/${parent_dir_name}/${variant}")
188-
install(
189-
DIRECTORY ${destination_directory}
190-
DESTINATION ${parent_dir_name}
191-
)
189+
if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION)
190+
install(
191+
DIRECTORY ${destination_directory}
192+
DESTINATION ${parent_dir_name}
193+
)
194+
endif()
192195
set(variant_json_file ${CMAKE_CURRENT_SOURCE_DIR}/json/variants/${variant_json})
193196

194197
# Read info from the variant specific json.
@@ -251,6 +254,8 @@ foreach(lib_idx RANGE ${lib_count_dec})
251254
TEST_EXCLUDE_FROM_MAIN TRUE
252255
)
253256

257+
list(APPEND all_runtime_targets runtimes-${variant})
258+
254259
if(ENABLE_PARALLEL_LIB_CONFIG OR ENABLE_PARALLEL_LIB_BUILD)
255260
# Create additional steps to configure/build the subprojects.
256261
# These are collected to be run together, so that all the
@@ -337,6 +342,9 @@ foreach(lib_idx RANGE ${lib_count_dec})
337342
foreach(flag ${multilib_flags_list})
338343
string(APPEND multilib_yaml_content " - ${flag}\n")
339344
endforeach()
345+
string(APPEND multilib_yaml_content " IncludePath:\n")
346+
string(APPEND multilib_yaml_content " - ${parent_dir_name}/include\n")
347+
string(APPEND multilib_yaml_content " - ${parent_dir_name}/${variant}/include\n")
340348
string(APPEND multilib_yaml_content " Group: stdlibs\n")
341349
else()
342350
# In place of a json, an error message is expected.
@@ -354,6 +362,7 @@ foreach(lib_idx RANGE ${lib_count_dec})
354362

355363
endforeach()
356364

365+
357366
# Multilib file is generated in two parts.
358367
# 1. Template is filled with multilib flags from json
359368
configure_file(
@@ -390,3 +399,19 @@ install(
390399
FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
391400
DESTINATION .
392401
)
402+
403+
# Extract common header files which spans across multilib variant directories.
404+
if(ENABLE_MULTILIB_HEADER_OPTIMISATION)
405+
add_custom_target(generate-multilib-common-headers ALL
406+
COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/common-headers-generate.py"
407+
"${CMAKE_CURRENT_BINARY_DIR}/multilib"
408+
"${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised"
409+
COMMENT "Generating common headers"
410+
DEPENDS ${all_runtime_targets}
411+
)
412+
413+
install(
414+
DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised/"
415+
DESTINATION .
416+
)
417+
endif()

‎arm-software/embedded/arm-multilib/common-headers-generate.py‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
#!/usr/bin/env python3
2+
3+
"""
4+
Identifies and extracts header files that are common across multiple multilib variant directories.
5+
6+
This script scans all variant folders within a multilib target directory
7+
(e.g., `arm-none-eabi/variant1/include`, `arm-none-eabi/variant2/include`, etc.) and compares all
8+
`.h` files. If the same file (by name and content) appears in multiple variants, it will be moved
9+
to a shared include directory at:
10+
11+
<CMAKE_BINARY_DIR>/multilib-optimised/<target>/include/
12+
13+
for the following multilib targets:
14+
- arm-none-eabi
15+
- aarch64-none-elf
16+
17+
Arguments:
18+
<CMAKE_BINARY_DIR>/multilib Path to the CMake build directory containing non optmised multilib.
19+
eg: build/multilib-builds/multilib/picolibc-build/multilib
20+
<CMAKE_BINARY_DIR>/multilib-optimised Path to the CMake build directory where optimised multilib should be generated..
21+
eg: build/multilib-builds/multilib/picolibc-build/multilib-optimised
22+
23+
This is useful to reduce duplication in the toolchain by centralizing common headers
24+
that are shared across architecture variants.
25+
"""
26+
import argparse
27+
import os
28+
import filecmp
29+
import shutil
30+
31+
# Define the multilib target dirs which want to process
32+
MULTILIB_TARGET_DIRS = ["arm-none-eabi", "aarch64-none-elf"]
33+
34+
35+
def files_are_identical(f1, f2):
36+
return filecmp.cmp(f1, f2, shallow=False)
37+
38+
39+
def collect_variant_include_paths(input_target_dir):
40+
"""
41+
Extracts each multilib variant and its corresponding include path from the non-optimized multilib directory.
42+
Stores the results to enable later comparison of header contents across different non-optimized multilib variant
43+
include paths.
44+
45+
"""
46+
variant_include_path = {}
47+
for variant in os.listdir(input_target_dir):
48+
include_path = os.path.join(input_target_dir, variant, "include")
49+
if os.path.isdir(include_path):
50+
variant_include_path[variant] = include_path
51+
return variant_include_path
52+
53+
54+
def extract_common_headers_for_targets(args):
55+
for target in MULTILIB_TARGET_DIRS:
56+
input_target_dir = os.path.join(
57+
os.path.abspath(args.multilib_non_optimised_dir), target
58+
)
59+
output_target_dir = os.path.join(
60+
os.path.abspath(args.multilib_optimised_dir), target
61+
)
62+
output_include_dir = os.path.join(output_target_dir, "include")
63+
64+
if not os.path.isdir(input_target_dir):
65+
print(
66+
f"Skipping extracting the common headers for {target}: input path {input_target_dir} not found"
67+
)
68+
continue
69+
70+
os.makedirs(output_include_dir, exist_ok=True)
71+
72+
variant_includes = collect_variant_include_paths(input_target_dir)
73+
if len(variant_includes) < 2:
74+
print(
75+
f"Skipping extracing the common headers for {target}: not enough variants to compare.At least two variants must be enabled for the multilib header optimization phase to proceed."
76+
)
77+
return
78+
79+
# Step 1: compare first two variants and extract the common headers into the targets common include directory
80+
base_dir = list(variant_includes.values())[0]
81+
compare_dir = list(variant_includes.values())[1]
82+
for root, sub_dirs, header_files in os.walk(base_dir):
83+
sub_dir_root = os.path.relpath(root, base_dir)
84+
for header in header_files:
85+
h1 = os.path.join(base_dir, sub_dir_root, header)
86+
h2 = os.path.join(compare_dir, sub_dir_root, header)
87+
if os.path.exists(h2) and files_are_identical(h1, h2):
88+
out_dir = os.path.join(output_include_dir, sub_dir_root)
89+
os.makedirs(out_dir, exist_ok=True)
90+
shutil.copy2(h1, os.path.join(out_dir, header))
91+
92+
# Step 2: Compare all the variants with the new common include. Any headers that do not match
93+
# and do not exit in common include should retain in their respective variant specific directories.
94+
for variant, include_path in variant_includes.items():
95+
for root, sub_dirs, header_files in os.walk(include_path):
96+
sub_dir_root = os.path.relpath(root, include_path)
97+
for header in header_files:
98+
variant_header = os.path.join(include_path, sub_dir_root, header)
99+
common_header = os.path.join(
100+
output_include_dir, sub_dir_root, header
101+
)
102+
if not os.path.exists(common_header) or not files_are_identical(
103+
variant_header, common_header
104+
):
105+
out_dir = os.path.join(
106+
os.path.abspath(args.multilib_optimised_dir),
107+
target,
108+
variant,
109+
sub_dir_root,
110+
"include",
111+
)
112+
os.makedirs(out_dir, exist_ok=True)
113+
shutil.copy2(variant_header, os.path.join(out_dir, header))
114+
115+
# Step3: For each variant, the lib and share directories should be copied from the non-optimized multilib
116+
# directory as it is.
117+
for variant in variant_includes:
118+
remaining_dirs = ["lib", "share"]
119+
for folder in remaining_dirs:
120+
src_dir = os.path.join(input_target_dir, variant, folder)
121+
dst_dir = os.path.join(output_target_dir, variant, folder)
122+
if os.path.exists(src_dir):
123+
# If destination exists, remove it first
124+
if os.path.exists(dst_dir):
125+
shutil.rmtree(dst_dir)
126+
os.makedirs(os.path.dirname(dst_dir), exist_ok=True)
127+
shutil.copytree(src_dir, dst_dir)
128+
else:
129+
print(f"Warning: {src_dir} does not exist and will be skipped.")
130+
131+
# Step4: Copy multilib.yaml file as it is from the non-optimized multilib directoy.
132+
src_yaml = os.path.join(args.multilib_non_optimised_dir, "multilib.yaml")
133+
dst_yaml = os.path.join(args.multilib_optimised_dir, "multilib.yaml")
134+
if os.path.exists(src_yaml):
135+
shutil.copy2(src_yaml, dst_yaml)
136+
else:
137+
print(f"Warning: {src_yaml} does not exist.")
138+
139+
140+
def main():
141+
parser = argparse.ArgumentParser(
142+
description=__doc__,
143+
formatter_class=argparse.RawDescriptionHelpFormatter,
144+
)
145+
parser.add_argument(
146+
"multilib_non_optimised_dir",
147+
help="CMake binary directory containing the non-optimised multilib headers",
148+
)
149+
parser.add_argument(
150+
"multilib_optimised_dir",
151+
help="CMake binary directory where the optimised multilib headers should be generated",
152+
)
153+
args = parser.parse_args()
154+
155+
extract_common_headers_for_targets(args)
156+
157+
158+
if __name__ == "__main__":
159+
main()

0 commit comments

Comments
 (0)