diff --git a/arm-software/embedded/CMakeLists.txt b/arm-software/embedded/CMakeLists.txt index 95e74fdb695c..25680c3a39fa 100644 --- a/arm-software/embedded/CMakeLists.txt +++ b/arm-software/embedded/CMakeLists.txt @@ -180,6 +180,11 @@ option( OFF ) +option( + ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase" + ON +) + # Previously, the LLVM_TOOLCHAIN_LIBRARY_OVERLAY_INSTALL option was # called LLVM_TOOLCHAIN_NEWLIB_OVERLAY_INSTALL. Detect a setting of # that name, in case it's in an existing CMakeCache.txt or command @@ -646,6 +651,7 @@ if(NOT PREBUILT_TARGET_LIBRARIES) -DLLVM_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/llvm -DMULTILIB_JSON=${LLVM_TOOLCHAIN_MULTILIB_JSON} -DENABLE_VARIANTS=${ENABLE_VARIANTS_PASSTHROUGH} + -DENABLE_MULTILIB_HEADER_OPTIMISATION=${ENABLE_MULTILIB_HEADER_OPTIMISATION} -DENABLE_PARALLEL_LIB_CONFIG=${ENABLE_PARALLEL_LIB_CONFIG} -DENABLE_PARALLEL_LIB_BUILD=${ENABLE_PARALLEL_LIB_BUILD} -DPARALLEL_LIB_BUILD_LEVELS=${PARALLEL_LIB_BUILD_LEVELS} diff --git a/arm-software/embedded/arm-multilib/CMakeLists.txt b/arm-software/embedded/arm-multilib/CMakeLists.txt index 041ddc30d802..589fa842739c 100644 --- a/arm-software/embedded/arm-multilib/CMakeLists.txt +++ b/arm-software/embedded/arm-multilib/CMakeLists.txt @@ -48,6 +48,7 @@ set( fvp/get_fvps.sh" ) set(FVP_CONFIG_DIR "${TOOLCHAIN_SOURCE_DIR}/fvp/config" CACHE STRING "The directory in which the FVP models are installed.") +option(ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase" ON) option( ENABLE_PARALLEL_LIB_CONFIG "Run the library variant configuration steps in parallel." @@ -185,10 +186,12 @@ foreach(lib_idx RANGE ${lib_count_dec}) set(parent_dir_name arm-none-eabi) endif() set(destination_directory "${CMAKE_CURRENT_BINARY_DIR}/multilib/${parent_dir_name}/${variant}") - install( - DIRECTORY ${destination_directory} - DESTINATION ${parent_dir_name} - ) + if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION) + install( + DIRECTORY ${destination_directory} + DESTINATION ${parent_dir_name} + ) + endif() set(variant_json_file ${CMAKE_CURRENT_SOURCE_DIR}/json/variants/${variant_json}) # Read info from the variant specific json. @@ -251,6 +254,8 @@ foreach(lib_idx RANGE ${lib_count_dec}) TEST_EXCLUDE_FROM_MAIN TRUE ) + list(APPEND all_runtime_targets runtimes-${variant}) + if(ENABLE_PARALLEL_LIB_CONFIG OR ENABLE_PARALLEL_LIB_BUILD) # Create additional steps to configure/build the subprojects. # These are collected to be run together, so that all the @@ -337,6 +342,11 @@ foreach(lib_idx RANGE ${lib_count_dec}) foreach(flag ${multilib_flags_list}) string(APPEND multilib_yaml_content " - ${flag}\n") endforeach() + if(ENABLE_MULTILIB_HEADER_OPTIMISATION) + string(APPEND multilib_yaml_content " IncludeDirs:\n") + string(APPEND multilib_yaml_content " - ${parent_dir_name}/${variant}/include\n") + string(APPEND multilib_yaml_content " - ${parent_dir_name}/include\n") + endif() string(APPEND multilib_yaml_content " Group: stdlibs\n") else() # In place of a json, an error message is expected. @@ -386,7 +396,25 @@ add_custom_command( ) add_custom_target(multilib-yaml ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml) -install( - FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml - DESTINATION . -) +if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION) + install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml + DESTINATION . + ) +endif() + +# Extract common header files which spans across multilib variant directories. +if(ENABLE_MULTILIB_HEADER_OPTIMISATION) + add_custom_target(generate-multilib-common-headers ALL + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/common-headers-generate.py" + "${CMAKE_CURRENT_BINARY_DIR}/multilib" + "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised" + COMMENT "Generating common headers" + DEPENDS ${all_runtime_targets} multilib-yaml + ) + + install( + DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised/" + DESTINATION . + ) +endif() diff --git a/arm-software/embedded/arm-multilib/common-headers-generate.py b/arm-software/embedded/arm-multilib/common-headers-generate.py new file mode 100755 index 000000000000..22498fe0eaf7 --- /dev/null +++ b/arm-software/embedded/arm-multilib/common-headers-generate.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 + +""" +Identifies and extracts header files that are common across multiple multilib variant directories. + +This script scans all variant folders within a multilib target directory +(e.g., `arm-none-eabi/variant1/include`, `arm-none-eabi/variant2/include`, etc.) and compares all +`.h` files. If the same file (by name and content) appears in multiple variants, it will be moved +to a shared include directory at: + + /multilib-optimised//include/ + +for the following multilib targets: +- arm-none-eabi +- aarch64-none-elf + +Arguments: + /multilib Path to the CMake build directory containing non optmised multilib. + eg: build/multilib-builds/multilib/picolibc-build/multilib + /multilib-optimised Path to the CMake build directory where optimised multilib should be generated.. + eg: build/multilib-builds/multilib/picolibc-build/multilib-optimised + +This is useful to reduce duplication in the toolchain by centralising common headers +that are shared across architecture variants. +""" +import argparse +import os +import filecmp +import shutil + +# Define the multilib target dirs which want to process +MULTILIB_TARGET_DIRS = ["arm-none-eabi", "aarch64-none-elf"] + + +def files_are_identical(f1, f2): + return filecmp.cmp(f1, f2, shallow=False) + + +def collect_variant_include_paths(input_target_dir): + """ + Extracts each multilib variant and its corresponding include path from the non-optimised multilib directory. + Stores the results to enable later comparison of header contents across different non-optimised multilib variant + include paths. + + """ + variant_include_path = {} + for variant in os.listdir(input_target_dir): + include_path = os.path.join(input_target_dir, variant, "include") + if os.path.isdir(include_path): + variant_include_path[variant] = include_path + return variant_include_path + + +def extract_common_headers_for_targets(args): + if os.path.exists(args.multilib_optimised_dir): + shutil.rmtree(args.multilib_optimised_dir) + + for target in MULTILIB_TARGET_DIRS: + input_target_dir = os.path.join( + os.path.abspath(args.multilib_non_optimised_dir), target + ) + output_target_dir = os.path.join( + os.path.abspath(args.multilib_optimised_dir), target + ) + output_include_dir = os.path.join(output_target_dir, "include") + + if not os.path.isdir(input_target_dir): + print( + f"Skipping extracting the common headers for {target}: input path {input_target_dir} not found" + ) + continue + + variant_includes = collect_variant_include_paths(input_target_dir) + if len(variant_includes) < 2: + print( + f"Skipping extracting the common headers for {target}: not enough variants to compare.At least two variants must be enabled for the multilib header optimisation phase to proceed." + ) + # The script always creates the multilib-optimised folder, even when there's only one variant and no + # optimization is applied. In that case, multilib-optimised will just contain a copy of the + # single variant from the non-optimised multilib directory. + if os.path.exists(args.multilib_non_optimised_dir): + shutil.copytree(args.multilib_non_optimised_dir, args.multilib_optimised_dir) + return + + # Creating the common include headers for each target + os.makedirs(output_include_dir, exist_ok=True) + + # Step 1: compare first two variants and extract the common headers into the targets common include directory + base_dir = list(variant_includes.values())[0] + compare_dir = list(variant_includes.values())[1] + for root, sub_dirs, header_files in os.walk(base_dir): + sub_dir_root = os.path.relpath(root, base_dir) + for header in header_files: + h1 = os.path.join(base_dir, sub_dir_root, header) + h2 = os.path.join(compare_dir, sub_dir_root, header) + if os.path.exists(h2) and files_are_identical(h1, h2): + out_dir = os.path.join(output_include_dir, sub_dir_root) + os.makedirs(out_dir, exist_ok=True) + shutil.copy2(h1, os.path.join(out_dir, header)) + + # Step 2: Compare all the variants with the new common include. Any headers that do not match + # and do not exit in common include should retain in their respective variant specific directories. + for variant, include_path in variant_includes.items(): + for root, sub_dirs, header_files in os.walk(include_path): + sub_dir_root = os.path.relpath(root, include_path) + for header in header_files: + variant_header = os.path.join(include_path, sub_dir_root, header) + common_header = os.path.join( + output_include_dir, sub_dir_root, header + ) + if not os.path.exists(common_header) or not files_are_identical( + variant_header, common_header + ): + out_dir = os.path.join( + os.path.abspath(args.multilib_optimised_dir), + target, + variant, + sub_dir_root, + "include", + ) + os.makedirs(out_dir, exist_ok=True) + shutil.copy2(variant_header, os.path.join(out_dir, header)) + + # Step3: For each variant, the lib and share directories should be copied from the non-optimised multilib + # directory as it is. + for variant in variant_includes: + remaining_dirs = ["lib", "share"] + for folder in remaining_dirs: + src_dir = os.path.join(input_target_dir, variant, folder) + dst_dir = os.path.join(output_target_dir, variant, folder) + if os.path.exists(src_dir): + # If destination exists, remove it first + if os.path.exists(dst_dir): + shutil.rmtree(dst_dir) + os.makedirs(os.path.dirname(dst_dir), exist_ok=True) + shutil.copytree(src_dir, dst_dir) + else: + print(f"Warning: {src_dir} does not exist and will be skipped.") + + # Step4: Copy multilib.yaml file as it is from the non-optimised multilib directoy. + src_yaml = os.path.join(args.multilib_non_optimised_dir, "multilib.yaml") + dst_yaml = os.path.join(args.multilib_optimised_dir, "multilib.yaml") + if os.path.exists(src_yaml): + shutil.copy2(src_yaml, dst_yaml) + else: + raise FileNotFoundError(f"Source yaml '{src_yaml}' does not exist.") + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "multilib_non_optimised_dir", + help="CMake binary directory containing the non-optimised multilib headers", + ) + parser.add_argument( + "multilib_optimised_dir", + help="CMake binary directory where the optimised multilib headers should be generated", + ) + args = parser.parse_args() + + extract_common_headers_for_targets(args) + + +if __name__ == "__main__": + main() diff --git a/clang/include/clang/Driver/Multilib.h b/clang/include/clang/Driver/Multilib.h index fc071ef48ca0..10c4457773f7 100644 --- a/clang/include/clang/Driver/Multilib.h +++ b/clang/include/clang/Driver/Multilib.h @@ -35,12 +35,18 @@ class Driver; class Multilib { public: using flags_list = std::vector; + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + using includedirs_list = std::vector; private: std::string GCCSuffix; std::string OSSuffix; std::string IncludeSuffix; flags_list Flags; + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + includedirs_list IncludeDirs; // Optionally, a multilib can be assigned a string tag indicating that it's // part of a group of mutually exclusive possibilities. If two or more @@ -62,6 +68,9 @@ class Multilib { /// This is enforced with an assert in the constructor. Multilib(StringRef GCCSuffix = {}, StringRef OSSuffix = {}, StringRef IncludeSuffix = {}, const flags_list &Flags = flags_list(), + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + const includedirs_list &IncludeDirs = includedirs_list(), StringRef ExclusiveGroup = {}, std::optional Error = std::nullopt); @@ -81,6 +90,12 @@ class Multilib { /// All elements begin with either '-' or '!' const flags_list &flags() const { return Flags; } + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + /// Get the include directories specified in multilib.yaml under the + /// 'IncludeDirs' field + const includedirs_list &includeDirs() const { return IncludeDirs; } + /// Get the exclusive group label. const std::string &exclusiveGroup() const { return ExclusiveGroup; } diff --git a/clang/lib/Driver/Multilib.cpp b/clang/lib/Driver/Multilib.cpp index 87fa1af54a8e..2faf33fb47c9 100644 --- a/clang/lib/Driver/Multilib.cpp +++ b/clang/lib/Driver/Multilib.cpp @@ -29,9 +29,15 @@ using namespace llvm::sys; Multilib::Multilib(StringRef GCCSuffix, StringRef OSSuffix, StringRef IncludeSuffix, const flags_list &Flags, + // Downstream issue: #446 (Extend the Multilib system to + // support an IncludeDirs field) + const includedirs_list &IncludeDirs, StringRef ExclusiveGroup, std::optional Error) : GCCSuffix(GCCSuffix), OSSuffix(OSSuffix), IncludeSuffix(IncludeSuffix), - Flags(Flags), ExclusiveGroup(ExclusiveGroup), Error(Error) { + Flags(Flags), + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + IncludeDirs(IncludeDirs), ExclusiveGroup(ExclusiveGroup), Error(Error) { assert(GCCSuffix.empty() || (StringRef(GCCSuffix).front() == '/' && GCCSuffix.size() > 1)); assert(OSSuffix.empty() || @@ -299,6 +305,9 @@ struct MultilibSerialization { std::string Dir; // if this record successfully selects a library dir std::string Error; // if this record reports a fatal error message std::vector Flags; + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + std::vector IncludeDirs; std::string Group; }; @@ -350,6 +359,9 @@ template <> struct llvm::yaml::MappingTraits { io.mapOptional("Dir", V.Dir); io.mapOptional("Error", V.Error); io.mapRequired("Flags", V.Flags); + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + io.mapOptional("IncludeDirs", V.IncludeDirs); io.mapOptional("Group", V.Group); } static std::string validate(IO &io, MultilibSerialization &V) { @@ -359,6 +371,12 @@ template <> struct llvm::yaml::MappingTraits { return "the 'Dir' and 'Error' keys may not both be specified"; if (StringRef(V.Dir).starts_with("/")) return "paths must be relative but \"" + V.Dir + "\" starts with \"/\""; + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + for (const auto &Path : V.IncludeDirs) { + if (StringRef(Path).starts_with("/")) + return "paths must be relative but \"" + Path + "\" starts with \"/\""; + } return std::string{}; } }; @@ -489,7 +507,10 @@ MultilibSet::parseYaml(llvm::MemoryBufferRef Input, Multilibs.reserve(MS.Multilibs.size()); for (const auto &M : MS.Multilibs) { if (!M.Error.empty()) { - Multilibs.emplace_back("", "", "", M.Flags, M.Group, M.Error); + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + Multilibs.emplace_back("", "", "", M.Flags, M.IncludeDirs, M.Group, + M.Error); } else { std::string Dir; if (M.Dir != ".") @@ -498,7 +519,9 @@ MultilibSet::parseYaml(llvm::MemoryBufferRef Input, // Multilib constructor. If we later support more than one type of group, // we'll have to look up the group name in MS.Groups, check its type, and // decide what to do here. - Multilibs.emplace_back(Dir, Dir, Dir, M.Flags, M.Group); + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + Multilibs.emplace_back(Dir, Dir, Dir, M.Flags, M.IncludeDirs, M.Group); } } diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp index 497f3330237b..723ba8d5402b 100644 --- a/clang/lib/Driver/ToolChains/BareMetal.cpp +++ b/clang/lib/Driver/ToolChains/BareMetal.cpp @@ -427,10 +427,22 @@ void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs, const SmallString<128> SysRootDir(computeSysRoot()); if (!SysRootDir.empty()) { for (const Multilib &M : getOrderedMultilibs()) { - SmallString<128> Dir(SysRootDir); - llvm::sys::path::append(Dir, M.includeSuffix()); - llvm::sys::path::append(Dir, "include"); - addSystemInclude(DriverArgs, CC1Args, Dir.str()); + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + if (!M.includeDirs().empty()) { + // Add include directories specified in multilib.yaml under the + // 'IncludeDirs' field + for (const std::string &Path : M.includeDirs()) { + SmallString<128> Dir(SysRoot); + llvm::sys::path::append(Dir, Path); + addSystemInclude(DriverArgs, CC1Args, Dir.str()); + } + } else { + SmallString<128> Dir(SysRootDir); + llvm::sys::path::append(Dir, M.includeSuffix()); + llvm::sys::path::append(Dir, "include"); + addSystemInclude(DriverArgs, CC1Args, Dir.str()); + } } } } @@ -511,6 +523,18 @@ void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs, addSystemInclude(DriverArgs, CC1Args, TargetDir.str()); break; } + // Downstream issue: #446 (Extend the Multilib system to support an + // IncludeDirs field) + if (!M.includeDirs().empty()) { + // Add include directories specified in multilib.yaml under the + // 'IncludeDirs' field + for (const std::string &Path : M.includeDirs()) { + Dir = SysRoot; + llvm::sys::path::append(Dir, Path, "c++", "v1"); + addSystemInclude(DriverArgs, CC1Args, Dir.str()); + } + break; + } // Add generic path if nothing else succeeded so far. llvm::sys::path::append(Dir, "include", "c++", "v1"); addSystemInclude(DriverArgs, CC1Args, Dir.str()); diff --git a/clang/test/Driver/baremetal-multilib-includedirs-error.yaml b/clang/test/Driver/baremetal-multilib-includedirs-error.yaml new file mode 100644 index 000000000000..781f5f907f8f --- /dev/null +++ b/clang/test/Driver/baremetal-multilib-includedirs-error.yaml @@ -0,0 +1,30 @@ +# Downstream issue: #446 (Extend the Multilib system to support an IncludeDirs field) +# REQUIRES: shell +# UNSUPPORTED: system-windows + +# This test demonstrates the new "IncludeDirs" field. +# This field allows specifying include directories through +# multilib.yaml configuration using relative paths. +# Absolute paths should be rejected by the driver +# with an appropriate diagnostic message. +# +# This test verifies that passing an absolute path +# (/include) to IncludeDirs triggers the expected +# error. + +# RUN: not %clang --target=aarch64-none-elf --multi-lib-config=%s %s -o /dev/null 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-ERROR +# CHECK-ERROR: error: paths must be relative but "/include" starts with "/" + +--- +MultilibVersion: 1.0 +Variants: +- Dir: aarch64-none-elf/aarch64a_exn_rtti_unaligned + Flags: + - --target=aarch64-unknown-none-elf + - -munaligned-access + IncludeDirs: + - /include + - aarch64-none-elf/include + +... diff --git a/clang/test/Driver/baremetal-multilib-includedirs.yaml b/clang/test/Driver/baremetal-multilib-includedirs.yaml new file mode 100644 index 000000000000..dc3f8aca5deb --- /dev/null +++ b/clang/test/Driver/baremetal-multilib-includedirs.yaml @@ -0,0 +1,33 @@ +# Downstream issue: #446 (Extend the Multilib system to support an IncludeDirs field) +# REQUIRES: shell +# UNSUPPORTED: system-windows + +# This test demonstrates the new "IncludeDirs" field. +# This field allows specifying include directories through +# multilib.yaml configuration. When this field is present +# it is appended to the sysroot during header path +# resolution ignoring the default bare-metal assumptions. + +# RUN: %clang --multi-lib-config=%s -no-canonical-prefixes -x c++ %s -### -o %t.out 2>&1 \ +# RUN: --target=thumbv7m-none-eabi -mfloat-abi=soft --sysroot= \ +# RUN: | FileCheck %s +# CHECK: "-cc1" "-triple" "thumbv7m-unknown-none-eabi" +# CHECK-SAME: "-internal-isystem" "[[SYSROOT:[^"]*]]/bin/../lib/clang-runtimes/include/c++/v1" +# CHECK-SAME: "-internal-isystem" "[[SYSROOT]]/bin/../lib/clang-runtimes/include-libunwind/c++/v1" +# CHECK-SAME: "-internal-isystem" "[[SYSROOT]]/bin/../lib/clang-runtimes/soft/include/c++/v1" +# CHECK-SAME: "-internal-isystem" "[[SYSROOT]]/bin/../lib/clang-runtimes/include" +# CHECK-SAME: "-internal-isystem" "[[SYSROOT]]/bin/../lib/clang-runtimes/include-libunwind" +# CHECK-SAME: "-internal-isystem" "[[SYSROOT]]/bin/../lib/clang-runtimes/soft/include" + +--- +MultilibVersion: 1.0 +Variants: +- Dir: soft + Flags: + - -mfloat-abi=soft + IncludeDirs: + - include + - include-libunwind + - soft/include + +...