[ATFE] Centralising common headers across multilib variants.

arm-software/embedded/CMakeLists.txt

@@ -180,6 +180,11 @@ option(
     OFF
 )
 
+option(
+    ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase"
+    ON
+)
+
 # Previously, the LLVM_TOOLCHAIN_LIBRARY_OVERLAY_INSTALL option was
 # called LLVM_TOOLCHAIN_NEWLIB_OVERLAY_INSTALL. Detect a setting of
 # that name, in case it's in an existing CMakeCache.txt or command
@@ -646,6 +651,7 @@ if(NOT PREBUILT_TARGET_LIBRARIES)
         -DLLVM_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/llvm
         -DMULTILIB_JSON=${LLVM_TOOLCHAIN_MULTILIB_JSON}
         -DENABLE_VARIANTS=${ENABLE_VARIANTS_PASSTHROUGH}
+	-DENABLE_MULTILIB_HEADER_OPTIMISATION=${ENABLE_MULTILIB_HEADER_OPTIMISATION}
         -DENABLE_PARALLEL_LIB_CONFIG=${ENABLE_PARALLEL_LIB_CONFIG}
         -DENABLE_PARALLEL_LIB_BUILD=${ENABLE_PARALLEL_LIB_BUILD}
         -DPARALLEL_LIB_BUILD_LEVELS=${PARALLEL_LIB_BUILD_LEVELS}

arm-software/embedded/arm-multilib/CMakeLists.txt

@@ -48,6 +48,7 @@ set(
     fvp/get_fvps.sh"
 )
 set(FVP_CONFIG_DIR "${TOOLCHAIN_SOURCE_DIR}/fvp/config" CACHE STRING "The directory in which the FVP models are installed.")
+option(ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase" ON)
 option(
     ENABLE_PARALLEL_LIB_CONFIG
     "Run the library variant configuration steps in parallel."
@@ -185,10 +186,12 @@ foreach(lib_idx RANGE ${lib_count_dec})
                 set(parent_dir_name arm-none-eabi)
             endif()
             set(destination_directory "${CMAKE_CURRENT_BINARY_DIR}/multilib/${parent_dir_name}/${variant}")
-            install(
-                DIRECTORY ${destination_directory}
-                DESTINATION ${parent_dir_name}
-            )
+            if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION)
+                install(
+                    DIRECTORY ${destination_directory}
+                    DESTINATION ${parent_dir_name}
+                )
+            endif()
             set(variant_json_file ${CMAKE_CURRENT_SOURCE_DIR}/json/variants/${variant_json})
 
             # Read info from the variant specific json.
@@ -251,6 +254,8 @@ foreach(lib_idx RANGE ${lib_count_dec})
                 TEST_EXCLUDE_FROM_MAIN TRUE
             )
 
+            list(APPEND all_runtime_targets runtimes-${variant})
+
             if(ENABLE_PARALLEL_LIB_CONFIG OR ENABLE_PARALLEL_LIB_BUILD)
                 # Create additional steps to configure/build the subprojects.
                 # These are collected to be run together, so that all the
@@ -337,6 +342,11 @@ foreach(lib_idx RANGE ${lib_count_dec})
             foreach(flag ${multilib_flags_list})
                 string(APPEND multilib_yaml_content "  - ${flag}\n")
             endforeach()
+            if(ENABLE_MULTILIB_HEADER_OPTIMISATION)
+                string(APPEND multilib_yaml_content "  IncludeDirs:\n")
+                string(APPEND multilib_yaml_content "  - ${parent_dir_name}/${variant}/include\n")
+                string(APPEND multilib_yaml_content "  - ${parent_dir_name}/include\n")
+            endif()
             string(APPEND multilib_yaml_content "  Group: stdlibs\n")
         else()
             # In place of a json, an error message is expected.
@@ -386,7 +396,25 @@ add_custom_command(
 )
 
 add_custom_target(multilib-yaml ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml)
-install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
-    DESTINATION .
-)
+if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION)
+    install(
+        FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
+        DESTINATION .
+    )
+endif()
+
+# Extract common header files which spans across multilib variant directories.
+if(ENABLE_MULTILIB_HEADER_OPTIMISATION)
+    add_custom_target(generate-multilib-common-headers ALL
+        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/common-headers-generate.py"
+                "${CMAKE_CURRENT_BINARY_DIR}/multilib"
+                "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised"
+        COMMENT "Generating common headers"
+        DEPENDS ${all_runtime_targets} multilib-yaml
+    )
+
+    install(
+        DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised/"
+        DESTINATION .
+    )
+endif()

arm-software/embedded/arm-multilib/common-headers-generate.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+"""
+Identifies and extracts header files that are common across multiple multilib variant directories.
+
+This script scans all variant folders within a multilib target directory
+(e.g., `arm-none-eabi/variant1/include`, `arm-none-eabi/variant2/include`, etc.) and compares all
+`.h` files. If the same file (by name and content) appears in multiple variants, it will be moved
+to a shared include directory at:
+
+    <CMAKE_BINARY_DIR>/multilib-optimised/<target>/include/
+
+for the following multilib targets:
+- arm-none-eabi
+- aarch64-none-elf
+
+Arguments:
+    <CMAKE_BINARY_DIR>/multilib Path to the CMake build directory containing non optmised multilib.
+    eg: build/multilib-builds/multilib/picolibc-build/multilib
+    <CMAKE_BINARY_DIR>/multilib-optimised  Path to the CMake build directory where optimised multilib should be generated..
+    eg: build/multilib-builds/multilib/picolibc-build/multilib-optimised
+
+This is useful to reduce duplication in the toolchain by centralising common headers
+that are shared across architecture variants.
+"""
+import argparse
+import os
+import filecmp
+import shutil
+
+# Define the multilib target dirs which want to process
+MULTILIB_TARGET_DIRS = ["arm-none-eabi", "aarch64-none-elf"]
+
+
+def files_are_identical(f1, f2):
+    return filecmp.cmp(f1, f2, shallow=False)
+
+
+def collect_variant_include_paths(input_target_dir):
+    """
+    Extracts each multilib variant and its corresponding include path from the non-optimised multilib directory.
+    Stores the results to enable later comparison of header contents across different non-optimised multilib variant
+    include paths.
+
+    """
+    variant_include_path = {}
+    for variant in os.listdir(input_target_dir):
+        include_path = os.path.join(input_target_dir, variant, "include")
+        if os.path.isdir(include_path):
+            variant_include_path[variant] = include_path
+    return variant_include_path
+
+
+def extract_common_headers_for_targets(args):
+    if os.path.exists(args.multilib_optimised_dir):
+        shutil.rmtree(args.multilib_optimised_dir)
+
+    for target in MULTILIB_TARGET_DIRS:
+        input_target_dir = os.path.join(
+            os.path.abspath(args.multilib_non_optimised_dir), target
+        )
+        output_target_dir = os.path.join(
+            os.path.abspath(args.multilib_optimised_dir), target
+        )
+        output_include_dir = os.path.join(output_target_dir, "include")
+
+        if not os.path.isdir(input_target_dir):
+            print(
+                f"Skipping extracting the common headers for {target}: input path {input_target_dir} not found"
+            )
+            continue
+
+        variant_includes = collect_variant_include_paths(input_target_dir)
+        if len(variant_includes) < 2:
+            print(
+                f"Skipping extracting the common headers for {target}: not enough variants to compare.At least two variants must be enabled for the multilib header optimisation phase to proceed."
+            )
+            # The script always creates the multilib-optimised folder, even when there's only one variant and no
+            # optimization is applied. In that case, multilib-optimised will just contain a copy of the
+            # single variant from the non-optimised multilib directory.
+            if os.path.exists(args.multilib_non_optimised_dir):
+                shutil.copytree(args.multilib_non_optimised_dir, args.multilib_optimised_dir)
+            return
+
+        # Creating the common include headers for each target
+        os.makedirs(output_include_dir, exist_ok=True)
+
+        # Step 1: compare first two variants and extract the common headers into the targets common include directory
+        base_dir = list(variant_includes.values())[0]
+        compare_dir = list(variant_includes.values())[1]
+        for root, sub_dirs, header_files in os.walk(base_dir):
+            sub_dir_root = os.path.relpath(root, base_dir)
+            for header in header_files:
+                h1 = os.path.join(base_dir, sub_dir_root, header)
+                h2 = os.path.join(compare_dir, sub_dir_root, header)
+                if os.path.exists(h2) and files_are_identical(h1, h2):
+                    out_dir = os.path.join(output_include_dir, sub_dir_root)
+                    os.makedirs(out_dir, exist_ok=True)
+                    shutil.copy2(h1, os.path.join(out_dir, header))
+
+        # Step 2: Compare all the variants with the new common include. Any headers that do not match
+        # and do not exit in common include should retain in their respective variant specific directories.
+        for variant, include_path in variant_includes.items():
+            for root, sub_dirs, header_files in os.walk(include_path):
+                sub_dir_root = os.path.relpath(root, include_path)
+                for header in header_files:
+                    variant_header = os.path.join(include_path, sub_dir_root, header)
+                    common_header = os.path.join(
+                        output_include_dir, sub_dir_root, header
+                    )
+                    if not os.path.exists(common_header) or not files_are_identical(
+                        variant_header, common_header
+                    ):
+                        out_dir = os.path.join(
+                            os.path.abspath(args.multilib_optimised_dir),
+                            target,
+                            variant,
+                            sub_dir_root,
+                            "include",
+                        )
+                        os.makedirs(out_dir, exist_ok=True)
+                        shutil.copy2(variant_header, os.path.join(out_dir, header))
+
+        # Step3: For each variant, the lib and share directories should be copied from the non-optimised multilib
+        # directory as it is.
+        for variant in variant_includes:
+            remaining_dirs = ["lib", "share"]
+            for folder in remaining_dirs:
+                src_dir = os.path.join(input_target_dir, variant, folder)
+                dst_dir = os.path.join(output_target_dir, variant, folder)
+                if os.path.exists(src_dir):
+                    # If destination exists, remove it first
+                    if os.path.exists(dst_dir):
+                        shutil.rmtree(dst_dir)
+                    os.makedirs(os.path.dirname(dst_dir), exist_ok=True)
+                    shutil.copytree(src_dir, dst_dir)
+                else:
+                    print(f"Warning: {src_dir} does not exist and will be skipped.")
+
+        # Step4: Copy multilib.yaml file as it is from the non-optimised multilib directoy.
+        src_yaml = os.path.join(args.multilib_non_optimised_dir, "multilib.yaml")
+        dst_yaml = os.path.join(args.multilib_optimised_dir, "multilib.yaml")
+        if os.path.exists(src_yaml):
+            shutil.copy2(src_yaml, dst_yaml)
+        else:
+            raise FileNotFoundError(f"Source yaml '{src_yaml}' does not exist.")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "multilib_non_optimised_dir",
+        help="CMake binary directory containing the non-optimised multilib headers",
+    )
+    parser.add_argument(
+        "multilib_optimised_dir",
+        help="CMake binary directory where the optimised multilib headers should be generated",
+    )
+    args = parser.parse_args()
+
+    extract_common_headers_for_targets(args)
+
+
+if __name__ == "__main__":
+    main()

clang/include/clang/Driver/Multilib.h

@@ -35,12 +35,18 @@ class Driver;
 class Multilib {
 public:
   using flags_list = std::vector<std::string>;
+  // Downstream issue: #446 (Extend the Multilib system to support an
+  // IncludeDirs field)
+  using includedirs_list = std::vector<std::string>;
 
 private:
   std::string GCCSuffix;
   std::string OSSuffix;
   std::string IncludeSuffix;
   flags_list Flags;
+  // Downstream issue: #446 (Extend the Multilib system to support an
+  // IncludeDirs field)
+  includedirs_list IncludeDirs;
 
   // Optionally, a multilib can be assigned a string tag indicating that it's
   // part of a group of mutually exclusive possibilities. If two or more
@@ -62,6 +68,9 @@ class Multilib {
   /// This is enforced with an assert in the constructor.
   Multilib(StringRef GCCSuffix = {}, StringRef OSSuffix = {},
            StringRef IncludeSuffix = {}, const flags_list &Flags = flags_list(),
+           // Downstream issue: #446 (Extend the Multilib system to support an
+           // IncludeDirs field)
+           const includedirs_list &IncludeDirs = includedirs_list(),
            StringRef ExclusiveGroup = {},
            std::optional<StringRef> Error = std::nullopt);
 
@@ -81,6 +90,12 @@ class Multilib {
   /// All elements begin with either '-' or '!'
   const flags_list &flags() const { return Flags; }
 
+  // Downstream issue: #446 (Extend the Multilib system to support an
+  // IncludeDirs field)
+  /// Get the include directories specified in multilib.yaml under the
+  /// 'IncludeDirs' field
+  const includedirs_list &includeDirs() const { return IncludeDirs; }
+
   /// Get the exclusive group label.
   const std::string &exclusiveGroup() const { return ExclusiveGroup; }
 

clang/lib/Driver/Multilib.cpp

@@ -29,9 +29,15 @@ using namespace llvm::sys;
 
 Multilib::Multilib(StringRef GCCSuffix, StringRef OSSuffix,
                    StringRef IncludeSuffix, const flags_list &Flags,
+                   // Downstream issue: #446 (Extend the Multilib system to
+                   // support an IncludeDirs field)
+                   const includedirs_list &IncludeDirs,
                    StringRef ExclusiveGroup, std::optional<StringRef> Error)
     : GCCSuffix(GCCSuffix), OSSuffix(OSSuffix), IncludeSuffix(IncludeSuffix),
-      Flags(Flags), ExclusiveGroup(ExclusiveGroup), Error(Error) {
+      Flags(Flags),
+      // Downstream issue: #446 (Extend the Multilib system to support an
+      // IncludeDirs field)
+      IncludeDirs(IncludeDirs), ExclusiveGroup(ExclusiveGroup), Error(Error) {
   assert(GCCSuffix.empty() ||
          (StringRef(GCCSuffix).front() == '/' && GCCSuffix.size() > 1));
   assert(OSSuffix.empty() ||
@@ -299,6 +305,9 @@ struct MultilibSerialization {
   std::string Dir;        // if this record successfully selects a library dir
   std::string Error;      // if this record reports a fatal error message
   std::vector<std::string> Flags;
+  // Downstream issue: #446 (Extend the Multilib system to support an
+  // IncludeDirs field)
+  std::vector<std::string> IncludeDirs;
   std::string Group;
 };
 
@@ -350,6 +359,9 @@ template <> struct llvm::yaml::MappingTraits<MultilibSerialization> {
     io.mapOptional("Dir", V.Dir);
     io.mapOptional("Error", V.Error);
     io.mapRequired("Flags", V.Flags);
+    // Downstream issue: #446 (Extend the Multilib system to support an
+    // IncludeDirs field)
+    io.mapOptional("IncludeDirs", V.IncludeDirs);
     io.mapOptional("Group", V.Group);
   }
   static std::string validate(IO &io, MultilibSerialization &V) {
@@ -359,6 +371,12 @@ template <> struct llvm::yaml::MappingTraits<MultilibSerialization> {
       return "the 'Dir' and 'Error' keys may not both be specified";
     if (StringRef(V.Dir).starts_with("/"))
       return "paths must be relative but \"" + V.Dir + "\" starts with \"/\"";
+    // Downstream issue: #446 (Extend the Multilib system to support an
+    // IncludeDirs field)
+    for (const auto &Path : V.IncludeDirs) {
+      if (StringRef(Path).starts_with("/"))
+        return "paths must be relative but \"" + Path + "\" starts with \"/\"";
+    }
     return std::string{};
   }
 };
@@ -489,7 +507,10 @@ MultilibSet::parseYaml(llvm::MemoryBufferRef Input,
   Multilibs.reserve(MS.Multilibs.size());
   for (const auto &M : MS.Multilibs) {
     if (!M.Error.empty()) {
-      Multilibs.emplace_back("", "", "", M.Flags, M.Group, M.Error);
+      // Downstream issue: #446 (Extend the Multilib system to support an
+      // IncludeDirs field)
+      Multilibs.emplace_back("", "", "", M.Flags, M.IncludeDirs, M.Group,
+                             M.Error);
     } else {
       std::string Dir;
       if (M.Dir != ".")
@@ -498,7 +519,9 @@ MultilibSet::parseYaml(llvm::MemoryBufferRef Input,
       // Multilib constructor. If we later support more than one type of group,
       // we'll have to look up the group name in MS.Groups, check its type, and
       // decide what to do here.
-      Multilibs.emplace_back(Dir, Dir, Dir, M.Flags, M.Group);
+      // Downstream issue: #446 (Extend the Multilib system to support an
+      // IncludeDirs field)
+      Multilibs.emplace_back(Dir, Dir, Dir, M.Flags, M.IncludeDirs, M.Group);
     }
   }