[ATFE] Package common multilib headers into target triple directory. (arm#423)

The current ATFE package redundantly duplicates identical C and C++
headers across more than 100 ibrary variants, despite the fact that
these headers do not vary within a given target triple aside from few.
As a result, each variant currently maintains its own include/ and
include/c++/v1/ directories, leading to unnecessary duplication.

This patch is to refactor the multilib layout to use a single shared
header directory per target triple, eliminating redundant copies of
identical headers across library variants, while keeping
variant-specific headers within each variant’s own include/ directory.

Changing the layout of the package structure will require corresponding
updates to the ATFE build system, as it currently assumes a variant
specific directory hierarchy for headers and libraries during
installation and packaging.

A new Python script has been introduced that runs after the runtime
subproject has been executed for each variant and the non-optimised
multilib directories have been generated. This script identifies and
extracts common headers from the non-optimised multilibs and creates an
optimized multilib directory. In this optimised directory, only the
common headers are centralised, while the remaining contents are
preserved as-is from the non-optimised layout.

A CMake flag called ENABLE_MULTILIB_HEADER_OPTIMISATION has been added
to disable this multilib optimisation phase. When this flag is not set,
the optimised multilib directory will not be generated. Additionally,
the optimisation step is skipped automatically if there are fewer than
two multilib variants to build.

To support this new layout with a centralised include directory, a new
field called IncludeDirs has been added to multilib.yaml. This field
specifies a base directory for locating header files which gets added to
the header search path sequence by clang.

Corresponding changes in Clang:
llvm/llvm-project#146651

Author: simpal01

arm-software/embedded/CMakeLists.txt

@@ -180,6 +180,11 @@ option(
     OFF
 )
 
+option(
+    ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase"
+    ON
+)
+
 # Previously, the LLVM_TOOLCHAIN_LIBRARY_OVERLAY_INSTALL option was
 # called LLVM_TOOLCHAIN_NEWLIB_OVERLAY_INSTALL. Detect a setting of
 # that name, in case it's in an existing CMakeCache.txt or command
@@ -646,6 +651,7 @@ if(NOT PREBUILT_TARGET_LIBRARIES)
         -DLLVM_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/llvm
         -DMULTILIB_JSON=${LLVM_TOOLCHAIN_MULTILIB_JSON}
         -DENABLE_VARIANTS=${ENABLE_VARIANTS_PASSTHROUGH}
+	-DENABLE_MULTILIB_HEADER_OPTIMISATION=${ENABLE_MULTILIB_HEADER_OPTIMISATION}
         -DENABLE_PARALLEL_LIB_CONFIG=${ENABLE_PARALLEL_LIB_CONFIG}
         -DENABLE_PARALLEL_LIB_BUILD=${ENABLE_PARALLEL_LIB_BUILD}
         -DPARALLEL_LIB_BUILD_LEVELS=${PARALLEL_LIB_BUILD_LEVELS}

arm-software/embedded/arm-multilib/CMakeLists.txt

@@ -48,6 +48,7 @@ set(
     fvp/get_fvps.sh"
 )
 set(FVP_CONFIG_DIR "${TOOLCHAIN_SOURCE_DIR}/fvp/config" CACHE STRING "The directory in which the FVP models are installed.")
+option(ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase" ON)
 option(
     ENABLE_PARALLEL_LIB_CONFIG
     "Run the library variant configuration steps in parallel."
@@ -185,10 +186,12 @@ foreach(lib_idx RANGE ${lib_count_dec})
                 set(parent_dir_name arm-none-eabi)
             endif()
             set(destination_directory "${CMAKE_CURRENT_BINARY_DIR}/multilib/${parent_dir_name}/${variant}")
-            install(
-                DIRECTORY ${destination_directory}
-                DESTINATION ${parent_dir_name}
-            )
+            if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION)
+                install(
+                    DIRECTORY ${destination_directory}
+                    DESTINATION ${parent_dir_name}
+                )
+            endif()
             set(variant_json_file ${CMAKE_CURRENT_SOURCE_DIR}/json/variants/${variant_json})
 
             # Read info from the variant specific json.
@@ -251,6 +254,8 @@ foreach(lib_idx RANGE ${lib_count_dec})
                 TEST_EXCLUDE_FROM_MAIN TRUE
             )
 
+            list(APPEND all_runtime_targets runtimes-${variant})
+
             if(ENABLE_PARALLEL_LIB_CONFIG OR ENABLE_PARALLEL_LIB_BUILD)
                 # Create additional steps to configure/build the subprojects.
                 # These are collected to be run together, so that all the
@@ -337,6 +342,11 @@ foreach(lib_idx RANGE ${lib_count_dec})
             foreach(flag ${multilib_flags_list})
                 string(APPEND multilib_yaml_content "  - ${flag}\n")
             endforeach()
+            if(ENABLE_MULTILIB_HEADER_OPTIMISATION)
+                string(APPEND multilib_yaml_content "  IncludeDirs:\n")
+                string(APPEND multilib_yaml_content "  - ${parent_dir_name}/include\n")
+                string(APPEND multilib_yaml_content "  - ${parent_dir_name}/${variant}/include\n")
+            endif()
             string(APPEND multilib_yaml_content "  Group: stdlibs\n")
         else()
             # In place of a json, an error message is expected.
@@ -386,7 +396,25 @@ add_custom_command(
 )
 
 add_custom_target(multilib-yaml ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml)
-install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
-    DESTINATION .
-)
+if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION)
+    install(
+        FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
+        DESTINATION .
+    )
+endif()
+
+# Extract common header files which spans across multilib variant directories.
+if(ENABLE_MULTILIB_HEADER_OPTIMISATION)
+    add_custom_target(generate-multilib-common-headers ALL
+        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/common-headers-generate.py"
+                "${CMAKE_CURRENT_BINARY_DIR}/multilib"
+                "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised"
+        COMMENT "Generating common headers"
+        DEPENDS ${all_runtime_targets} multilib-yaml
+    )
+
+    install(
+        DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised/"
+        DESTINATION .
+    )
+endif()

arm-software/embedded/arm-multilib/common-headers-generate.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+"""
+Identifies and extracts header files that are common across multiple multilib variant directories.
+
+This script scans all variant folders within a multilib target directory
+(e.g., `arm-none-eabi/variant1/include`, `arm-none-eabi/variant2/include`, etc.) and compares all
+`.h` files. If the same file (by name and content) appears in multiple variants, it will be moved
+to a shared include directory at:
+
+    <CMAKE_BINARY_DIR>/multilib-optimised/<target>/include/
+
+for the following multilib targets:
+- arm-none-eabi
+- aarch64-none-elf
+
+Arguments:
+    <CMAKE_BINARY_DIR>/multilib Path to the CMake build directory containing non optmised multilib.
+    eg: build/multilib-builds/multilib/picolibc-build/multilib
+    <CMAKE_BINARY_DIR>/multilib-optimised  Path to the CMake build directory where optimised multilib should be generated..
+    eg: build/multilib-builds/multilib/picolibc-build/multilib-optimised
+
+This is useful to reduce duplication in the toolchain by centralising common headers
+that are shared across architecture variants.
+"""
+import argparse
+import os
+import filecmp
+import shutil
+
+# Define the multilib target dirs which want to process
+MULTILIB_TARGET_DIRS = ["arm-none-eabi", "aarch64-none-elf"]
+
+
+def files_are_identical(f1, f2):
+    return filecmp.cmp(f1, f2, shallow=False)
+
+
+def collect_variant_include_paths(input_target_dir):
+    """
+    Extracts each multilib variant and its corresponding include path from the non-optimised multilib directory.
+    Stores the results to enable later comparison of header contents across different non-optimised multilib variant
+    include paths.
+
+    """
+    variant_include_path = {}
+    for variant in os.listdir(input_target_dir):
+        include_path = os.path.join(input_target_dir, variant, "include")
+        if os.path.isdir(include_path):
+            variant_include_path[variant] = include_path
+    return variant_include_path
+
+
+def extract_common_headers_for_targets(args):
+    if os.path.exists(args.multilib_optimised_dir):
+        shutil.rmtree(args.multilib_optimised_dir)
+
+    for target in MULTILIB_TARGET_DIRS:
+        input_target_dir = os.path.join(
+            os.path.abspath(args.multilib_non_optimised_dir), target
+        )
+        output_target_dir = os.path.join(
+            os.path.abspath(args.multilib_optimised_dir), target
+        )
+        output_include_dir = os.path.join(output_target_dir, "include")
+
+        if not os.path.isdir(input_target_dir):
+            print(
+                f"Skipping extracting the common headers for {target}: input path {input_target_dir} not found"
+            )
+            continue
+
+        variant_includes = collect_variant_include_paths(input_target_dir)
+        if len(variant_includes) < 2:
+            print(
+                f"Skipping extracting the common headers for {target}: not enough variants to compare.At least two variants must be enabled for the multilib header optimisation phase to proceed."
+            )
+            # The script always creates the multilib-optimised folder, even when there's only one variant and no
+            # optimization is applied. In that case, multilib-optimised will just contain a copy of the
+            # single variant from the non-optimised multilib directory.
+            if os.path.exists(args.multilib_non_optimised_dir):
+                shutil.copytree(args.multilib_non_optimised_dir, args.multilib_optimised_dir)
+            return
+
+        # Creating the common include headers for each target
+        os.makedirs(output_include_dir, exist_ok=True)
+
+        # Step 1: compare first two variants and extract the common headers into the targets common include directory
+        base_dir = list(variant_includes.values())[0]
+        compare_dir = list(variant_includes.values())[1]
+        for root, sub_dirs, header_files in os.walk(base_dir):
+            sub_dir_root = os.path.relpath(root, base_dir)
+            for header in header_files:
+                h1 = os.path.join(base_dir, sub_dir_root, header)
+                h2 = os.path.join(compare_dir, sub_dir_root, header)
+                if os.path.exists(h2) and files_are_identical(h1, h2):
+                    out_dir = os.path.join(output_include_dir, sub_dir_root)
+                    os.makedirs(out_dir, exist_ok=True)
+                    shutil.copy2(h1, os.path.join(out_dir, header))
+
+        # Step 2: Compare all the variants with the new common include. Any headers that do not match
+        # and do not exit in common include should retain in their respective variant specific directories.
+        for variant, include_path in variant_includes.items():
+            for root, sub_dirs, header_files in os.walk(include_path):
+                sub_dir_root = os.path.relpath(root, include_path)
+                for header in header_files:
+                    variant_header = os.path.join(include_path, sub_dir_root, header)
+                    common_header = os.path.join(
+                        output_include_dir, sub_dir_root, header
+                    )
+                    if not os.path.exists(common_header) or not files_are_identical(
+                        variant_header, common_header
+                    ):
+                        out_dir = os.path.join(
+                            os.path.abspath(args.multilib_optimised_dir),
+                            target,
+                            variant,
+                            sub_dir_root,
+                            "include",
+                        )
+                        os.makedirs(out_dir, exist_ok=True)
+                        shutil.copy2(variant_header, os.path.join(out_dir, header))
+
+        # Step3: For each variant, the lib and share directories should be copied from the non-optimised multilib
+        # directory as it is.
+        for variant in variant_includes:
+            remaining_dirs = ["lib", "share"]
+            for folder in remaining_dirs:
+                src_dir = os.path.join(input_target_dir, variant, folder)
+                dst_dir = os.path.join(output_target_dir, variant, folder)
+                if os.path.exists(src_dir):
+                    # If destination exists, remove it first
+                    if os.path.exists(dst_dir):
+                        shutil.rmtree(dst_dir)
+                    os.makedirs(os.path.dirname(dst_dir), exist_ok=True)
+                    shutil.copytree(src_dir, dst_dir)
+                else:
+                    print(f"Warning: {src_dir} does not exist and will be skipped.")
+
+        # Step4: Copy multilib.yaml file as it is from the non-optimised multilib directoy.
+        src_yaml = os.path.join(args.multilib_non_optimised_dir, "multilib.yaml")
+        dst_yaml = os.path.join(args.multilib_optimised_dir, "multilib.yaml")
+        if os.path.exists(src_yaml):
+            shutil.copy2(src_yaml, dst_yaml)
+        else:
+            raise FileNotFoundError(f"Source yaml '{src_yaml}' does not exist.")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "multilib_non_optimised_dir",
+        help="CMake binary directory containing the non-optimised multilib headers",
+    )
+    parser.add_argument(
+        "multilib_optimised_dir",
+        help="CMake binary directory where the optimised multilib headers should be generated",
+    )
+    args = parser.parse_args()
+
+    extract_common_headers_for_targets(args)
+
+
+if __name__ == "__main__":
+    main()