[ATFE] Package common multilib headers into target triple directory.

arm-software/embedded/CMakeLists.txt

@@ -180,6 +180,11 @@ option(
     OFF
 )
 
+option(
+    ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase"
+    ON
+)
+
 # Previously, the LLVM_TOOLCHAIN_LIBRARY_OVERLAY_INSTALL option was
 # called LLVM_TOOLCHAIN_NEWLIB_OVERLAY_INSTALL. Detect a setting of
 # that name, in case it's in an existing CMakeCache.txt or command
@@ -646,6 +651,7 @@ if(NOT PREBUILT_TARGET_LIBRARIES)
         -DLLVM_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/llvm
         -DMULTILIB_JSON=${LLVM_TOOLCHAIN_MULTILIB_JSON}
         -DENABLE_VARIANTS=${ENABLE_VARIANTS_PASSTHROUGH}
+	-DENABLE_MULTILIB_HEADER_OPTIMISATION=${ENABLE_MULTILIB_HEADER_OPTIMISATION}
         -DENABLE_PARALLEL_LIB_CONFIG=${ENABLE_PARALLEL_LIB_CONFIG}
         -DENABLE_PARALLEL_LIB_BUILD=${ENABLE_PARALLEL_LIB_BUILD}
         -DPARALLEL_LIB_BUILD_LEVELS=${PARALLEL_LIB_BUILD_LEVELS}

arm-software/embedded/arm-multilib/CMakeLists.txt

@@ -48,6 +48,7 @@ set(
     fvp/get_fvps.sh"
 )
 set(FVP_CONFIG_DIR "${TOOLCHAIN_SOURCE_DIR}/fvp/config" CACHE STRING "The directory in which the FVP models are installed.")
+option(ENABLE_MULTILIB_HEADER_OPTIMISATION "Enable multilib header optimisation phase" ON)
 option(
     ENABLE_PARALLEL_LIB_CONFIG
     "Run the library variant configuration steps in parallel."
@@ -185,10 +186,12 @@ foreach(lib_idx RANGE ${lib_count_dec})
                 set(parent_dir_name arm-none-eabi)
             endif()
             set(destination_directory "${CMAKE_CURRENT_BINARY_DIR}/multilib/${parent_dir_name}/${variant}")
-            install(
-                DIRECTORY ${destination_directory}
-                DESTINATION ${parent_dir_name}
-            )
+            if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION)
+                install(
+                    DIRECTORY ${destination_directory}
+                    DESTINATION ${parent_dir_name}
+                )
+            endif()
             set(variant_json_file ${CMAKE_CURRENT_SOURCE_DIR}/json/variants/${variant_json})
 
             # Read info from the variant specific json.
@@ -251,6 +254,8 @@ foreach(lib_idx RANGE ${lib_count_dec})
                 TEST_EXCLUDE_FROM_MAIN TRUE
             )
 
+            list(APPEND all_runtime_targets runtimes-${variant})
+
             if(ENABLE_PARALLEL_LIB_CONFIG OR ENABLE_PARALLEL_LIB_BUILD)
                 # Create additional steps to configure/build the subprojects.
                 # These are collected to be run together, so that all the
@@ -337,6 +342,11 @@ foreach(lib_idx RANGE ${lib_count_dec})
             foreach(flag ${multilib_flags_list})
                 string(APPEND multilib_yaml_content "  - ${flag}\n")
             endforeach()
+            if(ENABLE_MULTILIB_HEADER_OPTIMISATION)
+                string(APPEND multilib_yaml_content "  IncludeDirs:\n")
+                string(APPEND multilib_yaml_content "  - ${parent_dir_name}/include\n")
+                string(APPEND multilib_yaml_content "  - ${parent_dir_name}/${variant}/include\n")
+            endif()
             string(APPEND multilib_yaml_content "  Group: stdlibs\n")
         else()
             # In place of a json, an error message is expected.
@@ -386,7 +396,25 @@ add_custom_command(
 )
 
 add_custom_target(multilib-yaml ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml)
-install(
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
-    DESTINATION .
-)
+if(NOT ENABLE_MULTILIB_HEADER_OPTIMISATION)
+    install(
+        FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
+        DESTINATION .
+    )
+endif()
+
+# Extract common header files which spans across multilib variant directories.
+if(ENABLE_MULTILIB_HEADER_OPTIMISATION)
+    add_custom_target(generate-multilib-common-headers ALL
+        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/common-headers-generate.py"
+                "${CMAKE_CURRENT_BINARY_DIR}/multilib"
+                "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised"
+        COMMENT "Generating common headers"
+        DEPENDS ${all_runtime_targets} multilib-yaml
+    )
+
+    install(
+        DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/multilib-optimised/"
+        DESTINATION .
+    )
+endif()

arm-software/embedded/arm-multilib/common-headers-generate.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+"""
+Identifies and extracts header files that are common across multiple multilib variant directories.
+
+This script scans all variant folders within a multilib target directory
+(e.g., `arm-none-eabi/variant1/include`, `arm-none-eabi/variant2/include`, etc.) and compares all
+`.h` files. If the same file (by name and content) appears in multiple variants, it will be moved
+to a shared include directory at:
+
+    <CMAKE_BINARY_DIR>/multilib-optimised/<target>/include/
+
+for the following multilib targets:
+- arm-none-eabi
+- aarch64-none-elf
+
+Arguments:
+    <CMAKE_BINARY_DIR>/multilib Path to the CMake build directory containing non optmised multilib.
+    eg: build/multilib-builds/multilib/picolibc-build/multilib
+    <CMAKE_BINARY_DIR>/multilib-optimised  Path to the CMake build directory where optimised multilib should be generated..
+    eg: build/multilib-builds/multilib/picolibc-build/multilib-optimised
+
+This is useful to reduce duplication in the toolchain by centralising common headers
+that are shared across architecture variants.
+"""
+import argparse
+import os
+import filecmp
+import shutil
+
+# Define the multilib target dirs which want to process
+MULTILIB_TARGET_DIRS = ["arm-none-eabi", "aarch64-none-elf"]
+
+
+def files_are_identical(f1, f2):
+    return filecmp.cmp(f1, f2, shallow=False)
+
+
+def collect_variant_include_paths(input_target_dir):
+    """
+    Extracts each multilib variant and its corresponding include path from the non-optimised multilib directory.
+    Stores the results to enable later comparison of header contents across different non-optimised multilib variant
+    include paths.
+
+    """
+    variant_include_path = {}
+    for variant in os.listdir(input_target_dir):
+        include_path = os.path.join(input_target_dir, variant, "include")
+        if os.path.isdir(include_path):
+            variant_include_path[variant] = include_path
+    return variant_include_path
+
+
+def extract_common_headers_for_targets(args):
+    if os.path.exists(args.multilib_optimised_dir):
+        shutil.rmtree(args.multilib_optimised_dir)
+
+    for target in MULTILIB_TARGET_DIRS:
+        input_target_dir = os.path.join(
+            os.path.abspath(args.multilib_non_optimised_dir), target
+        )
+        output_target_dir = os.path.join(
+            os.path.abspath(args.multilib_optimised_dir), target
+        )
+        output_include_dir = os.path.join(output_target_dir, "include")
+
+        if not os.path.isdir(input_target_dir):
+            print(
+                f"Skipping extracting the common headers for {target}: input path {input_target_dir} not found"
+            )
+            continue
+
+        variant_includes = collect_variant_include_paths(input_target_dir)
+        if len(variant_includes) < 2:
+            print(
+                f"Skipping extracting the common headers for {target}: not enough variants to compare.At least two variants must be enabled for the multilib header optimisation phase to proceed."
+            )
+            # The script always creates the multilib-optimised folder, even when there's only one variant and no
+            # optimization is applied. In that case, multilib-optimised will just contain a copy of the
+            # single variant from the non-optimised multilib directory.
+            if os.path.exists(args.multilib_non_optimised_dir):
+                shutil.copytree(args.multilib_non_optimised_dir, args.multilib_optimised_dir)
+            return
+
+        # Creating the common include headers for each target
+        os.makedirs(output_include_dir, exist_ok=True)
+
+        # Step 1: compare first two variants and extract the common headers into the targets common include directory
+        base_dir = list(variant_includes.values())[0]
+        compare_dir = list(variant_includes.values())[1]
+        for root, sub_dirs, header_files in os.walk(base_dir):
+            sub_dir_root = os.path.relpath(root, base_dir)
+            for header in header_files:
+                h1 = os.path.join(base_dir, sub_dir_root, header)
+                h2 = os.path.join(compare_dir, sub_dir_root, header)
+                if os.path.exists(h2) and files_are_identical(h1, h2):
+                    out_dir = os.path.join(output_include_dir, sub_dir_root)
+                    os.makedirs(out_dir, exist_ok=True)
+                    shutil.copy2(h1, os.path.join(out_dir, header))
+
+        # Step 2: Compare all the variants with the new common include. Any headers that do not match
+        # and do not exit in common include should retain in their respective variant specific directories.
+        for variant, include_path in variant_includes.items():
+            for root, sub_dirs, header_files in os.walk(include_path):
+                sub_dir_root = os.path.relpath(root, include_path)
+                for header in header_files:
+                    variant_header = os.path.join(include_path, sub_dir_root, header)
+                    common_header = os.path.join(
+                        output_include_dir, sub_dir_root, header
+                    )
+                    if not os.path.exists(common_header) or not files_are_identical(
+                        variant_header, common_header
+                    ):
+                        out_dir = os.path.join(
+                            os.path.abspath(args.multilib_optimised_dir),
+                            target,
+                            variant,
+                            sub_dir_root,
+                            "include",
+                        )
+                        os.makedirs(out_dir, exist_ok=True)
+                        shutil.copy2(variant_header, os.path.join(out_dir, header))
+
+        # Step3: For each variant, the lib and share directories should be copied from the non-optimised multilib
+        # directory as it is.
+        for variant in variant_includes:
+            remaining_dirs = ["lib", "share"]
+            for folder in remaining_dirs:
+                src_dir = os.path.join(input_target_dir, variant, folder)
+                dst_dir = os.path.join(output_target_dir, variant, folder)
+                if os.path.exists(src_dir):
+                    # If destination exists, remove it first
+                    if os.path.exists(dst_dir):
+                        shutil.rmtree(dst_dir)
+                    os.makedirs(os.path.dirname(dst_dir), exist_ok=True)
+                    shutil.copytree(src_dir, dst_dir)
+                else:
+                    print(f"Warning: {src_dir} does not exist and will be skipped.")
+
+        # Step4: Copy multilib.yaml file as it is from the non-optimised multilib directoy.
+        src_yaml = os.path.join(args.multilib_non_optimised_dir, "multilib.yaml")
+        dst_yaml = os.path.join(args.multilib_optimised_dir, "multilib.yaml")
+        if os.path.exists(src_yaml):
+            shutil.copy2(src_yaml, dst_yaml)
+        else:
+            raise FileNotFoundError(f"Source yaml '{src_yaml}' does not exist.")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "multilib_non_optimised_dir",
+        help="CMake binary directory containing the non-optimised multilib headers",
+    )
+    parser.add_argument(
+        "multilib_optimised_dir",
+        help="CMake binary directory where the optimised multilib headers should be generated",
+    )
+    args = parser.parse_args()
+
+    extract_common_headers_for_targets(args)
+
+
+if __name__ == "__main__":
+    main()