Move CMake code for building multilib and variants to subprojects

This patch restructures the top-level CMake script in order to split
building the library variants into a separate CMake project. This
better encapsulates the code to build the variants, and untethers it
from the toolchain builds so that libraries can be developed and
tested without having the rebuild the entire toolchain.

Similarly, a new subproject has been created to build a multilib set
of library variants, using a selected C library. A simple JSON format
has been used to define the flags and variants, which are then built
and collected into the appropriate layout.

Author: dcandler

CMakeLists.txt

@@ -0,0 +1,243 @@
+#
+# Copyright (c) 2024, Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# CMake build for a multilib layout of library variants, with each
+# variant in a subdirectory and a multilib.yaml file to map flags to
+# a variant.
+
+cmake_minimum_required(VERSION 3.20)
+
+project(arm-multilib)
+
+# Root directory of the repo.
+set(TOOLCHAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..)
+
+# Cache variables to be set by user
+set(MULTILIB_JSON "" CACHE STRING "JSON file to load library definitions from.")
+set(ENABLE_VARIANTS "all" CACHE STRING "Semicolon separated list of variants to build, or \"all\". Must match entries in the json.")
+set(C_LIBRARY "picolibc" CACHE STRING "Which C library to use.")
+set_property(CACHE C_LIBRARY PROPERTY STRINGS picolibc newlib llvmlibc)
+set(LLVM_BINARY_DIR "" CACHE PATH "Path to LLVM toolchain build or install root.")
+
+# If a compiler launcher such as ccache has been set, it should be
+# passed down to each subproject build.
+set(compiler_launcher_cmake_args "")
+if(CMAKE_C_COMPILER_LAUNCHER)
+    list(APPEND compiler_launcher_cmake_args "-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}")
+endif()
+if(CMAKE_CXX_COMPILER_LAUNCHER)
+    list(APPEND compiler_launcher_cmake_args "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}")
+endif()
+
+# Arguments to pass down to the library projects.
+foreach(arg
+    LLVM_BINARY_DIR
+    FVP_INSTALL_DIR
+    FVP_CONFIG_DIR
+)
+    if(${arg})
+        list(APPEND passthrough_dirs "-D${arg}=${${arg}}")
+    endif()
+endforeach()
+
+include(ExternalProject)
+include(${TOOLCHAIN_SOURCE_DIR}/cmake/fetch_llvm.cmake)
+list(APPEND passthrough_dirs "-DFETCHCONTENT_SOURCE_DIR_LLVMPROJECT=${FETCHCONTENT_SOURCE_DIR_LLVMPROJECT}")
+if(C_LIBRARY STREQUAL picolibc)
+    include(${TOOLCHAIN_SOURCE_DIR}/cmake/fetch_picolibc.cmake)
+    list(APPEND passthrough_dirs "-DFETCHCONTENT_SOURCE_DIR_PICOLIBC=${FETCHCONTENT_SOURCE_DIR_PICOLIBC}")
+elseif(C_LIBRARY STREQUAL newlib)
+    include(${TOOLCHAIN_SOURCE_DIR}/cmake/fetch_newlib.cmake)
+    list(APPEND passthrough_dirs "-DFETCHCONTENT_SOURCE_DIR_NEWLIB=${FETCHCONTENT_SOURCE_DIR_NEWLIB}")
+endif()
+
+# Create one target to run all the tests.
+add_custom_target(check-${C_LIBRARY})
+add_custom_target(check-compiler-rt)
+add_custom_target(check-cxx)
+add_custom_target(check-cxxabi)
+add_custom_target(check-unwind)
+
+add_custom_target(check-all)
+add_dependencies(
+    check-all
+    check-${C_LIBRARY}
+    check-compiler-rt
+    check-cxx
+    check-cxxabi
+    check-unwind
+)
+
+# Read the JSON file to load a multilib configuration.
+file(READ ${MULTILIB_JSON} multilib_json_str)
+string(JSON multilib_defs GET ${multilib_json_str} "libs")
+
+string(JSON lib_count LENGTH ${multilib_defs})
+math(EXPR lib_count_dec "${lib_count} - 1")
+
+foreach(lib_idx RANGE ${lib_count_dec})
+    string(JSON lib_def GET ${multilib_defs} ${lib_idx})
+    string(JSON variant GET ${lib_def} "variant")
+
+    if (variant IN_LIST ENABLE_VARIANTS OR ENABLE_VARIANTS STREQUAL "all")
+        string(JSON variant_multilib_flags GET ${lib_def} "flags")
+        # Placeholder libraries won't have a json, so store the error in
+        # a variable so a fatal error isn't generated.
+        string(JSON variant_json ERROR_VARIABLE json_error GET ${lib_def} "json")
+        
+        if(NOT variant_json STREQUAL "json-NOTFOUND")
+            # Sort by target triple
+            if(variant MATCHES "^aarch64")
+                set(parent_dir_name aarch64-none-elf)
+            else()
+                set(parent_dir_name arm-none-eabi)
+            endif()
+            set(destination_directory "${CMAKE_CURRENT_BINARY_DIR}/multilib/${parent_dir_name}/${variant}")
+            install(
+                DIRECTORY ${destination_directory}
+                DESTINATION ${parent_dir_name}
+            )
+            set(variant_json_file ${CMAKE_CURRENT_BINARY_DIR}/json/variants/${variant_json})
+
+            ExternalProject_Add(
+                runtimes-${variant}
+                PREFIX ${CMAKE_BINARY_DIR}/lib-builds
+                SOURCE_DIR ${TOOLCHAIN_SOURCE_DIR}/arm-runtimes
+                INSTALL_DIR ${destination_directory}
+                CMAKE_ARGS
+                ${compiler_launcher_cmake_args}
+                ${passthrough_dirs}
+                -DVARIANT_JSON=${variant_json_file}
+                -DC_LIBRARY=${C_LIBRARY}
+                -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
+                STEP_TARGETS build install
+                USES_TERMINAL_CONFIGURE FALSE
+                USES_TERMINAL_BUILD TRUE
+                USES_TERMINAL_TEST TRUE
+                LIST_SEPARATOR ,
+                CONFIGURE_HANDLED_BY_BUILD TRUE
+                TEST_EXCLUDE_FROM_MAIN TRUE
+            )
+
+            # Read info from the variant specific json.
+            # From the json, check which tests are enabled.
+            file(READ ${variant_json_file} variant_json_str)
+            foreach(test_enable_var
+                ENABLE_LIBC_TESTS
+                ENABLE_COMPILER_RT_TESTS
+                ENABLE_LIBCXX_TESTS
+            )
+                string(JSON read_${test_enable_var} ERROR_VARIABLE json_error GET ${variant_json_str} "args" ${C_LIBRARY} ${test_enable_var})
+                if(read_${test_enable_var} STREQUAL "json-NOTFOUND")
+                    string(JSON read_${test_enable_var} ERROR_VARIABLE json_error GET ${variant_json_str} "args" "common" ${test_enable_var})
+                    if(read_${test_enable_var} STREQUAL "json-NOTFOUND")
+                        set(read_${test_enable_var} "OFF")
+                    endif()
+                endif()
+            endforeach()
+            set(check_targets "")
+            if(read_ENABLE_LIBC_TESTS)
+                list(APPEND check_targets check-${C_LIBRARY})
+            endif()
+            if(read_ENABLE_COMPILER_RT_TESTS)
+                list(APPEND check_targets check-compiler-rt)
+            endif()
+            if(read_ENABLE_LIBCXX_TESTS)
+                list(APPEND check_targets check-cxx)
+                list(APPEND check_targets check-cxxabi)
+                list(APPEND check_targets check-unwind)
+            endif()
+            foreach(check_target ${check_targets})
+                ExternalProject_Add_Step(
+                    runtimes-${variant}
+                    ${check_target}
+                    COMMAND "${CMAKE_COMMAND}" --build <BINARY_DIR> --target ${check_target}
+                    USES_TERMINAL TRUE
+                    EXCLUDE_FROM_MAIN TRUE
+                    ALWAYS TRUE
+                )
+                ExternalProject_Add_StepTargets(runtimes-${variant} ${check_target})
+                ExternalProject_Add_StepDependencies(
+                    runtimes-${variant}
+                    ${check_target}
+                    runtimes-${variant}-build
+                )
+                add_custom_target(${check_target}-${variant})
+                add_dependencies(${check_target} runtimes-${variant}-${check_target})
+                add_dependencies(${check_target}-${variant} runtimes-${variant}-${check_target})
+            endforeach()
+
+            # Add the variant to the multilib yaml
+            string(APPEND multilib_yaml_content "- Dir: ${parent_dir_name}/${variant}\n")
+            string(APPEND multilib_yaml_content "  Flags:\n")
+            string(REPLACE " " ";" multilib_flags_list ${variant_multilib_flags})
+            foreach(flag ${multilib_flags_list})
+                string(APPEND multilib_yaml_content "  - ${flag}\n")
+            endforeach()
+            string(APPEND multilib_yaml_content "  Group: stdlibs\n")
+        else()
+            # In place of a json, an error message is expected.
+            string(JSON variant_error_msg GET ${lib_def} "error")
+
+            string(APPEND multilib_yaml_content "- Error: \"${variant_error_msg}\"\n")
+            string(APPEND multilib_yaml_content "  Flags:\n")
+            string(REPLACE " " ";" multilib_flags_list ${variant_multilib_flags})
+            foreach(flag ${multilib_flags_list})
+                string(APPEND multilib_yaml_content "  - ${flag}\n")
+            endforeach()
+            string(APPEND multilib_yaml_content "  Group: stdlibs\n")
+        endif()
+    endif()
+
+endforeach()
+
+# Multilib file is generated in two parts.
+# 1. Template is filled with multilib flags from json
+configure_file(
+    ${CMAKE_CURRENT_SOURCE_DIR}/multilib.yaml.in
+    ${CMAKE_CURRENT_BINARY_DIR}/multilib-without-fpus.yaml
+    @ONLY
+)
+
+# 2. multilib-generate.py maps compiler command line options to flags
+add_custom_command(
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/multilib-fpus.yaml
+    COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/multilib-generate.py"
+        "--clang=${LLVM_BINARY_DIR}/bin/clang${CMAKE_EXECUTABLE_SUFFIX}"
+        "--llvm-source=${FETCHCONTENT_SOURCE_DIR_LLVMPROJECT}"
+    >> ${CMAKE_CURRENT_BINARY_DIR}/multilib-fpus.yaml
+)
+
+# Combine the two parts.
+add_custom_command(
+    OUTPUT 
+        ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
+    COMMAND
+        ${CMAKE_COMMAND} -E cat
+        ${CMAKE_CURRENT_BINARY_DIR}/multilib-without-fpus.yaml
+        ${CMAKE_CURRENT_BINARY_DIR}/multilib-fpus.yaml
+        > ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
+    DEPENDS
+        ${CMAKE_CURRENT_BINARY_DIR}/multilib-without-fpus.yaml
+        ${CMAKE_CURRENT_BINARY_DIR}/multilib-fpus.yaml
+)
+
+add_custom_target(multilib-yaml ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml)
+install(
+    FILES ${CMAKE_CURRENT_BINARY_DIR}/multilib/multilib.yaml
+    DESTINATION .
+)