fixup! Move CMake code for building multilib and variants to subprojects

dcandler · dcandler · commit 8314dc0fd367 · 2024-11-14T18:40:50.000Z
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -138,6 +138,13 @@ option(
     the tools every time you update llvm-project."
     ON
 )
+option(
+    LIBS_USE_COMPILER_LAUNCHER
+    "Pass CMAKE_C_COMPILER_LAUNCHER and CMAKE_CXX_COMPILER_LAUNCHER
+    down to the library builds, so that programs such as ccache can
+    be used to speed up repeated builds. This is not done by default,
+    as it can also make the inital build slower due to the cold cache."
+)
 option(
     APPLY_LLVM_PERFORMANCE_PATCHES
     "During checkout, apply optional downstream patches to
@@ -261,38 +268,11 @@ include(ProcessorCount)
 #
 # If you want to stop cmake updating the repos then run
 # cmake . -DFETCHCONTENT_FULLY_DISCONNECTED=ON
-
-# Read which revisions of the repos to use.
-file(READ versions.json VERSIONS_JSON)
-function(read_repo_version output_variable_prefix repo)
-    string(JSON tag GET ${VERSIONS_JSON} "repos" "${repo}" "tag")
-    string(JSON tagType GET ${VERSIONS_JSON} "repos" "${repo}" "tagType")
-    if(tagType STREQUAL "commithash")
-        # GIT_SHALLOW doesn't work with commit hashes.
-        set(shallow OFF)
-    elseif(tagType STREQUAL "branch")
-        set(shallow ON)
-        # CMake docs recommend that "branch names and tags should
-        # generally be specified as remote names"
-        set(tag "origin/${tag}")
-    elseif(tagType STREQUAL "tag")
-        set(shallow ON)
-    else()
-        message(FATAL_ERROR "Unrecognised tagType ${tagType}")
-    endif()
-
-    set(${output_variable_prefix}_TAG "${tag}" PARENT_SCOPE)
-    set(${output_variable_prefix}_SHALLOW "${shallow}" PARENT_SCOPE)
-endfunction()
-read_repo_version(llvmproject llvm-project)
-
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/fetch_llvm.cmake)
 if(LLVM_TOOLCHAIN_C_LIBRARY STREQUAL picolibc)
-    read_repo_version(picolibc picolibc)
     include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/fetch_picolibc.cmake)
 endif()
 if(LLVM_TOOLCHAIN_C_LIBRARY STREQUAL newlib)
-    read_repo_version(newlib newlib)
     include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/fetch_newlib.cmake)
 endif()
 
@@ -366,11 +346,6 @@ if(LLVM_TOOLCHAIN_C_LIBRARY STREQUAL llvmlibc)
     ExternalProject_Get_property(libc_hdrgen BINARY_DIR)
     set(LIBC_HDRGEN ${BINARY_DIR}/bin/libc-hdrgen${CMAKE_EXECUTABLE_SUFFIX})
 
-    # Add an empty check target, to simplify the logic below that expects to
-    # find one for every libc type. We have no current setup to run the LLVM
-    # libc test suite.
-    add_custom_target(check-llvmlibc)
-
     # LLVM libc lacks a configuration for AArch64, but the AArch32 one works
     # fine. However, setting the configuration for both architectures to the
     # arm config directory means the baremetal config.json is never loaded,
@@ -581,17 +556,28 @@ else()
     set(library_subdir "")
 endif()
 
+if (LIBS_USE_COMPILER_LAUNCHER)
+    if(CMAKE_C_COMPILER_LAUNCHER)
+        list(APPEND compiler_launcher_cmake_args "-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}")
+    endif()
+    if(CMAKE_CXX_COMPILER_LAUNCHER)
+        list(APPEND compiler_launcher_cmake_args "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}")
+    endif()
+endif()
+
 ExternalProject_Add(
     multilib-${LLVM_TOOLCHAIN_C_LIBRARY}
     PREFIX ${CMAKE_BINARY_DIR}/multilib-builds
     SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/arm-multilib
     INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/llvm/${TARGET_LIBRARIES_DIR}${library_subdir}
     DEPENDS ${lib_tool_dependencies}
     CMAKE_ARGS
+    ${compiler_launcher_cmake_args}
     -DC_LIBRARY=${LLVM_TOOLCHAIN_C_LIBRARY}
     -DLLVM_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/llvm
     -DMULTILIB_JSON=${LLVM_TOOLCHAIN_MULTILIB_JSON}
     -DENABLE_VARIANTS=${LLVM_TOOLCHAIN_LIBRARY_VARIANTS}
+    -DLIBC_HDRGEN=${LIBC_HDRGEN}
     -DFVP_INSTALL_DIR=${FVP_INSTALL_DIR}
     -DFVP_CONFIG_DIR=${CMAKE_CURRENT_SOURCE_DIR}/fvp/config
     -DFETCHCONTENT_SOURCE_DIR_LLVMPROJECT=${FETCHCONTENT_SOURCE_DIR_LLVMPROJECT}
diff --git a/arm-multilib/CMakeLists.txt b/arm-multilib/CMakeLists.txt
@@ -32,6 +32,15 @@ set(ENABLE_VARIANTS "all" CACHE STRING "Semicolon separated list of variants to
 set(C_LIBRARY "picolibc" CACHE STRING "Which C library to use.")
 set_property(CACHE C_LIBRARY PROPERTY STRINGS picolibc newlib llvmlibc)
 set(LLVM_BINARY_DIR "" CACHE PATH "Path to LLVM toolchain build or install root.")
+set(LIBC_HDRGEN "" CACHE PATH "Path to prebuilt lbc-hdrgen if not included in LLVM binaries set by LLVM_BINARY_DIR")
+set(
+    FVP_INSTALL_DIR
+    "" CACHE STRING
+    "The directory in which the FVP models are installed. These are not
+    included in this repository, but can be downloaded by the script
+    fvp/get_fvps.sh"
+)
+set(FVP_CONFIG_DIR "${TOOLCHAIN_SOURCE_DIR}/fvp/config" CACHE STRING "The directory in which the FVP models are installed.")
 
 # If a compiler launcher such as ccache has been set, it should be
 # passed down to each subproject build.
@@ -46,6 +55,7 @@ endif()
 # Arguments to pass down to the library projects.
 foreach(arg
     LLVM_BINARY_DIR
+    LIBC_HDRGEN
     FVP_INSTALL_DIR
     FVP_CONFIG_DIR
 )
@@ -65,6 +75,34 @@ elseif(C_LIBRARY STREQUAL newlib)
     list(APPEND passthrough_dirs "-DFETCHCONTENT_SOURCE_DIR_NEWLIB=${FETCHCONTENT_SOURCE_DIR_NEWLIB}")
 endif()
 
+# Target for any dependencies to build the runtimes project.
+add_custom_target(runtimes-depends)
+
+# If building llvm-libc, ensure libc-hdrgen is available.
+if(C_LIBRARY STREQUAL llvmlibc)
+    if(NOT LIBC_HDRGEN)
+        if(EXISTS ${LLVM_BINARY_DIR}/bin/libc-hdrgen${CMAKE_EXECUTABLE_SUFFIX})
+            set(LIBC_HDRGEN ${LLVM_BINARY_DIR}/bin/libc-hdrgen${CMAKE_EXECUTABLE_SUFFIX})
+        else()
+            ExternalProject_Add(
+                libc_hdrgen
+                SOURCE_DIR ${llvmproject_SOURCE_DIR}/llvm
+                CMAKE_ARGS
+                    -DLLVM_ENABLE_RUNTIMES=libc
+                    -DLLVM_LIBC_FULL_BUILD=ON
+                    -DCMAKE_BUILD_TYPE=Debug
+                BUILD_COMMAND ${CMAKE_COMMAND} --build . --target libc-hdrgen
+                INSTALL_COMMAND ${CMAKE_COMMAND} -E true
+                CONFIGURE_HANDLED_BY_BUILD TRUE
+            )
+            ExternalProject_Get_property(libc_hdrgen BINARY_DIR)
+            set(LIBC_HDRGEN ${BINARY_DIR}/bin/libc-hdrgen${CMAKE_EXECUTABLE_SUFFIX})
+            add_dependencies(runtimes-depends libc_hdrgen)
+        endif()
+    endif()
+    list(APPEND passthrough_dirs "-DLIBC_HDRGEN=${LIBC_HDRGEN}")
+endif()
+
 # Create one target to run all the tests.
 add_custom_target(check-${C_LIBRARY})
 add_custom_target(check-compiler-rt)
@@ -111,13 +149,14 @@ foreach(lib_idx RANGE ${lib_count_dec})
                 DIRECTORY ${destination_directory}
                 DESTINATION ${parent_dir_name}
             )
-            set(variant_json_file ${CMAKE_CURRENT_BINARY_DIR}/json/variants/${variant_json})
+            set(variant_json_file ${CMAKE_CURRENT_SOURCE_DIR}/json/variants/${variant_json})
 
             ExternalProject_Add(
                 runtimes-${variant}
                 PREFIX ${CMAKE_BINARY_DIR}/lib-builds
                 SOURCE_DIR ${TOOLCHAIN_SOURCE_DIR}/arm-runtimes
                 INSTALL_DIR ${destination_directory}
+                DEPENDS runtimes-depends
                 CMAKE_ARGS
                 ${compiler_launcher_cmake_args}
                 ${passthrough_dirs}
diff --git a/arm-multilib/multilib-generate.py b/arm-multilib/multilib-generate.py
@@ -239,6 +239,9 @@ def generate_extensions(args):
     aarch64_features = get_extension_list(args.clang, "aarch64-none-eabi")
     aarch32_features = get_extension_list(args.clang, "arm-none-eabi")
     all_features = list(aarch64_features)
+    # Combine the aarch64 and aarch32 lists without duplication.
+    # Casting to sets and merging would be simpler, but creates
+    # non-deterministic output.
     all_features.extend(feat for feat in list(aarch32_features) if feat not in all_features)
 
     print("# Expand -march=...+[no]feature... into individual options we can match")
diff --git a/arm-runtimes/CMakeLists.txt b/arm-runtimes/CMakeLists.txt
@@ -96,6 +96,7 @@ set(ENABLE_LIBC_TESTS ${ENABLE_LIBC_TESTS_def} CACHE BOOL "Enable libc tests (pi
 set(ENABLE_COMPILER_RT_TESTS ${ENABLE_COMPILER_RT_TESTS_def} CACHE BOOL "Enable compiler-rt tests.")
 set(ENABLE_LIBCXX_TESTS ${ENABLE_LIBCXX_TESTS_def} CACHE BOOL "Enable libcxx tests.")
 set(LLVM_BINARY_DIR "" CACHE PATH "Path to LLVM toolchain root to build libraries with")
+set(LIBC_HDRGEN "" CACHE PATH "Path to prebuilt lbc-hdrgen if not included in LLVM binaries set by LLVM_BINARY_DIR")
 
 # Temporary location to collect the libraries as they are built.
 set(TEMP_LIB_DIR "${CMAKE_CURRENT_BINARY_DIR}/tmp_install")
@@ -137,16 +138,23 @@ else()
     else()
         set(target_triple "arm-none-eabi")
     endif()
-    if(flags MATCHES "-mfloat-abi=hard")
+    if(COMPILE_FLAGS MATCHES "-mfloat-abi=hard")
         # Also, compiler-rt looks in the ABI component of the
         # triple to decide whether to use the hard float ABI.
         set(target_triple "${target_triple}hf")
     endif()
     set(cpu_family arm)
 endif()
 
+# Create a single target for all testing. If no testing is enabled, this
+# will simply do nothing.
+add_custom_target(check-all)
+
 # If any testing is enabled, prepare test executor settings.
 if (ENABLE_LIBC_TESTS OR ENABLE_COMPILER_RT_TESTS OR ENABLE_LIBCXX_TESTS)
+    # Flags required to link tests.
+    set(test_link_flags "-nostartfiles -lcrt0-semihost -lsemihost -T picolibcpp.ld")
+
     if(TEST_EXECUTOR STREQUAL qemu)
         if(TARGET_ARCH MATCHES "^aarch64")
             find_program(QEMU_EXECUTABLE qemu-system-aarch64)
@@ -198,7 +206,6 @@ if (ENABLE_LIBC_TESTS OR ENABLE_COMPILER_RT_TESTS OR ENABLE_LIBCXX_TESTS)
 endif()
 
 set(compile_arch_flags "--target=${target_triple} ${COMPILE_FLAGS}")
-set(link_flags "-nostartfiles -lcrt0-semihost -lsemihost -T picolibcpp.ld")
 # Compiling the libraries benefits from some extra optimization
 # flags, and requires a sysroot.
 set(lib_compile_flags "${compile_arch_flags} -ffunction-sections -fdata-sections -fno-ident --sysroot ${TEMP_LIB_DIR}")
@@ -238,7 +245,7 @@ if(VARIANT STREQUAL "armv6m_soft_nofp")
 endif()
 
 if (ENABLE_COMPILER_RT_TESTS)
-    set(compiler_rt_test_flags "${lib_compile_flags} ${link_flags}")
+    set(compiler_rt_test_flags "${lib_compile_flags} ${test_link_flags}")
     set(compiler_rt_lit_args "${LLVM_LIT_ARGS} --xunit-xml-output=results.junit.xml")
     set(
         compiler_rt_test_cmake_args 
@@ -298,6 +305,7 @@ ExternalProject_Add(
 )
 
 add_custom_target(check-compiler-rt)
+add_dependencies(check-all check-compiler-rt)
 if (ENABLE_COMPILER_RT_TESTS)
     ExternalProject_Add_Step(
         compiler_rt
@@ -357,7 +365,7 @@ if(C_LIBRARY STREQUAL picolibc)
     to_meson_list("${picolibc_flags}" picolibc_meson_flags)
     
     if (ENABLE_LIBC_TESTS)
-        set(picolibc_test_executor_bin ${CMAKE_CURRENT_SOURCE_DIR}/picolibc-test-wrapper.py)
+        set(picolibc_test_executor_bin ${CMAKE_CURRENT_SOURCE_DIR}/test-support/picolibc-test-wrapper.py)
         to_meson_list("${test_executor_params}" meson_test_executor_params)
     endif()
 
@@ -394,6 +402,7 @@ if(C_LIBRARY STREQUAL picolibc)
     )
 
     add_custom_target(check-picolibc)
+    add_dependencies(check-all check-picolibc)
     if (ENABLE_LIBC_TESTS)
         # meson builds the tests at the same time as the library.
         # So reconfigure to enable tests at a later point.
@@ -416,7 +425,7 @@ if(C_LIBRARY STREQUAL picolibc)
             check
             COMMAND ${MESON_EXECUTABLE} test -C <BINARY_DIR>
             COMMAND ${Python3_EXECUTABLE}
-            ${TOOLCHAIN_SOURCE_DIR}/test-support/modify-picolibc-xml.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/test-support/modify-picolibc-xml.py
             --picolibc-build-dir <BINARY_DIR>
             --variant ${VARIANT}
             USES_TERMINAL TRUE
@@ -542,6 +551,8 @@ if(C_LIBRARY STREQUAL llvmlibc)
         DESTINATION
         ${LIBC_CFG_DIR}
     )
+    
+    set(lib_compile_flags "${lib_compile_flags} -Wno-error=atomic-alignment")
 
     set(common_llvmlibc_cmake_args
         -DCMAKE_AR=${LLVM_BINARY_DIR}/bin/llvm-ar${CMAKE_EXECUTABLE_SUFFIX}
@@ -568,6 +579,9 @@ if(C_LIBRARY STREQUAL llvmlibc)
         # If libc-hdrgen is provided, there is no need to build it,
         # but a target is still needed to satisfy the dependency.
         add_custom_target(libc_hdrgen)
+    elseif(EXISTS ${LLVM_BINARY_DIR}/bin/libc-hdrgen${CMAKE_EXECUTABLE_SUFFIX})
+        set(LIBC_HDRGEN ${LLVM_BINARY_DIR}/bin/libc-hdrgen${CMAKE_EXECUTABLE_SUFFIX})
+        add_custom_target(libc_hdrgen)
     else()
         ExternalProject_Add(
             libc_hdrgen
@@ -576,10 +590,8 @@ if(C_LIBRARY STREQUAL llvmlibc)
                 -DLLVM_ENABLE_RUNTIMES=libc
                 -DLLVM_LIBC_FULL_BUILD=ON
                 -DCMAKE_BUILD_TYPE=Debug
-            STEP_TARGETS build install
             BUILD_COMMAND ${CMAKE_COMMAND} --build . --target libc-hdrgen
             INSTALL_COMMAND ${CMAKE_COMMAND} -E true
-            # Always run the build command so that incremental builds are correct.
             CONFIGURE_HANDLED_BY_BUILD TRUE
         )
         ExternalProject_Get_property(libc_hdrgen BINARY_DIR)
@@ -674,7 +686,7 @@ if(ENABLE_CXX_LIBS)
                 -DLIBUNWIND_TEST_CONFIG=${CMAKE_CURRENT_SOURCE_DIR}/test-support/llvm-libunwind-picolibc.cfg.in
                 -DLIBUNWIND_TEST_PARAMS=executor=${lit_test_executor}
                 -DRUNTIME_TEST_ARCH_FLAGS=${compile_arch_flags}
-                -DRUNTIME_TEST_LINK_FLAGS=${link_flags}
+                -DRUNTIME_TEST_LINK_FLAGS=${test_link_flags}
                 -DLLVM_LIT_ARGS=${cxxlibs_lit_args}
             )
         endif()
@@ -750,8 +762,11 @@ if(ENABLE_CXX_LIBS)
         CONFIGURE_HANDLED_BY_BUILD TRUE
     )
     add_custom_target(check-cxx)
+    add_dependencies(check-all check-cxx)
     add_custom_target(check-cxxabi)
+    add_dependencies(check-all check-cxxabi)
     add_custom_target(check-unwind)
+    add_dependencies(check-all check-unwind)
     if (ENABLE_LIBCXX_TESTS)
         foreach(check_target check-cxx check-cxxabi check-unwind)
             ExternalProject_Add_Step(
@@ -772,9 +787,9 @@ if(ENABLE_CXX_LIBS)
         endforeach()
     endif()
 
-    install(
-        DIRECTORY ${TEMP_LIB_DIR}/
-        DESTINATION .
-    )
-
 endif()
+
+install(
+    DIRECTORY ${TEMP_LIB_DIR}/
+    DESTINATION .
+)
diff --git a/arm-runtimes/test-support/modify-compiler-rt-xml.py b/arm-runtimes/test-support/modify-compiler-rt-xml.py
@@ -47,7 +47,6 @@ def main():
             new_classname = old_classname.replace(old_suitename, new_suitename)
             testcase.set("classname", new_classname)
 
-    xml_file = os.path.join(args.compiler_rt_build_dir, "test", "results2.junit.xml")
     tree.write(xml_file)
     print(f"Results written to {xml_file}")
 
diff --git a/cmake/fetch_llvm.cmake b/cmake/fetch_llvm.cmake
@@ -6,8 +6,13 @@
 
 include(FetchContent)
 
-set(llvm_patch_script ${CMAKE_CURRENT_SOURCE_DIR}/patch_llvm.py)
-set(patch_dir ${CMAKE_CURRENT_SOURCE_DIR}/../patches)
+if(NOT VERSIONS_JSON)
+    include(${CMAKE_CURRENT_LIST_DIR}/read_versions.cmake)
+endif()
+read_repo_version(llvmproject llvm-project)
+
+set(llvm_patch_script ${CMAKE_CURRENT_LIST_DIR}/patch_llvm.py)
+set(patch_dir ${CMAKE_CURRENT_LIST_DIR}/../patches)
 set(LLVM_PATCH_COMMAND ${Python3_EXECUTABLE} ${llvm_patch_script} ${patch_dir}/llvm-project)
 if(APPLY_LLVM_PERFORMANCE_PATCHES)
     set(LLVM_PATCH_COMMAND ${LLVM_PATCH_COMMAND} && ${Python3_EXECUTABLE} ${llvm_patch_script} ${patch_dir}/llvm-project-perf)
diff --git a/cmake/fetch_newlib.cmake b/cmake/fetch_newlib.cmake
@@ -6,7 +6,12 @@
 
 include(FetchContent)
 
-set(newlib_patch ${CMAKE_CURRENT_SOURCE_DIR}/../patches/newlib.patch)
+if(NOT VERSIONS_JSON)
+    include(${CMAKE_CURRENT_LIST_DIR}/read_versions.cmake)
+endif()
+read_repo_version(newlib newlib)
+
+set(newlib_patch ${CMAKE_CURRENT_LIST_DIR}/../patches/newlib.patch)
 
 FetchContent_Declare(newlib
     GIT_REPOSITORY https://sourceware.org/git/newlib-cygwin.git
diff --git a/cmake/fetch_picolibc.cmake b/cmake/fetch_picolibc.cmake
@@ -6,6 +6,11 @@
 
 include(FetchContent)
 
+if(NOT VERSIONS_JSON)
+    include(${CMAKE_CURRENT_LIST_DIR}/read_versions.cmake)
+endif()
+read_repo_version(picolibc picolibc)
+
 set(
     picolibc_patches
     ${CMAKE_CURRENT_SOURCE_DIR}/patches/picolibc/0001-Enable-libcxx-builds.patch
diff --git a/cmake/read_versions.cmake b/cmake/read_versions.cmake
diff --git a/docs/building-from-source.md b/docs/building-from-source.md
