Improve build efficiency by building library subprojects in parallel (#630)

dcandler · web-flow · commit 49bfb9d02dfa · 2025-01-23T14:09:56.000Z
Each library variant build consists of three subprojects, which require
sequential configuring and building. While the build steps can be
parallelized by using multiple processes to compile the sources, the
configuration step is largely single threaded.

The multilib project has to build many variants now, and so a
significant amount of time is spent waiting for the configuration of
various projects to finish.

This patch attempts to make the build process more efficient by running
all the configuration steps for every variant subproject in parallel.
The configuration/build steps in the underlying runtimes project are
exposed, so that the higher level multilib project can invoke them as a
single target. So instead of building each variant in turn, all the
versions of compiler-rt are configured, then all versions are built,
then all the C libraries are configured, and so on. This maximizes the
amount of work that can be done at any given time.

The build steps already benefit from parallelization: although built in
series, each has the full number of available processes to use. However,
at higher CPU counts there is an observable limit on how many processes
each build can effectively use. So it may also be more efficient to
instead run the build steps in parallel with a smaller number of
proceses available to each. The option to control it
(ENABLE_PARALLEL_LIB_BUILD) is OFF by default as which strategy is
faster may come down to your available hardware, whereas
ENABLE_PARALLEL_LIB_CONFIG should always be beneficial, so I've
defaulted that to ON.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -150,6 +150,20 @@ option(
     "During checkout, apply optional downstream patches to
     llvm-project to improve performance."
 )
+option(
+    ENABLE_PARALLEL_LIB_CONFIG
+    "Run the library variant configuration steps in parallel."
+    ON
+)
+option(
+    ENABLE_PARALLEL_LIB_BUILD
+    "Run the library variant build steps in parallel."
+    OFF
+)
+set(PARALLEL_LIB_BUILD_LEVELS
+    "1" CACHE STRING
+    "If ENABLE_PARALLEL_LIB_BUILD is ON, this number of processes will be assigned to each variant built."
+)
 option(
     ENABLE_QEMU_TESTING
     "Enable tests that use QEMU. This option is ON by default."
@@ -599,6 +613,9 @@ if(NOT PREBUILT_TARGET_LIBRARIES)
         -DLLVM_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/llvm
         -DMULTILIB_JSON=${LLVM_TOOLCHAIN_MULTILIB_JSON}
         -DENABLE_VARIANTS=${ENABLE_VARIANTS_PASSTHROUGH}
+        -DENABLE_PARALLEL_LIB_CONFIG=${ENABLE_PARALLEL_LIB_CONFIG}
+        -DENABLE_PARALLEL_LIB_BUILD=${ENABLE_PARALLEL_LIB_BUILD}
+        -DPARALLEL_LIB_BUILD_LEVELS=${PARALLEL_LIB_BUILD_LEVELS}
         -DLIBC_HDRGEN=${LIBC_HDRGEN}
         -DFVP_INSTALL_DIR=${FVP_INSTALL_DIR}
         -DENABLE_QEMU_TESTING=${ENABLE_QEMU_TESTING}
@@ -608,9 +625,8 @@ if(NOT PREBUILT_TARGET_LIBRARIES)
         -DFETCHCONTENT_SOURCE_DIR_PICOLIBC=${FETCHCONTENT_SOURCE_DIR_PICOLIBC}
         -DFETCHCONTENT_SOURCE_DIR_NEWLIB=${FETCHCONTENT_SOURCE_DIR_NEWLIB}
         -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
-        USES_TERMINAL_CONFIGURE FALSE
+        USES_TERMINAL_CONFIGURE TRUE
         USES_TERMINAL_BUILD TRUE
-        USES_TERMINAL_TEST TRUE
         LIST_SEPARATOR ,
         CONFIGURE_HANDLED_BY_BUILD TRUE
         TEST_EXCLUDE_FROM_MAIN TRUE
diff --git a/arm-multilib/CMakeLists.txt b/arm-multilib/CMakeLists.txt
@@ -50,6 +50,25 @@ set(
     fvp/get_fvps.sh"
 )
 set(FVP_CONFIG_DIR "${TOOLCHAIN_SOURCE_DIR}/fvp/config" CACHE STRING "The directory in which the FVP models are installed.")
+option(
+    ENABLE_PARALLEL_LIB_CONFIG
+    "Run the library variant configuration steps in parallel."
+    ON
+)
+option(
+    ENABLE_PARALLEL_LIB_BUILD
+    "Run the library variant build steps in parallel."
+    OFF
+)
+set(PARALLEL_LIB_BUILD_LEVELS
+    "1" CACHE STRING
+    "If ENABLE_PARALLEL_LIB_BUILD is ON, this number of processes will be assigned to each variant built."
+)
+if(NOT CMAKE_GENERATOR MATCHES "Ninja")
+    if (ENABLE_PARALLEL_LIB_CONFIG OR ENABLE_PARALLEL_LIB_BUILD)
+        message(WARNING "Library build parallelization should only be enabled with the Ninja generator.")
+    endif()
+endif()
 
 # If a compiler launcher such as ccache has been set, it should be
 # passed down to each subproject build.
@@ -131,6 +150,34 @@ add_dependencies(
     check-unwind
 )
 
+if(ENABLE_PARALLEL_LIB_CONFIG OR ENABLE_PARALLEL_LIB_BUILD)
+    # Additional targets to build the variant subprojects in parallel.
+    # The build steps can use multible jobs to compile in parallel, but
+    # the configuration steps are largely single threaded. This creates a
+    # bottleneck if each variant is built in series.
+    # It is significantly faster to run all the subproject configuration
+    # steps in parallel, run the build steps, then run the next set of
+    # configuration steps in parallel, etc.
+    set(
+        subtargets
+        compiler_rt-configure
+        compiler_rt-build
+        clib-configure
+        clib-build
+        cxxlibs-configure
+        cxxlibs-build
+    )
+    set(subtarget_deps none ${subtargets})
+    list(REMOVE_AT subtarget_deps 6)
+
+    foreach(subtarget subtarget_dep IN ZIP_LISTS subtargets subtarget_deps)
+        add_custom_target(${subtarget}-all)
+        if(NOT subtarget_dep STREQUAL "none")
+            add_dependencies(${subtarget}-all ${subtarget_dep}-all)
+        endif()
+    endforeach()
+endif()
+
 # Read the JSON file to load a multilib configuration.
 file(READ ${MULTILIB_JSON} multilib_json_str)
 string(JSON multilib_defs GET ${multilib_json_str} "libs")
@@ -218,11 +265,58 @@ foreach(lib_idx RANGE ${lib_count_dec})
                 STEP_TARGETS build install
                 USES_TERMINAL_CONFIGURE FALSE
                 USES_TERMINAL_BUILD TRUE
-                USES_TERMINAL_TEST TRUE
                 LIST_SEPARATOR ,
                 CONFIGURE_HANDLED_BY_BUILD TRUE
                 TEST_EXCLUDE_FROM_MAIN TRUE
             )
+
+            if(ENABLE_PARALLEL_LIB_CONFIG OR ENABLE_PARALLEL_LIB_BUILD)
+                # Create additional steps to configure/build the subprojects.
+                # These are collected to be run together, so that all the
+                # configuration steps can be run in parallel.
+                # Each step should depend on the previous, with the first depending on the pre-defined
+                # 'configure' step, and the pre-defined 'build' step depending on the last.
+                set(subtarget_deps configure ${subtargets} build)
+                list(SUBLIST subtarget_deps 0 6 subtarget_dependees)
+                list(SUBLIST subtarget_deps 2 6 subtarget_dependers)
+
+                # First loop to add the steps and targets.
+                foreach(subtarget subtarget_dependee IN ZIP_LISTS subtargets subtarget_dependees)
+                    # Enabling USES_TERMINAL puts the step in Ninja's "console" job pool, which
+                    # prevents the steps from being run in parallel since each must be given
+                    # exclusive access to the terminal. When disabled, the console won't be updated
+                    # with any output from the step until it completes.
+                    set(step_uses_terminal ON)
+                    set(step_extra_env "")
+                    if(${subtarget} MATCHES "-configure$" AND ENABLE_PARALLEL_LIB_CONFIG)
+                        set(step_uses_terminal OFF)
+                    elseif(${subtarget} MATCHES "-build$" AND ENABLE_PARALLEL_LIB_BUILD)
+                        set(step_uses_terminal OFF)
+                        set(step_extra_env ${CMAKE_COMMAND} -E env CMAKE_BUILD_PARALLEL_LEVEL=${PARALLEL_LIB_BUILD_LEVELS})
+                    endif()
+                    ExternalProject_Add_Step(
+                        runtimes-${variant}
+                        ${subtarget}
+                        COMMAND ${step_extra_env} ${CMAKE_COMMAND} --build <BINARY_DIR> --target ${subtarget}
+                        DEPENDEES ${subtarget_dependee}
+                        DEPENDERS build
+                        USES_TERMINAL ${step_uses_terminal}
+                    )
+                    ExternalProject_Add_StepTargets(runtimes-${variant} ${subtarget})
+                    add_dependencies(${subtarget}-all runtimes-${variant}-${subtarget})
+                endforeach()
+
+                # Second loop to set the steps that will depend on the new targets.
+                foreach(subtarget subtarget_depender IN ZIP_LISTS subtargets subtarget_dependers)
+                    ExternalProject_Add_StepDependencies(
+                        runtimes-${variant}
+                        ${subtarget_depender}
+                        ${subtarget}-all
+                    )
+                endforeach()
+            endif()
+
+            # Add custom check targets.
             set(check_targets "")
             if(read_ENABLE_LIBC_TESTS)
                 list(APPEND check_targets check-${C_LIBRARY})
diff --git a/arm-runtimes/CMakeLists.txt b/arm-runtimes/CMakeLists.txt
@@ -210,7 +210,10 @@ set(compile_arch_flags "--target=${target_triple} ${COMPILE_FLAGS}")
 # flags, and requires a sysroot.
 set(lib_compile_flags "${compile_arch_flags} -ffunction-sections -fdata-sections -fno-ident --sysroot ${TEMP_LIB_DIR}")
 
-# Declare this target now, since compiler-rt requires the dependency.
+# Generic target names for the C library.
+# Declare these now, since compiler-rt requires the 'install' dependency.
+add_custom_target(clib-configure)
+add_custom_target(clib-build)
 add_custom_target(clib-install)
 
 ###############################################################################
@@ -288,8 +291,8 @@ ExternalProject_Add(
     -DLLVM_CMAKE_DIR=${LLVM_BINARY_DIR}
     -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON
     ${compiler_rt_test_cmake_args}
-    STEP_TARGETS build install
-    USES_TERMINAL_CONFIGURE FALSE
+    STEP_TARGETS configure build install
+    USES_TERMINAL_CONFIGURE TRUE
     USES_TERMINAL_BUILD TRUE
     USES_TERMINAL_INSTALL TRUE
     LIST_SEPARATOR ,
@@ -399,12 +402,13 @@ if(C_LIBRARY STREQUAL picolibc)
             <SOURCE_DIR>
         BUILD_COMMAND ${MESON_EXECUTABLE} compile
         INSTALL_COMMAND ${MESON_EXECUTABLE} install ${MESON_INSTALL_QUIET}
-        USES_TERMINAL_CONFIGURE FALSE
+        USES_TERMINAL_CONFIGURE TRUE
         USES_TERMINAL_BUILD TRUE
+        USES_TERMINAL_INSTALL TRUE
         LIST_SEPARATOR ,
         CONFIGURE_HANDLED_BY_BUILD TRUE
         TEST_EXCLUDE_FROM_MAIN TRUE
-        STEP_TARGETS build install
+        STEP_TARGETS configure build install
     )
 
     add_custom_target(check-picolibc)
@@ -523,12 +527,13 @@ if(C_LIBRARY STREQUAL newlib)
             <BINARY_DIR>/${target_triple}/libgloss/${cpu_family}/libcrt0-nosys.a
             ${TEMP_LIB_DIR}/lib
         # FIXME: TEST_COMMAND?
-        USES_TERMINAL_CONFIGURE FALSE
+        USES_TERMINAL_CONFIGURE TRUE
         USES_TERMINAL_BUILD TRUE
+        USES_TERMINAL_INSTALL TRUE
         # Always run the build command so that incremental builds are correct.
         CONFIGURE_HANDLED_BY_BUILD TRUE
         TEST_EXCLUDE_FROM_MAIN TRUE
-        STEP_TARGETS install # FIXME: test?
+        STEP_TARGETS configure build install # FIXME: test?
     )
 endif()
 
@@ -620,11 +625,10 @@ if(C_LIBRARY STREQUAL llvmlibc)
         -DLLVM_ENABLE_RUNTIMES=libc
         -DLLVM_INCLUDE_TESTS=OFF # llvmlibc's tests require C++, so can't be built until llvmlibc can support libc++
         -DLLVM_LIBC_FULL_BUILD=ON
-        STEP_TARGETS build install
-        USES_TERMINAL_CONFIGURE FALSE
+        STEP_TARGETS configure build install
+        USES_TERMINAL_CONFIGURE TRUE
         USES_TERMINAL_BUILD TRUE
         USES_TERMINAL_INSTALL TRUE
-        USES_TERMINAL_TEST TRUE
         LIST_SEPARATOR ,
         CONFIGURE_HANDLED_BY_BUILD TRUE
         INSTALL_COMMAND ${CMAKE_COMMAND} --install .
@@ -652,7 +656,7 @@ if(C_LIBRARY STREQUAL llvmlibc)
         ${compiler_launcher_cmake_args}
         ${common_llvmlibc_cmake_args}
         STEP_TARGETS build install
-        USES_TERMINAL_CONFIGURE FALSE
+        USES_TERMINAL_CONFIGURE TRUE
         USES_TERMINAL_BUILD TRUE
         USES_TERMINAL_INSTALL TRUE
         USES_TERMINAL_TEST TRUE
@@ -661,6 +665,8 @@ if(C_LIBRARY STREQUAL llvmlibc)
     )
 endif()
 
+add_dependencies(clib-configure ${C_LIBRARY}-configure)
+add_dependencies(clib-build ${C_LIBRARY}-build)
 add_dependencies(clib-install ${C_LIBRARY}-install)
 
 ###############################################################################
@@ -758,11 +764,10 @@ if(ENABLE_CXX_LIBS)
         -DRUNTIME_VARIANT_NAME=${VARIANT}
         ${cxxlibs_extra_cmake_options}
         ${cxxlibs_test_cmake_options}
-        STEP_TARGETS build install
-        USES_TERMINAL_CONFIGURE FALSE
+        STEP_TARGETS configure build install
+        USES_TERMINAL_CONFIGURE TRUE
         USES_TERMINAL_BUILD TRUE
         USES_TERMINAL_INSTALL TRUE
-        USES_TERMINAL_TEST TRUE
         LIST_SEPARATOR ,
         CONFIGURE_HANDLED_BY_BUILD TRUE
     )
