Merge branch 'main' into dev/runOnNewStack

Author: Bigcheese

.ci/compute_projects.py

@@ -52,6 +52,9 @@
     "clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
     "clang-tools-extra": {"libc"},
     "mlir": {"flang"},
+    # Test everything if ci scripts are changed.
+    # FIXME: Figure out what is missing and add here.
+    ".ci": {"llvm", "clang", "lld", "lldb"},
 }
 
 DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
@@ -130,12 +133,11 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
 def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     projects_to_test = set()
     for modified_project in modified_projects:
-        # Skip all projects where we cannot run tests.
-        if modified_project not in PROJECT_CHECK_TARGETS:
-            continue
         if modified_project in RUNTIMES:
             continue
-        projects_to_test.add(modified_project)
+        # Skip all projects where we cannot run tests.
+        if modified_project in PROJECT_CHECK_TARGETS:
+            projects_to_test.add(modified_project)
         if modified_project not in DEPENDENTS_TO_TEST:
             continue
         for dependent_project in DEPENDENTS_TO_TEST[modified_project]:

.ci/compute_projects_test.py

@@ -188,6 +188,19 @@ def test_exclude_gn(self):
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
 
+     def test_ci(self):
+        env_variables = compute_projects.get_env_variables(
+            [".ci/compute_projects.py"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"],
+                         "clang;lld;llvm;lldb")
+        self.assertEqual(env_variables["project_check_targets"], "check-clang
+                         check-lld check-llvm check-lldb")
+        self.assertEqual(env_variables["runtimes_to_build"],
+                         "libcxx;libcxxabi;libunwind")
+        self.assertEqual(env_variables["runtimes_check_targets"], "check-cxx
+                         check-cxxabi check-unwind")
+
 
 if __name__ == "__main__":
     unittest.main()

.ci/monolithic-linux.sh

@@ -18,7 +18,6 @@ set -o pipefail
 
 MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
 BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
-INSTALL_DIR="${BUILD_DIR}/install"
 rm -rf "${BUILD_DIR}"
 
 ccache --zero-stats
@@ -28,11 +27,16 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then
   ccache --clear
 fi
 
+mkdir -p artifacts/reproducers
+
+# Make sure any clang reproducers will end up as artifacts.
+export CLANG_CRASH_DIAGNOSTICS_DIR=`realpath artifacts/reproducers`
+
 function at-exit {
   retcode=$?
 
-  mkdir -p artifacts
   ccache --print-stats > artifacts/ccache_stats.txt
+  cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
 
   # If building fails there will be no results files.
   shopt -s nullglob
@@ -49,17 +53,28 @@ trap at-exit EXIT
 
 projects="${1}"
 targets="${2}"
+runtimes="${3}"
 
 lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
 
 echo "--- cmake"
+
 export PIP_BREAK_SYSTEM_PACKAGES=1
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
+
+# Set the system llvm-symbolizer as preferred.
+export LLVM_SYMBOLIZER_PATH=`which llvm-symbolizer`
+[[ ! -f "${LLVM_SYMBOLIZER_PATH}" ]] && echo "llvm-symbolizer not found!"
+
+# Set up all runtimes either way. libcxx is a dependency of LLDB.
+# If it ends up being unused, not much harm.
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
+      -D LLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
       -G Ninja \
+      -D CMAKE_PREFIX_PATH="${HOME}/.local" \
       -D CMAKE_BUILD_TYPE=Release \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
@@ -68,69 +83,47 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_LLD=ON \
       -D CMAKE_CXX_FLAGS=-gmlt \
       -D LLVM_CCACHE_BUILD=ON \
+      -D LIBCXX_CXX_ABI=libcxxabi \
       -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
+      -D LLDB_ENABLE_PYTHON=ON \
+      -D LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=ON
 
 echo "--- ninja"
+
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
 
-runtimes="${3}"
 runtime_targets="${4}"
 
-# Compiling runtimes with just-built Clang and running their tests
-# as an additional testing for Clang.
+# Run runtimes tests.
+# We don't need to do a clean separate build of runtimes, because runtimes
+# will be built against just built clang, and because LIBCXX_TEST_PARAMS
+# and LIBCXXABI_TEST_PARAMS only affect lit configuration, which successfully
+# propagates without a clean build. Other that those two variables, builds
+# are supposed to be the same.
 if [[ "${runtimes}" != "" ]]; then
   if [[ "${runtime_targets}" == "" ]]; then
     echo "Runtimes to build are specified, but targets are not."
     exit 1
   fi
 
-  echo "--- ninja install-clang"
-
-  ninja -C ${BUILD_DIR} install-clang install-clang-resource-headers
-
-  RUNTIMES_BUILD_DIR="${MONOREPO_ROOT}/build-runtimes"
-  INSTALL_DIR="${BUILD_DIR}/install"
-  mkdir -p ${RUNTIMES_BUILD_DIR}
-
   echo "--- cmake runtimes C++26"
 
-  rm -rf "${RUNTIMES_BUILD_DIR}"
-  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
-      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
-      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
-      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
-      -D LIBCXX_CXX_ABI=libcxxabi \
-      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+  cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LIBCXX_TEST_PARAMS="std=c++26" \
-      -D LIBCXXABI_TEST_PARAMS="std=c++26" \
-      -D LLVM_LIT_ARGS="${lit_args}"
+      -D LIBCXXABI_TEST_PARAMS="std=c++26"
 
   echo "--- ninja runtimes C++26"
 
-  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
+  ninja -vC "${BUILD_DIR}" ${runtime_targets}
 
   echo "--- cmake runtimes clang modules"
 
-  # We don't need to do a clean build of runtimes, because LIBCXX_TEST_PARAMS
-  # and LIBCXXABI_TEST_PARAMS only affect lit configuration, which successfully
-  # propagates without a clean build. Other that those two variables, builds
-  # are supposed to be the same.
-
-  cmake -S "${MONOREPO_ROOT}/runtimes" -B "${RUNTIMES_BUILD_DIR}" -GNinja \
-      -D CMAKE_C_COMPILER="${INSTALL_DIR}/bin/clang" \
-      -D CMAKE_CXX_COMPILER="${INSTALL_DIR}/bin/clang++" \
-      -D LLVM_ENABLE_RUNTIMES="${runtimes}" \
-      -D LIBCXX_CXX_ABI=libcxxabi \
-      -D CMAKE_BUILD_TYPE=RelWithDebInfo \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+  cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
-      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
-      -D LLVM_LIT_ARGS="${lit_args}"
+      -D LIBCXXABI_TEST_PARAMS="enable_modules=clang"
 
   echo "--- ninja runtimes clang modules"
 
-  ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets}
+  ninja -vC "${BUILD_DIR}" ${runtime_targets}
 fi

.ci/monolithic-windows.sh

@@ -32,6 +32,7 @@ function at-exit {
 
   mkdir -p artifacts
   sccache --show-stats >> artifacts/sccache_stats.txt
+  cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
 
   # If building fails there will be no results files.
   shopt -s nullglob

.github/workflows/containers/github-action-ci/Dockerfile

@@ -32,7 +32,7 @@ RUN cmake -B ./build -G Ninja ./llvm \
   -DLLVM_ENABLE_RUNTIMES="compiler-rt" \
   -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \
   -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \
-  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build" \
+  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build;llvm-symbolizer" \
   -DCLANG_DEFAULT_LINKER="lld"
 
 RUN ninja -C ./build stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution

.github/workflows/libcxx-build-and-test.yaml

@@ -216,7 +216,8 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
         with:
-          xcode-version: 'latest'
+          # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md
+          xcode-version: '16.3'
       - uses: seanmiddleditch/gha-setup-ninja@3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6
       - name: Build and test
         run: |

.github/workflows/premerge.yaml

@@ -65,6 +65,14 @@ jobs:
           export CXX=/opt/llvm/bin/clang++
 
           ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}"
+            - name: "Upload artifact"
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: Premerge Artifacts (Linux)
+          path: artifacts/
+          retention-days: 5
+          include-hidden-files: 'true'
 
   premerge-checks-windows:
     name: Windows Premerge Checks (Test Only - Please Ignore Results)
@@ -113,6 +121,13 @@ jobs:
           set MAX_PARALLEL_LINK_JOBS=64
           call C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64
           bash .ci/monolithic-windows.sh "${{ steps.vars.outputs.windows-projects }}" "${{ steps.vars.outputs.windows-check-targets }}"
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: Premerge Artifacts (Windows)
+          path: artifacts/
+          retention-days: 5
+          include-hidden-files: 'true'
 
   premerge-check-macos:
     name: MacOS Premerge Checks

.mailmap

@@ -29,7 +29,8 @@
 <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
-<[email protected]> <[email protected]>
+Min Hsu <[email protected]> <[email protected]>
+Min Hsu <[email protected]> <[email protected]>
 <[email protected]> <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
 <[email protected]> <[email protected]>
@@ -48,6 +49,6 @@ Ramkumar Ramachandra <[email protected]> <[email protected]>
 Ramkumar Ramachandra <[email protected]> <[email protected]>
 Saleem Abdulrasool <[email protected]>
 Tommy Chiang <[email protected]> <[email protected]>
-Wang Pengcheng <[email protected]>
-Wang Pengcheng <[email protected]> <[email protected]>
-Wang Pengcheng <[email protected]> <[email protected]>
+Pengcheng Wang <[email protected]>
+Pengcheng Wang <[email protected]> <[email protected]>
+Pengcheng Wang <[email protected]> <[email protected]>

bolt/CMakeLists.txt

@@ -136,7 +136,7 @@ if (LLVM_INCLUDE_TESTS)
 endif()
 
 if (BOLT_ENABLE_RUNTIME)
-  message(STATUS "Building BOLT runtime libraries for X86")
+  message(STATUS "Building BOLT runtime libraries for ${CMAKE_SYSTEM_PROCESSOR}")
   set(extra_args "")
   if(CMAKE_SYSROOT)
     list(APPEND extra_args -DCMAKE_SYSROOT=${CMAKE_SYSROOT})

bolt/lib/Core/BinaryContext.cpp

@@ -579,6 +579,21 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
       TrimmedSize = EntriesAsAddress->size();
   };
 
+  auto printEntryDiagnostics = [&](raw_ostream &OS,
+                                   const BinaryFunction *TargetBF) {
+    OS << "FAIL: function doesn't contain this address\n";
+    if (!TargetBF)
+      return;
+    OS << "  ! function containing this address: " << *TargetBF << '\n';
+    if (!TargetBF->isFragment())
+      return;
+    OS << "  ! is a fragment with parents: ";
+    ListSeparator LS;
+    for (BinaryFunction *Parent : TargetBF->ParentFragments)
+      OS << LS << *Parent;
+    OS << '\n';
+  };
+
   ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
   if (!Section)
     return false;
@@ -646,25 +661,9 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
 
     // Function or one of its fragments.
     const BinaryFunction *TargetBF = getBinaryFunctionContainingAddress(Value);
-    const bool DoesBelongToFunction =
-        BF.containsAddress(Value) ||
-        (TargetBF && areRelatedFragments(TargetBF, &BF));
-    if (!DoesBelongToFunction) {
-      LLVM_DEBUG({
-        if (!BF.containsAddress(Value)) {
-          dbgs() << "FAIL: function doesn't contain this address\n";
-          if (TargetBF) {
-            dbgs() << "  ! function containing this address: "
-                   << TargetBF->getPrintName() << '\n';
-            if (TargetBF->isFragment()) {
-              dbgs() << "  ! is a fragment";
-              for (BinaryFunction *Parent : TargetBF->ParentFragments)
-                dbgs() << ", parent: " << Parent->getPrintName();
-              dbgs() << '\n';
-            }
-          }
-        }
-      });
+    if (!TargetBF || !areRelatedFragments(TargetBF, &BF)) {
+      LLVM_DEBUG(printEntryDiagnostics(dbgs(), TargetBF));
+      (void)printEntryDiagnostics;
       break;
     }
 
@@ -703,10 +702,7 @@ void BinaryContext::populateJumpTables() {
        ++JTI) {
     JumpTable *JT = JTI->second;
 
-    bool NonSimpleParent = false;
-    for (BinaryFunction *BF : JT->Parents)
-      NonSimpleParent |= !BF->isSimple();
-    if (NonSimpleParent)
+    if (!llvm::all_of(JT->Parents, std::mem_fn(&BinaryFunction::isSimple)))
       continue;
 
     uint64_t NextJTAddress = 0;
@@ -840,33 +836,26 @@ BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,
     assert(JT->Type == Type && "jump table types have to match");
     assert(Address == JT->getAddress() && "unexpected non-empty jump table");
 
-    // Prevent associating a jump table to a specific fragment twice.
-    if (!llvm::is_contained(JT->Parents, &Function)) {
-      assert(llvm::all_of(JT->Parents,
-                          [&](const BinaryFunction *BF) {
-                            return areRelatedFragments(&Function, BF);
-                          }) &&
-             "cannot re-use jump table of a different function");
-      // Duplicate the entry for the parent function for easy access
-      JT->Parents.push_back(&Function);
-      if (opts::Verbosity > 2) {
-        this->outs() << "BOLT-INFO: Multiple fragments access same jump table: "
-                     << JT->Parents[0]->getPrintName() << "; "
-                     << Function.getPrintName() << "\n";
-        JT->print(this->outs());
-      }
-      Function.JumpTables.emplace(Address, JT);
-      for (BinaryFunction *Parent : JT->Parents)
-        Parent->setHasIndirectTargetToSplitFragment(true);
-    }
+    if (llvm::is_contained(JT->Parents, &Function))
+      return JT->getFirstLabel();
 
-    bool IsJumpTableParent = false;
-    (void)IsJumpTableParent;
-    for (BinaryFunction *Frag : JT->Parents)
-      if (Frag == &Function)
-        IsJumpTableParent = true;
-    assert(IsJumpTableParent &&
+    // Prevent associating a jump table to a specific fragment twice.
+    auto isSibling = std::bind(&BinaryContext::areRelatedFragments, this,
+                               &Function, std::placeholders::_1);
+    assert(llvm::all_of(JT->Parents, isSibling) &&
            "cannot re-use jump table of a different function");
+    (void)isSibling;
+    if (opts::Verbosity > 2) {
+      this->outs() << "BOLT-INFO: multiple fragments access the same jump table"
+                   << ": " << *JT->Parents[0] << "; " << Function << '\n';
+      JT->print(this->outs());
+    }
+    if (JT->Parents.size() == 1)
+      JT->Parents.front()->setHasIndirectTargetToSplitFragment(true);
+    Function.setHasIndirectTargetToSplitFragment(true);
+    // Duplicate the entry for the parent function for easy access
+    JT->Parents.push_back(&Function);
+    Function.JumpTables.emplace(Address, JT);
     return JT->getFirstLabel();
   }