Merge branch 'main' into users/kparzysz/e04-dsp-privatize

Author: kparzysz

.ci/monolithic-linux.sh

@@ -23,11 +23,6 @@ rm -rf "${BUILD_DIR}"
 
 ccache --zero-stats
 
-if [[ -n "${CLEAR_CACHE:-}" ]]; then
-  echo "clearing cache"
-  ccache --clear
-fi
-
 mkdir -p artifacts/reproducers
 
 # Make sure any clang reproducers will end up as artifacts.

.ci/monolithic-windows.sh

@@ -21,11 +21,6 @@ BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
 
 rm -rf "${BUILD_DIR}"
 
-if [[ -n "${CLEAR_CACHE:-}" ]]; then
-  echo "clearing sccache"
-  rm -rf "$SCCACHE_DIR"
-fi
-
 sccache --zero-stats
 function at-exit {
   retcode=$?

.github/CODEOWNERS

@@ -31,9 +31,9 @@
 /clang/www/cxx_dr_status.html @Endilll
 /clang/www/make_cxx_dr_status @Endilll
 
-/clang/include/clang/CIR @lanza @bcardosolopes
-/clang/lib/CIR @lanza @bcardosolopes
-/clang/tools/cir-* @lanza @bcardosolopes
+/clang/include/clang/CIR @lanza @bcardosolopes @xlauko @andykaylor
+/clang/lib/CIR @lanza @bcardosolopes @xlauko @andykaylor
+/clang/tools/cir-* @lanza @bcardosolopes @xlauko @andykaylor
 
 /lldb/ @JDevlieghere
 

.github/workflows/build-ci-container-windows.yml

@@ -20,7 +20,7 @@ on:
 jobs:
   build-ci-container-windows:
     if: github.repository_owner == 'llvm'
-    runs-on: windows-2019
+    runs-on: windows-2022
     outputs:
       container-name: ${{ steps.vars.outputs.container-name }}
       container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
@@ -34,7 +34,7 @@ jobs:
         id: vars
         run: |
           $tag = [int64](Get-Date -UFormat %s)
-          $container_name="ghcr.io/$env:GITHUB_REPOSITORY_OWNER/ci-windows-2019"
+          $container_name="ghcr.io/$env:GITHUB_REPOSITORY_OWNER/ci-windows-2022"
           echo "container-name=${container_name}" >> $env:GITHUB_OUTPUT
           echo "container-name-tag=${container_name}:${tag}" >> $env:GITHUB_OUTPUT
           echo "container-filename=ci-windows-${tag}.tar" >> $env:GITHUB_OUTPUT
@@ -58,7 +58,7 @@ jobs:
       - build-ci-container-windows
     permissions:
       packages: write
-    runs-on: windows-2019
+    runs-on: windows-2022
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:

.github/workflows/containers/github-action-ci-windows/Dockerfile

@@ -1,6 +1,6 @@
 # Agent image for LLVM org cluster.
 # .net 4.8 is required by chocolately package manager.
-FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019
+FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2022
 
 # Restore the default Windows shell for correct batch processing.
 SHELL ["cmd", "/S", "/C"]
@@ -39,11 +39,16 @@ RUN regsvr32 /S "C:\BuildTools\DIA SDK\bin\amd64\msdia140.dll" & \
 
 # install tools as described in https://llvm.org/docs/GettingStartedVS.html
 # and a few more that were not documented...
-RUN choco install -y ninja git sccache
 # Pin an older version of Python; the current Python 3.10 fails when
 # doing "pip install" for the other dependencies, as it fails to find libxml
 # while compiling some package.
-RUN choco install -y python3 --version 3.9.7
+# We version pin the other packages as well to ensure the container build is as
+# reproducible as possible to prevent issues when upgrading only part of the
+# container.
+RUN choco install -y ninja --version 1.13.1 && \
+    choco install -y git --version 2.50.1 && \
+    choco install -y sccache --version 0.10.0 && \
+    choco install -y python3 --version 3.9.7
 
 # Testing requires psutil
 RUN pip install psutil

.github/workflows/premerge.yaml

@@ -76,7 +76,7 @@ jobs:
     if: >-
         github.repository_owner == 'llvm' &&
         (github.event_name != 'pull_request' || github.event.action != 'closed')
-    runs-on: llvm-premerge-windows-runners
+    runs-on: llvm-premerge-windows-2022-runners
     defaults:
       run:
         shell: bash

bolt/lib/Core/BinaryFunctionCallGraph.cpp

@@ -200,7 +200,8 @@ buildCallGraph(BinaryContext &BC, CgFilterFunction Filter, bool CgFromPerfData,
           if (CSI.Symbol)
             Counts.emplace_back(CSI.Symbol, CSI.Count);
       } else {
-        const uint64_t Count = BB->getExecutionCount();
+        const uint64_t Count = BC.MIB->getAnnotationWithDefault(
+            Inst, "Count", BB->getExecutionCount());
         Counts.emplace_back(DstSym, Count);
       }
 

bolt/runtime/instr.cpp

@@ -672,14 +672,15 @@ bool parseAddressRange(const char *Str, uint64_t &StartAddress,
   return true;
 }
 
+static constexpr uint32_t NameMax = 4096;
+static char TargetPath[NameMax] = {};
+
 /// Get full path to the real binary by getting current virtual address
 /// and searching for the appropriate link in address range in
 /// /proc/self/map_files
 static char *getBinaryPath() {
   const uint32_t BufSize = 1024;
-  const uint32_t NameMax = 4096;
   const char DirPath[] = "/proc/self/map_files/";
-  static char TargetPath[NameMax] = {};
   char Buf[BufSize];
 
   if (__bolt_instr_binpath[0] != '\0')
@@ -719,22 +720,31 @@ static char *getBinaryPath() {
   return nullptr;
 }
 
-ProfileWriterContext readDescriptions() {
+ProfileWriterContext readDescriptions(const uint8_t *BinContents,
+                                      uint64_t Size) {
   ProfileWriterContext Result;
-  const char *BinPath = getBinaryPath();
-  assert(BinPath && BinPath[0] != '\0', "failed to find binary path");
 
-  uint64_t FD = __open(BinPath, O_RDONLY,
-                       /*mode=*/0666);
-  assert(static_cast<int64_t>(FD) >= 0, "failed to open binary path");
+  assert((BinContents == nullptr) == (Size == 0),
+         "either empty or valid library content buffer");
+
+  if (BinContents) {
+    Result.FileDesc = -1;
+  } else {
+    const char *BinPath = getBinaryPath();
+    assert(BinPath && BinPath[0] != '\0', "failed to find binary path");
 
-  Result.FileDesc = FD;
+    uint64_t FD = __open(BinPath, O_RDONLY,
+                         /*mode=*/0666);
+    assert(static_cast<int64_t>(FD) >= 0, "failed to open binary path");
 
-  // mmap our binary to memory
-  uint64_t Size = __lseek(FD, 0, SEEK_END);
-  const uint8_t *BinContents = reinterpret_cast<uint8_t *>(
-      __mmap(0, Size, PROT_READ, MAP_PRIVATE, FD, 0));
-  assert(BinContents != MAP_FAILED, "readDescriptions: Failed to mmap self!");
+    Result.FileDesc = FD;
+
+    // mmap our binary to memory
+    Size = __lseek(FD, 0, SEEK_END);
+    BinContents = reinterpret_cast<uint8_t *>(
+        __mmap(0, Size, PROT_READ, MAP_PRIVATE, FD, 0));
+    assert(BinContents != MAP_FAILED, "readDescriptions: Failed to mmap self!");
+  }
   Result.MMapPtr = BinContents;
   Result.MMapSize = Size;
   const Elf64_Ehdr *Hdr = reinterpret_cast<const Elf64_Ehdr *>(BinContents);
@@ -1509,7 +1519,7 @@ extern "C" void __bolt_instr_clear_counters() {
 }
 
 /// This is the entry point for profile writing.
-/// There are three ways of getting here:
+/// There are four ways of getting here:
 ///
 ///  * Program execution ended, finalization methods are running and BOLT
 ///    hooked into FINI from your binary dynamic section;
@@ -1518,9 +1528,18 @@ extern "C" void __bolt_instr_clear_counters() {
 ///  * BOLT prints this function address so you can attach a debugger and
 ///    call this function directly to get your profile written to disk
 ///    on demand.
+///  * Application can, at interesting runtime point, iterate through all
+///    the loaded native libraries and for each call dlopen() and dlsym()
+///    to get a pointer to this function and call through the acquired
+///    function pointer to dump profile data.
 ///
 extern "C" void __attribute((force_align_arg_pointer))
-__bolt_instr_data_dump(int FD) {
+__bolt_instr_data_dump(int FD, const char *LibPath = nullptr,
+                       const uint8_t *LibContents = nullptr,
+                       uint64_t LibSize = 0) {
+  if (LibPath)
+    strCopy(TargetPath, LibPath, NameMax);
+
   // Already dumping
   if (!GlobalWriteProfileMutex->acquire())
     return;
@@ -1531,7 +1550,7 @@ __bolt_instr_data_dump(int FD) {
   assert(ret == 0, "Failed to ftruncate!");
   BumpPtrAllocator HashAlloc;
   HashAlloc.setMaxSize(0x6400000);
-  ProfileWriterContext Ctx = readDescriptions();
+  ProfileWriterContext Ctx = readDescriptions(LibContents, LibSize);
   Ctx.CallFlowTable = new (HashAlloc, 0) CallFlowHashTable(HashAlloc);
 
   DEBUG(printStats(Ctx));
@@ -1551,8 +1570,10 @@ __bolt_instr_data_dump(int FD) {
   Ctx.CallFlowTable->forEachElement(visitCallFlowEntry, FD, &Ctx);
 
   __fsync(FD);
-  __munmap((void *)Ctx.MMapPtr, Ctx.MMapSize);
-  __close(Ctx.FileDesc);
+  if (Ctx.FileDesc != -1) {
+    __munmap((void *)Ctx.MMapPtr, Ctx.MMapSize);
+    __close(Ctx.FileDesc);
+  }
   HashAlloc.destroy();
   GlobalWriteProfileMutex->release();
   DEBUG(report("Finished writing profile.\n"));

clang-tools-extra/README.txt

@@ -8,12 +8,13 @@ Clang frontend.  These tools are kept in a separate "extra" repository to
 allow lighter weight checkouts of the core Clang codebase.
 
 All discussion regarding Clang, Clang-based tools, and code in this repository
-should be held using the standard Clang forum:
+should be held using the standard Clang forums:
   https://discourse.llvm.org/c/clang
+  https://discourse.llvm.org/c/clang/clang-tidy/71
+  https://discourse.llvm.org/c/clang/clangd/34
 
-Code review for this tree should take place on the standard Clang patch and
-commit lists:
-  http://lists.llvm.org/mailman/listinfo/cfe-commits
+Code review for this tree should take place on Github:
+  https://github.com/llvm/llvm-project/pulls?q=label%3Aclang-tools-extra
 
 If you find a bug in these tools, please file it in the LLVM bug tracker:
   https://github.com/llvm/llvm-project/issues/

clang-tools-extra/clang-doc/JSONGenerator.cpp

@@ -45,7 +45,7 @@ static auto SerializeReferenceLambda = [](const auto &Ref, Object &Object) {
 
 static json::Object
 serializeLocation(const Location &Loc,
-                  const std::optional<StringRef> &RepositoryUrl) {
+                  const std::optional<StringRef> RepositoryUrl) {
   Object LocationObj = Object();
   LocationObj["LineNumber"] = Loc.StartLineNumber;
   LocationObj["Filename"] = Loc.Filename;
@@ -169,7 +169,7 @@ static json::Value serializeComment(const CommentInfo &I) {
 
 static void
 serializeCommonAttributes(const Info &I, json::Object &Obj,
-                          const std::optional<StringRef> &RepositoryUrl) {
+                          const std::optional<StringRef> RepositoryUrl) {
   Obj["Name"] = I.Name;
   Obj["USR"] = toHex(toStringRef(I.USR));
 
@@ -211,9 +211,9 @@ static void serializeReference(const Reference &Ref, Object &ReferenceObj) {
 // differently. Only enums, records, and typedefs are handled here.
 static void
 serializeCommonChildren(const ScopeChildren &Children, json::Object &Obj,
-                        const std::optional<StringRef> &RepositoryUrl) {
-  static auto SerializeInfo = [&RepositoryUrl](const auto &Info,
-                                               Object &Object) {
+                        const std::optional<StringRef> RepositoryUrl) {
+  static auto SerializeInfo = [RepositoryUrl](const auto &Info,
+                                              Object &Object) {
     serializeInfo(Info, Object, RepositoryUrl);
   };
 
@@ -304,7 +304,7 @@ static void serializeInfo(const FieldTypeInfo &I, Object &Obj) {
 }
 
 static void serializeInfo(const FunctionInfo &F, json::Object &Obj,
-                          const std::optional<StringRef> &RepositoryURL) {
+                          const std::optional<StringRef> RepositoryURL) {
   serializeCommonAttributes(F, Obj, RepositoryURL);
   Obj["IsStatic"] = F.IsStatic;
 
@@ -459,7 +459,7 @@ static void serializeInfo(const RecordInfo &I, json::Object &Obj,
 }
 
 static void serializeInfo(const VarInfo &I, json::Object &Obj,
-                          const std::optional<StringRef> &RepositoryUrl) {
+                          const std::optional<StringRef> RepositoryUrl) {
   serializeCommonAttributes(I, Obj, RepositoryUrl);
   Obj["IsStatic"] = I.IsStatic;
   auto TypeObj = Object();
@@ -468,15 +468,15 @@ static void serializeInfo(const VarInfo &I, json::Object &Obj,
 }
 
 static void serializeInfo(const NamespaceInfo &I, json::Object &Obj,
-                          const std::optional<StringRef> &RepositoryUrl) {
+                          const std::optional<StringRef> RepositoryUrl) {
   serializeCommonAttributes(I, Obj, RepositoryUrl);
 
   if (!I.Children.Namespaces.empty())
     serializeArray(I.Children.Namespaces, Obj, "Namespaces",
                    SerializeReferenceLambda);
 
-  static auto SerializeInfo = [&RepositoryUrl](const auto &Info,
-                                               Object &Object) {
+  static auto SerializeInfo = [RepositoryUrl](const auto &Info,
+                                              Object &Object) {
     serializeInfo(Info, Object, RepositoryUrl);
   };