Merge branch 'main' into inbelic/rs-resource-range-intervals

Author: inbelic

.ci/monolithic-linux.sh

@@ -38,7 +38,7 @@ function at-exit {
 
   ccache --print-stats > artifacts/ccache_stats.txt
   cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
-  cp "${BUILD_DIR}"/test-results.*.xml artifacts/
+  cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || :
 
   # If building fails there will be no results files.
   shopt -s nullglob
@@ -47,7 +47,7 @@ function at-exit {
     python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_buildkite.py ":linux: Linux x64 Test Results" \
       "linux-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
   else
-    python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":linux: Linux x64 Test Results" \
+    python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":penguin: Linux x64 Test Results" \
       $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
   fi
 }

.ci/monolithic-windows.sh

@@ -33,7 +33,7 @@ function at-exit {
   mkdir -p artifacts
   sccache --show-stats >> artifacts/sccache_stats.txt
   cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
-  cp "${BUILD_DIR}"/test-results.*.xml artifacts/
+  cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || :
 
   # If building fails there will be no results files.
   shopt -s nullglob
@@ -42,7 +42,7 @@ function at-exit {
     python "${MONOREPO_ROOT}"/.ci/generate_test_report_buildkite.py ":windows: Windows x64 Test Results" \
       "windows-x64-test-results" $retcode "${BUILD_DIR}"/test-results.*.xml
   else
-    python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":windows: Windows x64 Test Results" \
+    python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":window: Windows x64 Test Results" \
       $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
   fi
 }

.github/CODEOWNERS

@@ -128,7 +128,7 @@
 /mlir/**/Transforms/SROA.* @moxinilian
 
 # BOLT
-/bolt/ @aaupov @maksfb @rafaelauler @ayermolo @yota9
+/bolt/ @aaupov @maksfb @rafaelauler @ayermolo @yota9 @paschalis-mpeis
 
 # Bazel build system.
 /utils/bazel/ @rupprecht @keith @aaronmondal

bolt/docs/Heatmaps.md

@@ -89,7 +89,13 @@ For the generation, the default bucket size was used with a line size of 128.
 Some useful options are:
 
 ```
--line-size=<uint>   - number of entries per line (default 256)
+-line-size=<uint>   - number of entries per line (default 256).
+                      Use a smaller value (e.g. 128) if the heatmap doesn't fit
+                      the screen horizontally.
+-block-size=<initial size>[,<zoom-out size>,...] - heatmap bucket size,
+                      optionally followed by zoom-out sizes to produce coarse-
+                      grained heatmaps. Size can be specified in human-readable
+                      format with [kKmMgG][i][B] suffix. Default 64B, 4K, 256K.
 -max-address=<uint> - maximum address considered valid for heatmap (default 4GB)
 -print-mappings     - print mappings in the legend, between characters/blocks and text sections (default false)
 ```

bolt/include/bolt/Profile/Heatmap.h

@@ -85,6 +85,9 @@ class Heatmap {
   void printSectionHotness(raw_ostream &OS) const;
 
   size_t size() const { return Map.size(); }
+
+  /// Increase bucket size to \p NewSize, recomputing the heatmap.
+  void resizeBucket(uint64_t NewSize);
 };
 
 } // namespace bolt

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -23,6 +23,15 @@ enum HeatmapModeKind {
   HM_Optional   // perf2bolt --heatmap
 };
 
+using HeatmapBlockSizes = std::vector<unsigned>;
+struct HeatmapBlockSpecParser : public llvm::cl::parser<HeatmapBlockSizes> {
+  explicit HeatmapBlockSpecParser(llvm::cl::Option &O)
+      : llvm::cl::parser<HeatmapBlockSizes>(O) {}
+  // Return true on error.
+  bool parse(llvm::cl::Option &O, llvm::StringRef ArgName, llvm::StringRef Arg,
+             HeatmapBlockSizes &Val);
+};
+
 extern HeatmapModeKind HeatmapMode;
 extern bool BinaryAnalysisMode;
 
@@ -47,7 +56,8 @@ extern llvm::cl::opt<bool> EqualizeBBCounts;
 extern llvm::cl::opt<bool> ForcePatch;
 extern llvm::cl::opt<bool> RemoveSymtab;
 extern llvm::cl::opt<unsigned> ExecutionCountThreshold;
-extern llvm::cl::opt<unsigned> HeatmapBlock;
+extern llvm::cl::opt<HeatmapBlockSizes, false, HeatmapBlockSpecParser>
+    HeatmapBlock;
 extern llvm::cl::opt<unsigned long long> HeatmapMaxAddress;
 extern llvm::cl::opt<unsigned long long> HeatmapMinAddress;
 extern llvm::cl::opt<bool> HeatmapPrintMappings;

bolt/lib/Profile/DataAggregator.cpp

@@ -1314,8 +1314,9 @@ std::error_code DataAggregator::printLBRHeatMap() {
     opts::HeatmapMaxAddress = 0xffffffffffffffff;
     opts::HeatmapMinAddress = KernelBaseAddr;
   }
-  Heatmap HM(opts::HeatmapBlock, opts::HeatmapMinAddress,
-             opts::HeatmapMaxAddress, getTextSections(BC));
+  opts::HeatmapBlockSizes &HMBS = opts::HeatmapBlock;
+  Heatmap HM(HMBS[0], opts::HeatmapMinAddress, opts::HeatmapMaxAddress,
+             getTextSections(BC));
   auto getSymbolValue = [&](const MCSymbol *Symbol) -> uint64_t {
     if (Symbol)
       if (ErrorOr<uint64_t> SymValue = BC->getSymbolValue(*Symbol))
@@ -1365,6 +1366,14 @@ std::error_code DataAggregator::printLBRHeatMap() {
     HM.printCDF(opts::HeatmapOutput + ".csv");
     HM.printSectionHotness(opts::HeatmapOutput + "-section-hotness.csv");
   }
+  // Provide coarse-grained heatmaps if requested via zoom-out scales
+  for (const uint64_t NewBucketSize : ArrayRef(HMBS).drop_front()) {
+    HM.resizeBucket(NewBucketSize);
+    if (opts::HeatmapOutput == "-")
+      HM.print(opts::HeatmapOutput);
+    else
+      HM.print(formatv("{0}-{1}", opts::HeatmapOutput, NewBucketSize).str());
+  }
 
   return std::error_code();
 }

bolt/lib/Profile/Heatmap.cpp

@@ -55,6 +55,8 @@ void Heatmap::print(StringRef FileName) const {
     errs() << "error opening output file: " << EC.message() << '\n';
     exit(1);
   }
+  outs() << "HEATMAP: dumping heatmap with bucket size " << BucketSize << " to "
+         << FileName << '\n';
   print(OS);
 }
 
@@ -364,5 +366,13 @@ void Heatmap::printSectionHotness(raw_ostream &OS) const {
     OS << formatv("[unmapped], 0x0, 0x0, {0:f4}, 0, 0\n",
                   100.0 * UnmappedHotness / NumTotalCounts);
 }
+
+void Heatmap::resizeBucket(uint64_t NewSize) {
+  std::map<uint64_t, uint64_t> NewMap;
+  for (const auto [Bucket, Count] : Map)
+    NewMap[Bucket * BucketSize / NewSize] += Count;
+  Map = NewMap;
+  BucketSize = NewSize;
+}
 } // namespace bolt
 } // namespace llvm

bolt/lib/Rewrite/RewriteInstance.cpp

@@ -1341,6 +1341,19 @@ void RewriteInstance::discoverFileObjects() {
         }
       }
     }
+
+    // The linker may omit data markers for absolute long veneers. Introduce
+    // those markers artificially to assist the disassembler.
+    for (BinaryFunction &BF :
+         llvm::make_second_range(BC->getBinaryFunctions())) {
+      if (BF.getOneName().starts_with("__AArch64AbsLongThunk_") &&
+          BF.getSize() == 16 && !BF.getSizeOfDataInCodeAt(8)) {
+        BC->errs() << "BOLT-WARNING: missing data marker detected in veneer "
+                   << BF << '\n';
+        BF.markDataAtOffset(8);
+        BC->AddressToConstantIslandMap[BF.getAddress() + 8] = &BF;
+      }
+    }
   }
 
   if (!BC->IsLinuxKernel) {

bolt/lib/Utils/CommandLineOpts.cpp

@@ -12,6 +12,7 @@
 
 #include "bolt/Utils/CommandLineOpts.h"
 #include "VCSVersion.inc"
+#include "llvm/Support/Regex.h"
 
 using namespace llvm;
 
@@ -103,10 +104,56 @@ ExecutionCountThreshold("execution-count-threshold",
   cl::Hidden,
   cl::cat(BoltOptCategory));
 
-cl::opt<unsigned>
-    HeatmapBlock("block-size",
-                 cl::desc("size of a heat map block in bytes (default 64)"),
-                 cl::init(64), cl::cat(HeatmapCategory));
+bool HeatmapBlockSpecParser::parse(cl::Option &O, StringRef ArgName,
+                                   StringRef Arg, HeatmapBlockSizes &Val) {
+  // Parses a human-readable suffix into a shift amount or nullopt on error.
+  auto parseSuffix = [](StringRef Suffix) -> std::optional<unsigned> {
+    if (Suffix.empty())
+      return 0;
+    if (!Regex{"^[kKmMgG]i?[bB]?$"}.match(Suffix))
+      return std::nullopt;
+    // clang-format off
+    switch (Suffix.front()) {
+      case 'k': case 'K': return 10;
+      case 'm': case 'M': return 20;
+      case 'g': case 'G': return 30;
+    }
+    // clang-format on
+    llvm_unreachable("Unexpected suffix");
+  };
+
+  SmallVector<StringRef> Sizes;
+  Arg.split(Sizes, ',');
+  unsigned PreviousSize = 0;
+  for (StringRef Size : Sizes) {
+    StringRef OrigSize = Size;
+    unsigned &SizeVal = Val.emplace_back(0);
+    if (Size.consumeInteger(10, SizeVal)) {
+      O.error("'" + OrigSize + "' value can't be parsed as an integer");
+      return true;
+    }
+    if (std::optional<unsigned> ShiftAmt = parseSuffix(Size)) {
+      SizeVal <<= *ShiftAmt;
+    } else {
+      O.error("'" + Size + "' value can't be parsed as a suffix");
+      return true;
+    }
+    if (SizeVal <= PreviousSize || (PreviousSize && SizeVal % PreviousSize)) {
+      O.error("'" + OrigSize + "' must be a multiple of previous value");
+      return true;
+    }
+    PreviousSize = SizeVal;
+  }
+  return false;
+}
+
+cl::opt<opts::HeatmapBlockSizes, false, opts::HeatmapBlockSpecParser>
+    HeatmapBlock(
+        "block-size", cl::value_desc("initial_size{,zoom-out_size,...}"),
+        cl::desc("heatmap bucket size, optionally followed by zoom-out sizes "
+                 "for coarse-grained heatmaps (default 64B, 4K, 256K)."),
+        cl::init(HeatmapBlockSizes{/*Initial*/ 64, /*Zoom-out*/ 4096, 262144}),
+        cl::cat(HeatmapCategory));
 
 cl::opt<unsigned long long> HeatmapMaxAddress(
     "max-address", cl::init(0xffffffff),