Merge branch 'main' into xegpu_blocking_pass

Author: chencha3

.github/new-prs-labeler.yml

@@ -702,6 +702,7 @@ mlgo:
   - llvm/unittests/CodeGen/ML*
   - llvm/test/CodeGen/MLRegAlloc/**
   - llvm/utils/mlgo-utils/**
+  - llvm/docs/MLGO.rst
 
 tools:llvm-exegesis:
   - llvm/tools/llvm-exegesis/**

.github/workflows/containers/github-action-ci-windows/Dockerfile

@@ -108,7 +108,7 @@ RUN choco install -y handle
 
 RUN pip3 install pywin32 buildbot-worker==2.8.4
 
-ARG RUNNER_VERSION=2.323.0
+ARG RUNNER_VERSION=2.324.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 
 RUN powershell -Command \

.github/workflows/containers/github-action-ci/Dockerfile

@@ -2,7 +2,7 @@ FROM docker.io/library/ubuntu:24.04 as base
 ENV LLVM_SYSROOT=/opt/llvm
 
 FROM base as stage1-toolchain
-ENV LLVM_VERSION=20.1.1
+ENV LLVM_VERSION=20.1.4
 
 RUN apt-get update && \
     apt-get install -y \
@@ -86,7 +86,7 @@ WORKDIR /home/gha
 
 FROM ci-container as ci-container-agent
 
-ENV GITHUB_RUNNER_VERSION=2.323.0
+ENV GITHUB_RUNNER_VERSION=2.324.0
 
 RUN mkdir actions-runner && \
     cd actions-runner && \

bolt/include/bolt/Core/BinaryFunction.h

@@ -142,8 +142,8 @@ class BinaryFunction {
   /// Types of profile the function can use. Could be a combination.
   enum {
     PF_NONE = 0,     /// No profile.
-    PF_LBR = 1,      /// Profile is based on last branch records.
-    PF_SAMPLE = 2,   /// Non-LBR sample-based profile.
+    PF_BRANCH = 1,   /// Profile is based on branches or branch stacks.
+    PF_BASIC = 2,    /// Non-branch IP sample-based profile.
     PF_MEMEVENT = 4, /// Profile has mem events.
   };
 
@@ -392,7 +392,7 @@ class BinaryFunction {
   float ProfileMatchRatio{0.0f};
 
   /// Raw branch count for this function in the profile.
-  uint64_t RawBranchCount{0};
+  uint64_t RawSampleCount{0};
 
   /// Dynamically executed function bytes, used for density computation.
   uint64_t SampleCountInBytes{0};
@@ -1893,11 +1893,11 @@ class BinaryFunction {
 
   /// Return the raw profile information about the number of branch
   /// executions corresponding to this function.
-  uint64_t getRawBranchCount() const { return RawBranchCount; }
+  uint64_t getRawSampleCount() const { return RawSampleCount; }
 
   /// Set the profile data about the number of branch executions corresponding
   /// to this function.
-  void setRawBranchCount(uint64_t Count) { RawBranchCount = Count; }
+  void setRawSampleCount(uint64_t Count) { RawSampleCount = Count; }
 
   /// Return the number of dynamically executed bytes, from raw perf data.
   uint64_t getSampleCountInBytes() const { return SampleCountInBytes; }

bolt/include/bolt/Profile/DataAggregator.h

@@ -92,16 +92,6 @@ class DataAggregator : public DataReader {
     uint64_t Addr;
   };
 
-  /// Used for parsing specific pre-aggregated input files.
-  struct AggregatedLBREntry {
-    enum Type : char { BRANCH = 0, FT, FT_EXTERNAL_ORIGIN, TRACE };
-    Location From;
-    Location To;
-    uint64_t Count;
-    uint64_t Mispreds;
-    Type EntryType;
-  };
-
   struct Trace {
     uint64_t From;
     uint64_t To;
@@ -131,7 +121,6 @@ class DataAggregator : public DataReader {
   /// and use them later for processing and assigning profile.
   std::unordered_map<Trace, TakenBranchInfo, TraceHash> BranchLBRs;
   std::unordered_map<Trace, FTInfo, TraceHash> FallthroughLBRs;
-  std::vector<AggregatedLBREntry> AggregatedLBRs;
   std::unordered_map<uint64_t, uint64_t> BasicSamples;
   std::vector<PerfMemSample> MemSamples;
 
@@ -223,11 +212,6 @@ class DataAggregator : public DataReader {
   uint64_t NumTraces{0};
   uint64_t NumInvalidTraces{0};
   uint64_t NumLongRangeTraces{0};
-  /// Specifies how many samples were recorded in cold areas if we are dealing
-  /// with profiling data collected in a bolted binary. For LBRs, incremented
-  /// for the source of the branch to avoid counting cold activity twice (one
-  /// for source and another for destination).
-  uint64_t NumColdSamples{0};
   uint64_t NumTotalSamples{0};
 
   /// Looks into system PATH for Linux Perf and set up the aggregator to use it
@@ -257,7 +241,8 @@ class DataAggregator : public DataReader {
 
   /// Semantic actions - parser hooks to interpret parsed perf samples
   /// Register a sample (non-LBR mode), i.e. a new hit at \p Address
-  bool doSample(BinaryFunction &Func, const uint64_t Address, uint64_t Count);
+  bool doBasicSample(BinaryFunction &Func, const uint64_t Address,
+                     uint64_t Count);
 
   /// Register an intraprocedural branch \p Branch.
   bool doIntraBranch(BinaryFunction &Func, uint64_t From, uint64_t To,
@@ -422,9 +407,6 @@ class DataAggregator : public DataReader {
   /// an external tool.
   std::error_code parsePreAggregatedLBRSamples();
 
-  /// Process parsed pre-aggregated data.
-  void processPreAggregated();
-
   /// If \p Address falls into the binary address space based on memory
   /// mapping info \p MMI, then adjust it for further processing by subtracting
   /// the base load address. External addresses, i.e. addresses that do not
@@ -486,7 +468,6 @@ class DataAggregator : public DataReader {
   void dump(const PerfMemSample &Sample) const;
 
   /// Profile diagnostics print methods
-  void printColdSamplesDiagnostic() const;
   void printLongRangeTracesDiagnostic() const;
   void printBranchSamplesDiagnostics() const;
   void printBasicSamplesDiagnostics(uint64_t OutOfRangeSamples) const;

bolt/include/bolt/Profile/DataReader.h

@@ -114,10 +114,8 @@ struct FuncBranchData {
 
   FuncBranchData() {}
 
-  FuncBranchData(StringRef Name, ContainerTy Data)
-      : Name(Name), Data(std::move(Data)) {}
-
-  FuncBranchData(StringRef Name, ContainerTy Data, ContainerTy EntryData)
+  FuncBranchData(StringRef Name, ContainerTy Data = ContainerTy(),
+                 ContainerTy EntryData = ContainerTy())
       : Name(Name), Data(std::move(Data)), EntryData(std::move(EntryData)) {}
 
   ErrorOr<const BranchInfo &> getBranch(uint64_t From, uint64_t To) const;
@@ -205,22 +203,23 @@ struct FuncMemData {
 
   FuncMemData() {}
 
-  FuncMemData(StringRef Name, ContainerTy Data)
+  FuncMemData(StringRef Name, ContainerTy Data = ContainerTy())
       : Name(Name), Data(std::move(Data)) {}
 };
 
 /// Similar to BranchInfo, but instead of recording from-to address (an edge),
 /// it records the address of a perf event and the number of times samples hit
 /// this address.
-struct SampleInfo {
+struct BasicSampleInfo {
   Location Loc;
   int64_t Hits;
 
-  SampleInfo(Location Loc, int64_t Hits) : Loc(std::move(Loc)), Hits(Hits) {}
+  BasicSampleInfo(Location Loc, int64_t Hits)
+      : Loc(std::move(Loc)), Hits(Hits) {}
 
-  bool operator==(const SampleInfo &RHS) const { return Loc == RHS.Loc; }
+  bool operator==(const BasicSampleInfo &RHS) const { return Loc == RHS.Loc; }
 
-  bool operator<(const SampleInfo &RHS) const {
+  bool operator<(const BasicSampleInfo &RHS) const {
     if (Loc < RHS.Loc)
       return true;
 
@@ -229,18 +228,18 @@ struct SampleInfo {
 
   void print(raw_ostream &OS) const;
 
-  void mergeWith(const SampleInfo &SI);
+  void mergeWith(const BasicSampleInfo &SI);
 };
 
 /// Helper class to store samples recorded in the address space of a given
 /// function, analogous to FuncBranchData but for samples instead of branches.
-struct FuncSampleData {
-  typedef std::vector<SampleInfo> ContainerTy;
+struct FuncBasicSampleData {
+  typedef std::vector<BasicSampleInfo> ContainerTy;
 
   StringRef Name;
   ContainerTy Data;
 
-  FuncSampleData(StringRef Name, ContainerTy Data)
+  FuncBasicSampleData(StringRef Name, ContainerTy Data = ContainerTy())
       : Name(Name), Data(std::move(Data)) {}
 
   /// Get the number of samples recorded in [Start, End)
@@ -308,7 +307,7 @@ class DataReader : public ProfileReaderBase {
   /// The last step is to infer edge counts based on BB execution count. Note
   /// this is the opposite of the LBR way, where we infer BB execution count
   /// based on edge counts.
-  void readSampleData(BinaryFunction &BF);
+  void readBasicSampleData(BinaryFunction &BF);
 
   /// Convert function-level branch data into instruction annotations.
   void convertBranchData(BinaryFunction &BF) const;
@@ -382,7 +381,8 @@ class DataReader : public ProfileReaderBase {
   /// Return mem data matching one of the names in \p FuncNames.
   FuncMemData *getMemDataForNames(const std::vector<StringRef> &FuncNames);
 
-  FuncSampleData *getFuncSampleData(const std::vector<StringRef> &FuncNames);
+  FuncBasicSampleData *
+  getFuncBasicSampleData(const std::vector<StringRef> &FuncNames);
 
   /// Return a vector of all FuncBranchData matching the list of names.
   /// Internally use fuzzy matching to match special names like LTO-generated
@@ -425,7 +425,7 @@ class DataReader : public ProfileReaderBase {
   }
 
   using NamesToBranchesMapTy = std::map<StringRef, FuncBranchData>;
-  using NamesToSamplesMapTy = std::map<StringRef, FuncSampleData>;
+  using NamesToBasicSamplesMapTy = std::map<StringRef, FuncBasicSampleData>;
   using NamesToMemEventsMapTy = std::map<StringRef, FuncMemData>;
   using FuncsToBranchesMapTy =
       std::unordered_map<const BinaryFunction *, FuncBranchData *>;
@@ -474,7 +474,7 @@ class DataReader : public ProfileReaderBase {
     return parseLocation(EndChar, EndNl, true);
   }
   ErrorOr<BranchInfo> parseBranchInfo();
-  ErrorOr<SampleInfo> parseSampleInfo();
+  ErrorOr<BasicSampleInfo> parseSampleInfo();
   ErrorOr<MemInfo> parseMemInfo();
   ErrorOr<bool> maybeParseNoLBRFlag();
   ErrorOr<bool> maybeParseBATFlag();
@@ -488,7 +488,7 @@ class DataReader : public ProfileReaderBase {
   unsigned Line{0};
   unsigned Col{0};
   NamesToBranchesMapTy NamesToBranches;
-  NamesToSamplesMapTy NamesToSamples;
+  NamesToBasicSamplesMapTy NamesToBasicSamples;
   NamesToMemEventsMapTy NamesToMemEvents;
   FuncsToBranchesMapTy FuncsToBranches;
   FuncsToMemDataMapTy FuncsToMemData;

bolt/include/bolt/Profile/Heatmap.h

@@ -52,14 +52,17 @@ class Heatmap {
       : BucketSize(BucketSize), MinAddress(MinAddress), MaxAddress(MaxAddress),
         TextSections(TextSections) {}
 
+  uint64_t HotStart{0};
+  uint64_t HotEnd{0};
+
   inline bool ignoreAddress(uint64_t Address) const {
     return (Address > MaxAddress) || (Address < MinAddress);
   }
 
   /// Register a single sample at \p Address.
-  void registerAddress(uint64_t Address) {
+  void registerAddress(uint64_t Address, uint64_t Count) {
     if (!ignoreAddress(Address))
-      ++Map[Address / BucketSize];
+      Map[Address / BucketSize] += Count;
   }
 
   /// Register \p Count samples at [\p StartAddress, \p EndAddress ].

bolt/include/bolt/Profile/ProfileYAMLMapping.h

@@ -230,8 +230,8 @@ LLVM_YAML_STRONG_TYPEDEF(uint16_t, PROFILE_PF)
 
 template <> struct ScalarBitSetTraits<PROFILE_PF> {
   static void bitset(IO &io, PROFILE_PF &value) {
-    io.bitSetCase(value, "lbr", BinaryFunction::PF_LBR);
-    io.bitSetCase(value, "sample", BinaryFunction::PF_SAMPLE);
+    io.bitSetCase(value, "lbr", BinaryFunction::PF_BRANCH);
+    io.bitSetCase(value, "sample", BinaryFunction::PF_BASIC);
     io.bitSetCase(value, "memevent", BinaryFunction::PF_MEMEVENT);
   }
 };

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -17,7 +17,13 @@
 
 namespace opts {
 
-extern bool HeatmapMode;
+enum HeatmapModeKind {
+  HM_None = 0,
+  HM_Exclusive, // llvm-bolt-heatmap
+  HM_Optional   // perf2bolt --heatmap
+};
+
+extern HeatmapModeKind HeatmapMode;
 extern bool BinaryAnalysisMode;
 
 extern llvm::cl::OptionCategory BoltCategory;
@@ -45,6 +51,7 @@ extern llvm::cl::opt<unsigned> HeatmapBlock;
 extern llvm::cl::opt<unsigned long long> HeatmapMaxAddress;
 extern llvm::cl::opt<unsigned long long> HeatmapMinAddress;
 extern llvm::cl::opt<bool> HeatmapPrintMappings;
+extern llvm::cl::opt<std::string> HeatmapOutput;
 extern llvm::cl::opt<bool> HotData;
 extern llvm::cl::opt<bool> HotFunctionsAtEnd;
 extern llvm::cl::opt<bool> HotText;

bolt/lib/Core/BinaryFunction.cpp

@@ -66,7 +66,7 @@ extern cl::opt<bool> UpdateDebugSections;
 extern cl::opt<unsigned> Verbosity;
 
 extern bool BinaryAnalysisMode;
-extern bool HeatmapMode;
+extern HeatmapModeKind HeatmapMode;
 extern bool processAllFunctions();
 
 static cl::opt<bool> CheckEncoding(
@@ -473,7 +473,7 @@ void BinaryFunction::print(raw_ostream &OS, std::string Annotation) {
     OS << "\n  Image       : 0x" << Twine::utohexstr(getImageAddress());
   if (ExecutionCount != COUNT_NO_PROFILE) {
     OS << "\n  Exec Count  : " << ExecutionCount;
-    OS << "\n  Branch Count: " << RawBranchCount;
+    OS << "\n  Sample Count: " << RawSampleCount;
     OS << "\n  Profile Acc : " << format("%.1f%%", ProfileMatchRatio * 100.0f);
   }
 
@@ -3326,7 +3326,7 @@ void BinaryFunction::duplicateConstantIslands() {
 static std::string constructFilename(std::string Filename,
                                      std::string Annotation,
                                      std::string Suffix) {
-  std::replace(Filename.begin(), Filename.end(), '/', '-');
+  llvm::replace(Filename, '/', '-');
   if (!Annotation.empty())
     Annotation.insert(0, "-");
   if (Filename.size() + Annotation.size() + Suffix.size() > MAX_PATH) {