[𝘀𝗽𝗿] initial version

Created using spr 1.3.4

Author: aaupov

bolt/include/bolt/Profile/DataAggregator.h

@@ -370,37 +370,43 @@ class DataAggregator : public DataReader {
   /// memory.
   ///
   /// File format syntax:
-  /// {B|F|f|T} [<start_id>:]<start_offset> [<end_id>:]<end_offset> [<ft_end>]
-  ///       <count> [<mispred_count>]
+  /// {S|B|F|f|T} <start> [<end>] [<ft_end>] <count> [<mispred_count>]
   ///
-  /// B - indicates an aggregated branch
-  /// F - an aggregated fall-through
+  /// where <start>, <end>, <ft_end> have the format [<id>:]<offset>
+  ///
+  /// S - indicates an aggregated basic sample at <start>
+  /// B - indicates an aggregated branch from <start> to <end>
+  /// F - an aggregated fall-through from <start> to <end>
   /// f - an aggregated fall-through with external origin - used to disambiguate
   ///       between a return hitting a basic block head and a regular internal
   ///       jump to the block
-  /// T - an aggregated trace: branch with a fall-through (from, to, ft_end)
-  ///
-  /// <start_id> - build id of the object containing the start address. We can
-  /// skip it for the main binary and use "X" for an unknown object. This will
-  /// save some space and facilitate human parsing.
-  ///
-  /// <start_offset> - hex offset from the object base load address (0 for the
-  /// main executable unless it's PIE) to the start address.
+  /// T - an aggregated trace: branch from <start> to <end> with a fall-through
+  ///       to <ft_end>
   ///
-  /// <end_id>, <end_offset> - same for the end address.
+  /// <id> - build id of the object containing the address. We can skip it for
+  /// the main binary and use "X" for an unknown object. This will save some
+  /// space and facilitate human parsing.
   ///
-  /// <ft_end> - same for the fallthrough_end address.
+  /// <offset> - hex offset from the object base load address (0 for the
+  /// main executable unless it's PIE) to the address.
   ///
   /// <count> - total aggregated count of the branch or a fall-through.
   ///
   /// <mispred_count> - the number of times the branch was mispredicted.
   /// Omitted for fall-throughs.
   ///
   /// Example:
+  /// Basic samples profile:
+  /// S 41be50 3
+  ///
+  /// Soft-deprecated branch profile with separate branches and fall-throughs:
   /// F 41be50 41be50 3
   /// F 41be90 41be90 4
   /// B 4b1942 39b57f0 3 0
   /// B 4b196f 4b19e0 2 0
+  ///
+  /// Recommended branch profile with pre-aggregated traces:
+  /// T 4b196f 4b19e0 4b19ef 2
   void parsePreAggregated();
 
   /// Parse the full output of pre-aggregated LBR samples generated by

bolt/include/bolt/Profile/Heatmap.h

@@ -85,6 +85,9 @@ class Heatmap {
   void printSectionHotness(raw_ostream &OS) const;
 
   size_t size() const { return Map.size(); }
+
+  /// Increase bucket size to \p TargetSize, recomputing the heatmap.
+  bool resizeBucket(uint64_t TargetSize);
 };
 
 } // namespace bolt

bolt/lib/Profile/DataAggregator.cpp

@@ -68,6 +68,12 @@ FilterPID("pid",
   cl::Optional,
   cl::cat(AggregatorCategory));
 
+static cl::list<uint64_t>
+    HeatmapZoomOut("heatmap-zoom-out", cl::CommaSeparated,
+                   cl::desc("print secondary heatmaps with given bucket sizes"),
+                   cl::value_desc("bucket_size"), cl::Optional,
+                   cl::cat(HeatmapCategory));
+
 static cl::opt<bool>
 IgnoreBuildID("ignore-build-id",
   cl::desc("continue even if build-ids in input binary and perf.data mismatch"),
@@ -1210,71 +1216,80 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
   ErrorOr<StringRef> TypeOrErr = parseString(FieldSeparator);
   if (std::error_code EC = TypeOrErr.getError())
     return EC;
-  enum AggregatedLBREntry { TRACE, BRANCH, FT, FT_EXTERNAL_ORIGIN, INVALID };
+  enum AggregatedLBREntry {
+    TRACE,
+    SAMPLE,
+    BRANCH,
+    FT,
+    FT_EXTERNAL_ORIGIN,
+    INVALID
+  };
   auto Type = StringSwitch<AggregatedLBREntry>(TypeOrErr.get())
                   .Case("T", TRACE)
+                  .Case("S", SAMPLE)
                   .Case("B", BRANCH)
                   .Case("F", FT)
                   .Case("f", FT_EXTERNAL_ORIGIN)
                   .Default(INVALID);
   if (Type == INVALID) {
-    reportError("expected T, B, F or f");
+    reportError("expected T, S, B, F or f");
     return make_error_code(llvm::errc::io_error);
   }
 
-  while (checkAndConsumeFS()) {
-  }
-  ErrorOr<Location> From = parseLocationOrOffset();
-  if (std::error_code EC = From.getError())
-    return EC;
+  std::optional<Location> Addrs[3];
+  int AddrNum = 2;
+  if (Type == TRACE)
+    AddrNum = 3;
+  else if (Type == SAMPLE)
+    AddrNum = 1;
 
-  while (checkAndConsumeFS()) {
-  }
-  ErrorOr<Location> To = parseLocationOrOffset();
-  if (std::error_code EC = To.getError())
-    return EC;
+  int64_t Counters[2];
+  int CounterNum = 1;
+  if (Type == BRANCH)
+    CounterNum = 2;
 
-  ErrorOr<Location> TraceFtEnd = std::error_code();
-  if (Type == AggregatedLBREntry::TRACE) {
+  for (int I = 0; I < AddrNum; ++I) {
     while (checkAndConsumeFS()) {
     }
-    TraceFtEnd = parseLocationOrOffset();
-    if (std::error_code EC = TraceFtEnd.getError())
-      return EC;
-  }
-
-  while (checkAndConsumeFS()) {
+    if (ErrorOr<Location> Addr = parseLocationOrOffset())
+      Addrs[I] = Addr.get();
+    else
+      return Addr.getError();
   }
-  ErrorOr<int64_t> Frequency =
-      parseNumberField(FieldSeparator, Type != AggregatedLBREntry::BRANCH);
-  if (std::error_code EC = Frequency.getError())
-    return EC;
 
-  uint64_t Mispreds = 0;
-  if (Type == AggregatedLBREntry::BRANCH) {
+  for (int I = 0; I < CounterNum; ++I) {
     while (checkAndConsumeFS()) {
     }
-    ErrorOr<int64_t> MispredsOrErr = parseNumberField(FieldSeparator, true);
-    if (std::error_code EC = MispredsOrErr.getError())
-      return EC;
-    Mispreds = static_cast<uint64_t>(MispredsOrErr.get());
+    if (ErrorOr<int64_t> Count = parseNumberField(FieldSeparator, I + 1 == CounterNum))
+      Counters[I] = Count.get();
+    else
+      return Count.getError();
   }
 
   if (!checkAndConsumeNewLine()) {
     reportError("expected end of line");
     return make_error_code(llvm::errc::io_error);
   }
 
-  BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(From->Offset);
-  BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(To->Offset);
+  const uint64_t FromOffset = Addrs[0]->Offset;
+  BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(FromOffset);
+  if (FromFunc)
+    FromFunc->setHasProfileAvailable();
 
-  for (BinaryFunction *BF : {FromFunc, ToFunc})
-    if (BF)
-      BF->setHasProfileAvailable();
+  if (Type == SAMPLE) {
+    BasicSamples[FromOffset] += Counters[0];
+    return std::error_code();
+  }
+
+  const uint64_t ToOffset = Addrs[1]->Offset;
+  BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(ToOffset);
+  if (ToFunc)
+    ToFunc->setHasProfileAvailable();
 
-  uint64_t Count = static_cast<uint64_t>(Frequency.get());
+  int64_t Count = Counters[0];
+  int64_t Mispreds = Counters[1];
 
-  Trace Trace(From->Offset, To->Offset);
+  Trace Trace(FromOffset, ToOffset);
   // Taken trace
   if (Type == TRACE || Type == BRANCH) {
     TakenBranchInfo &Info = BranchLBRs[Trace];
@@ -1285,8 +1300,9 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
   }
   // Construct fallthrough part of the trace
   if (Type == TRACE) {
-    Trace.From = To->Offset;
-    Trace.To = TraceFtEnd->Offset;
+    const uint64_t TraceFtEndOffset = Addrs[2]->Offset;
+    Trace.From = ToOffset;
+    Trace.To = TraceFtEndOffset;
     Type = FromFunc == ToFunc ? FT : FT_EXTERNAL_ORIGIN;
   }
   // Add fallthrough trace
@@ -1365,6 +1381,15 @@ std::error_code DataAggregator::printLBRHeatMap() {
     HM.printCDF(opts::HeatmapOutput + ".csv");
     HM.printSectionHotness(opts::HeatmapOutput + "-section-hotness.csv");
   }
+  // Provide coarse-grained heatmap if requested via --heatmap-zoom-out
+  for (const uint64_t NewBucketSize : opts::HeatmapZoomOut) {
+    if (!HM.resizeBucket(NewBucketSize))
+      break;
+    if (opts::HeatmapOutput == "-")
+      HM.print(opts::HeatmapOutput);
+    else
+      HM.print(formatv("{0}-{1}", opts::HeatmapOutput, NewBucketSize).str());
+  }
 
   return std::error_code();
 }

bolt/lib/Profile/Heatmap.cpp

@@ -364,5 +364,18 @@ void Heatmap::printSectionHotness(raw_ostream &OS) const {
     OS << formatv("[unmapped], 0x0, 0x0, {0:f4}, 0, 0\n",
                   100.0 * UnmappedHotness / NumTotalCounts);
 }
+
+bool Heatmap::resizeBucket(uint64_t TargetSize) {
+  if (TargetSize <= BucketSize)
+    return false;
+  std::map<uint64_t, uint64_t> NewMap;
+  for (const auto [Bucket, Count] : Map) {
+    const uint64_t Address = Bucket * BucketSize;
+    NewMap[Address / TargetSize] += Count;
+  }
+  Map = NewMap;
+  BucketSize = TargetSize;
+  return true;
+}
 } // namespace bolt
 } // namespace llvm

bolt/test/X86/Inputs/pre-aggregated-basic.txt

@@ -0,0 +1,18 @@
+S 4005f0 1
+S 4005f0 1
+S 400610 1
+S 400ad1 2
+S 400b10 1
+S 400bb7 1
+S 400bbc 2
+S 400d90 1
+S 400dae 1
+S 400e00 2
+S 401170 22
+S 401180 58
+S 4011a0 33
+S 4011a9 33
+S 4011ad 58
+S 4011b2 22
+S X:7f36d18d60c0 2
+S X:7f36d18f2ce0 1

bolt/test/X86/heatmap-preagg.test

@@ -3,8 +3,12 @@
 RUN: yaml2obj %p/Inputs/blarge_new.yaml &> %t.exe
 ## Non-BOLTed input binary
 RUN: llvm-bolt-heatmap %t.exe -o %t --pa -p %p/Inputs/blarge_new.preagg.txt \
+RUN:   --heatmap-zoom-out 128,1024 --line-size 64 \
 RUN:   2>&1 | FileCheck --check-prefix CHECK-HEATMAP %s
 RUN: FileCheck %s --check-prefix CHECK-SEC-HOT --input-file %t-section-hotness.csv
+RUN: FileCheck %s --check-prefix CHECK-HM-64 --input-file %t
+RUN: FileCheck %s --check-prefix CHECK-HM-128 --input-file %t-128
+RUN: FileCheck %s --check-prefix CHECK-HM-1024 --input-file %t-1024
 
 ## BOLTed input binary
 RUN: llvm-bolt %t.exe -o %t.out --pa -p %p/Inputs/blarge_new.preagg.txt \
@@ -24,6 +28,29 @@ CHECK-SEC-HOT-NEXT: .plt, 0x401020, 0x4010b0, 4.7583, 66.6667, 0.0317
 CHECK-SEC-HOT-NEXT: .text, 0x4010b0, 0x401c25, 78.3872, 85.1064, 0.6671
 CHECK-SEC-HOT-NEXT: .fini, 0x401c28, 0x401c35, 0.0000, 0.0000, 0.0000
 
+# Only check x scales – can't check colors, and FileCheck doesn't strip color
+# codes by default.
+CHECK-HM-64: (299, 937]
+CHECK-HM-64-NEXT:   0
+CHECK-HM-64-NEXT:   0
+CHECK-HM-64-NEXT:   0   1   2   3   4   5   6   7   8   9   a   b   c   d   e   f
+CHECK-HM-64-NEXT:   048c048c048c048c048c048c048c048c048c048c048c048c048c048c048c048c
+CHECK-HM-64-NEXT:   0
+
+CHECK-HM-128: (299, 937]
+CHECK-HM-128-NEXT:  0
+CHECK-HM-128-NEXT:  0                               1
+CHECK-HM-128-NEXT:  0 1 2 3 4 5 6 7 8 9 a b c d e f 0 1 2 3 4 5 6 7 8 9 a b c d e f
+CHECK-HM-128-NEXT:  0808080808080808080808080808080808080808080808080808080808080808
+CHECK-HM-128-NEXT:  0
+
+CHECK-HM-1024: (483, 1663]
+CHECK-HM-1024-NEXT: 0
+CHECK-HM-1024-NEXT: 0   1   2   3   4   5   6   7   8   9   a   b   c   d   e   f
+CHECK-HM-1024-NEXT: 048c048c048c048c048c048c048c048c048c048c048c048c048c048c048c048c
+CHECK-HM-1024-NEXT: 0
+CHECK-HM-1024-NEXT: 0
+
 CHECK-HEATMAP-BAT: PERF2BOLT: read 79 aggregated LBR entries
 CHECK-HEATMAP-BAT: HEATMAP: invalid traces: 2
 

bolt/test/X86/pre-aggregated-perf.test

@@ -57,6 +57,14 @@ RUN: llvm-bolt %t.exe -o %t.bolt.yaml --pa -p %p/Inputs/pre-aggregated.txt \
 RUN:   --aggregate-only --profile-format=yaml --profile-use-dfs
 RUN: cat %t.bolt.yaml | FileCheck %s -check-prefix=NEWFORMAT
 
+## Test pre-aggregated basic profile
+RUN: perf2bolt %t.exe -o %t --pa -p %p/Inputs/pre-aggregated-basic.txt -o %t.ba \
+RUN:   2>&1 | FileCheck %s --check-prefix=BASIC-ERROR
+RUN: perf2bolt %t.exe -o %t --pa -p %p/Inputs/pre-aggregated-basic.txt -o %t.ba \
+RUN:   -nl 2>&1 | FileCheck %s --check-prefix=BASIC-SUCCESS
+BASIC-ERROR: BOLT-INFO: 0 out of 7 functions in the binary (0.0%) have non-empty execution profile
+BASIC-SUCCESS: BOLT-INFO: 4 out of 7 functions in the binary (57.1%) have non-empty execution profile
+
 PERF2BOLT: 0 [unknown] 7f36d18d60c0 1 main 53c 0 2
 PERF2BOLT: 1 main 451 1 SolveCubic 0 0 2
 PERF2BOLT: 1 main 490 0 [unknown] 4005f0 0 1

bolt/test/link_fdata.py

@@ -36,9 +36,9 @@
 fdata_pat = re.compile(r"([01].*) (?P<exec>\d+) (?P<mispred>\d+)")
 
 # Pre-aggregated profile:
-# {T|B|F|f} [<start_id>:]<start_offset> [<end_id>:]<end_offset> [<ft_end>]
-# <count> [<mispred_count>]
-preagg_pat = re.compile(r"(?P<type>[TBFf]) (?P<offsets_count>.*)")
+# {T|S|B|F|f} <start> [<end>] [<ft_end>] <count> [<mispred_count>]
+# <loc>: [<id>:]<offset>
+preagg_pat = re.compile(r"(?P<type>[TSBFf]) (?P<offsets_count>.*)")
 
 # No-LBR profile:
 # <is symbol?> <closest elf symbol or DSO name> <relative address> <count>