[𝘀𝗽𝗿] initial version

Created using spr 1.3.4

Author: aaupov

bolt/docs/BAT.md

@@ -54,7 +54,7 @@ Functions table:
 |      table       |
 |                  |
 | Secondary entry  |
-|      points      |
+|  points and LPs  |
 |------------------|
 
 ```
@@ -80,7 +80,7 @@ Hot indices are delta encoded, implicitly starting at zero.
 | `HotIndex` | Delta, ULEB128 | Index of corresponding hot function in hot functions table | Cold |
 | `FuncHash` | 8b | Function hash for input function | Hot |
 | `NumBlocks` | ULEB128 | Number of basic blocks in the original function | Hot |
-| `NumSecEntryPoints` | ULEB128 | Number of secondary entry points in the original function | Hot |
+| `NumSecEntryPoints` | ULEB128 | Number of secondary entry points and landing pads in the original function | Hot |
 | `ColdInputSkew` | ULEB128 | Skew to apply to all input offsets | Cold |
 | `NumEntries` | ULEB128 | Number of address translation entries for a function | Both |
 | `EqualElems` | ULEB128 | Number of equal offsets in the beginning of a function | Both |
@@ -116,7 +116,11 @@ input basic block mapping.
 
 ### Secondary Entry Points table
 The table is emitted for hot fragments only. It contains `NumSecEntryPoints`
-offsets denoting secondary entry points, delta encoded, implicitly starting at zero.
+offsets denoting secondary entry points and landing pads, delta encoded,
+implicitly starting at zero.
 | Entry | Encoding | Description |
 | ----- | -------- | ----------- |
-| `SecEntryPoint` | Delta, ULEB128 | Secondary entry point offset |
+| `SecEntryPoint` | Delta, ULEB128 | Secondary entry point offset with `LPENTRY` LSB bit |
+
+`LPENTRY` bit denotes whether a given offset is a landing pad block. If not set,
+the offset is a secondary entry point.

bolt/include/bolt/Core/BinaryFunction.h

@@ -908,6 +908,10 @@ class BinaryFunction {
     return BB && BB->getOffset() == Offset ? BB : nullptr;
   }
 
+  const BinaryBasicBlock *getBasicBlockAtOffset(uint64_t Offset) const {
+    return const_cast<BinaryFunction *>(this)->getBasicBlockAtOffset(Offset);
+  }
+
   /// Retrieve the landing pad BB associated with invoke instruction \p Invoke
   /// that is in \p BB. Return nullptr if none exists
   BinaryBasicBlock *getLandingPadBBFor(const BinaryBasicBlock &BB,

bolt/include/bolt/Profile/BoltAddressTranslation.h

@@ -182,7 +182,13 @@ class BoltAddressTranslation {
   /// translation map entry
   const static uint32_t BRANCHENTRY = 0x1;
 
+  /// Identifies a landing pad in secondary entry point map entry.
+  const static uint32_t LPENTRY = 0x1;
+
 public:
+  /// Returns whether a given \p Offset is a secondary entry point or a landing pad in function with address \p Address.
+  bool isSecondaryEntry(uint64_t Address, uint32_t Offset) const;
+
   /// Map basic block input offset to a basic block index and hash pair.
   class BBHashMapTy {
     struct EntryTy {

bolt/include/bolt/Profile/DataAggregator.h

@@ -266,7 +266,8 @@ class DataAggregator : public DataReader {
                      uint64_t Mispreds);
 
   /// Register a \p Branch.
-  bool doBranch(uint64_t From, uint64_t To, uint64_t Count, uint64_t Mispreds);
+  bool doBranch(uint64_t From, uint64_t To, uint64_t Count, uint64_t Mispreds,
+                bool IsPreagg);
 
   /// Register a trace between two LBR entries supplied in execution order.
   bool doTrace(const LBREntry &First, const LBREntry &Second,

bolt/lib/Profile/BoltAddressTranslation.cpp

@@ -86,21 +86,16 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
     if (Function.isIgnored() || (!BC.HasRelocations && !Function.isSimple()))
       continue;
 
-    uint32_t NumSecondaryEntryPoints = 0;
-    Function.forEachEntryPoint([&](uint64_t Offset, const MCSymbol *) {
-      if (!Offset)
-        return true;
-      ++NumSecondaryEntryPoints;
-      SecondaryEntryPointsMap[OutputAddress].push_back(Offset);
-      return true;
-    });
-
     LLVM_DEBUG(dbgs() << "Function name: " << Function.getPrintName() << "\n");
     LLVM_DEBUG(dbgs() << " Address reference: 0x"
                       << Twine::utohexstr(Function.getOutputAddress()) << "\n");
     LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBFHash(InputAddress)));
-    LLVM_DEBUG(dbgs() << " Secondary Entry Points: " << NumSecondaryEntryPoints
-                      << '\n');
+    LLVM_DEBUG({
+      uint32_t NumSecondaryEntryPoints = 0;
+      if (SecondaryEntryPointsMap.count(InputAddress))
+        NumSecondaryEntryPoints = SecondaryEntryPointsMap[InputAddress].size();
+      dbgs() << " Secondary Entry Points: " << NumSecondaryEntryPoints << '\n';
+    });
 
     MapTy Map;
     for (const BinaryBasicBlock *const BB :
@@ -206,10 +201,9 @@ void BoltAddressTranslation::writeMaps(uint64_t &PrevAddress, raw_ostream &OS) {
                       << Twine::utohexstr(Address) << ".\n");
     encodeULEB128(Address - PrevAddress, OS);
     PrevAddress = Address;
-    const uint32_t NumSecondaryEntryPoints =
-        SecondaryEntryPointsMap.count(Address)
-            ? SecondaryEntryPointsMap[Address].size()
-            : 0;
+    uint32_t NumSecondaryEntryPoints = 0;
+    if (SecondaryEntryPointsMap.count(HotInputAddress))
+      NumSecondaryEntryPoints = SecondaryEntryPointsMap[HotInputAddress].size();
     uint32_t Skew = 0;
     if (Cold) {
       auto HotEntryIt = llvm::lower_bound(HotFuncs, ColdPartSource[Address]);
@@ -281,7 +275,7 @@ void BoltAddressTranslation::writeMaps(uint64_t &PrevAddress, raw_ostream &OS) {
     if (!Cold && NumSecondaryEntryPoints) {
       LLVM_DEBUG(dbgs() << "Secondary entry points: ");
       // Secondary entry point offsets, delta-encoded
-      for (uint32_t Offset : SecondaryEntryPointsMap[Address]) {
+      for (uint32_t Offset : SecondaryEntryPointsMap[HotInputAddress]) {
         encodeULEB128(Offset - PrevOffset, OS);
         LLVM_DEBUG(dbgs() << formatv("{0:x} ", Offset));
         PrevOffset = Offset;
@@ -469,8 +463,12 @@ void BoltAddressTranslation::dump(raw_ostream &OS) const {
       const std::vector<uint32_t> &SecondaryEntryPoints =
           SecondaryEntryPointsIt->second;
       OS << SecondaryEntryPoints.size() << " secondary entry points:\n";
-      for (uint32_t EntryPointOffset : SecondaryEntryPoints)
-        OS << formatv("{0:x}\n", EntryPointOffset);
+      for (uint32_t EntryPointOffset : SecondaryEntryPoints) {
+        OS << formatv("{0:x}", EntryPointOffset >> 1);
+        if (EntryPointOffset & LPENTRY)
+          OS << " (lp)";
+        OS << '\n';
+      }
     }
     OS << "\n";
   }
@@ -582,14 +580,21 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) {
     // changed
     if (BF.isIgnored() || (!BC.HasRelocations && !BF.isSimple()))
       continue;
+    const uint64_t FuncAddress = BF.getAddress();
     // Prepare function and block hashes
-    FuncHashes.addEntry(BF.getAddress(), BF.computeHash());
+    FuncHashes.addEntry(FuncAddress, BF.computeHash());
     BF.computeBlockHashes();
-    BBHashMapTy &BBHashMap = getBBHashMap(BF.getAddress());
+    BBHashMapTy &BBHashMap = getBBHashMap(FuncAddress);
+    std::vector<uint32_t> SecondaryEntryPoints;
     // Set BF/BB metadata
-    for (const BinaryBasicBlock &BB : BF)
+    for (const BinaryBasicBlock &BB : BF) {
       BBHashMap.addEntry(BB.getInputOffset(), BB.getIndex(), BB.getHash());
-    NumBasicBlocksMap.emplace(BF.getAddress(), BF.size());
+      bool IsLandingPad = BB.isLandingPad();
+      if (IsLandingPad || BF.getSecondaryEntryPointSymbol(BB))
+        SecondaryEntryPoints.emplace_back(BB.getOffset() << 1 | IsLandingPad);
+    }
+    SecondaryEntryPointsMap.emplace(FuncAddress, SecondaryEntryPoints);
+    NumBasicBlocksMap.emplace(FuncAddress, BF.size());
   }
 }
 
@@ -599,13 +604,20 @@ BoltAddressTranslation::getSecondaryEntryPointId(uint64_t Address,
   auto FunctionIt = SecondaryEntryPointsMap.find(Address);
   if (FunctionIt == SecondaryEntryPointsMap.end())
     return 0;
-  const std::vector<uint32_t> &Offsets = FunctionIt->second;
-  auto OffsetIt = std::find(Offsets.begin(), Offsets.end(), Offset);
-  if (OffsetIt == Offsets.end())
-    return 0;
-  // Adding one here because main entry point is not stored in BAT, and
-  // enumeration for secondary entry points starts with 1.
-  return OffsetIt - Offsets.begin() + 1;
+  unsigned EntryPoints = 0;
+  // Note we need to scan the vector to get the entry point id because it
+  // contains both entry points and landing pads.
+  for (uint32_t Off : FunctionIt->second) {
+    // Skip landing pads.
+    if (Off & LPENTRY)
+      continue;
+    // Adding one here because main entry point is not stored in BAT, and
+    // enumeration for secondary entry points starts with 1.
+    if (Off >> 1 == Offset)
+      return EntryPoints + 1;
+    ++EntryPoints;
+  }
+  return 0;
 }
 
 std::pair<const BinaryFunction *, unsigned>
@@ -637,5 +649,16 @@ BoltAddressTranslation::translateSymbol(const BinaryContext &BC,
   return std::pair(ParentBF, SecondaryEntryId);
 }
 
+bool BoltAddressTranslation::isSecondaryEntry(uint64_t Address,
+                                              uint32_t Offset) const {
+  auto FunctionIt = SecondaryEntryPointsMap.find(Address);
+  if (FunctionIt == SecondaryEntryPointsMap.end())
+    return false;
+  const std::vector<uint32_t> &Offsets = FunctionIt->second;
+  uint64_t InputOffset = translate(Address, Offset, /*IsBranchSrc*/ false);
+  auto OffsetIt = llvm::lower_bound(Offsets, InputOffset << 1);
+  return OffsetIt != Offsets.end() && *OffsetIt >> 1 == InputOffset;
+}
+
 } // namespace bolt
 } // namespace llvm

bolt/lib/Profile/DataAggregator.cpp

@@ -778,42 +778,73 @@ bool DataAggregator::doInterBranch(BinaryFunction *FromFunc,
 }
 
 bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,
-                              uint64_t Mispreds) {
-  bool IsReturn = false;
-  auto handleAddress = [&](uint64_t &Addr, bool IsFrom) -> BinaryFunction * {
-    if (BinaryFunction *Func = getBinaryFunctionContainingAddress(Addr)) {
-      Addr -= Func->getAddress();
-      if (IsFrom) {
-        auto checkReturn = [&](auto MaybeInst) {
-          IsReturn = MaybeInst && BC->MIB->isReturn(*MaybeInst);
-        };
-        if (Func->hasInstructions())
-          checkReturn(Func->getInstructionAtOffset(Addr));
-        else
-          checkReturn(Func->disassembleInstructionAtOffset(Addr));
-      }
+                              uint64_t Mispreds, bool IsPreagg) {
+  // Returns whether \p Offset in \p Func contains a return instruction.
+  auto checkReturn = [&](const BinaryFunction &Func, const uint64_t Offset) {
+    auto isReturn = [&](auto MI) { return MI && BC->MIB->isReturn(*MI); };
+    return Func.hasInstructions()
+               ? isReturn(Func.getInstructionAtOffset(Offset))
+               : isReturn(Func.disassembleInstructionAtOffset(Offset));
+  };
 
-      if (BAT)
-        Addr = BAT->translate(Func->getAddress(), Addr, IsFrom);
+  // Returns whether \p Offset in \p Func may be a call continuation excluding
+  // entry points and landing pads.
+  auto checkCallCont = [&](const BinaryFunction &Func, const uint64_t Offset) {
+    // No call continuation at a function start.
+    if (!Offset)
+      return false;
 
-      if (BinaryFunction *ParentFunc = getBATParentFunction(*Func)) {
-        Func = ParentFunc;
-        if (IsFrom)
-          NumColdSamples += Count;
-      }
+    if (!Func.hasCFG())
+      return BAT && !BAT->isSecondaryEntry(Func.getAddress(), Offset);
 
-      return Func;
-    }
-    return nullptr;
+    // The offset should not be an entry point or a landing pad.
+    const BinaryBasicBlock *ContBB = Func.getBasicBlockAtOffset(Offset);
+    return ContBB && !ContBB->isEntryPoint() && !ContBB->isLandingPad();
   };
 
-  BinaryFunction *FromFunc = handleAddress(From, /*IsFrom=*/true);
+  // Mutates \p Addr to an offset into the containing function, performing BAT
+  // offset translation and parent lookup.
+  //
+  // Returns the containing function (or BAT parent) and whether the address
+  // corresponds to a return (if \p IsFrom) or a call continuation (otherwise).
+  auto handleAddress = [&](uint64_t &Addr, bool IsFrom) {
+    BinaryFunction *Func = getBinaryFunctionContainingAddress(Addr);
+    if (!Func)
+      return std::pair{Func, false};
+
+    Addr -= Func->getAddress();
+
+    bool IsRetOrCallCont =
+        IsFrom ? checkReturn(*Func, Addr) : checkCallCont(*Func, Addr);
+
+    if (BAT)
+      Addr = BAT->translate(Func->getAddress(), Addr, IsFrom);
+
+    BinaryFunction *ParentFunc = getBATParentFunction(*Func);
+    if (!ParentFunc)
+      return std::pair{Func, IsRetOrCallCont};
+
+    if (IsFrom)
+      NumColdSamples += Count;
+
+    return std::pair{ParentFunc, IsRetOrCallCont};
+  };
+
+  uint64_t ToOrig = To;
+  auto [FromFunc, IsReturn] = handleAddress(From, /*IsFrom=*/true);
+  auto [ToFunc, IsCallCont] = handleAddress(To, /*IsFrom=*/false);
+  if (!FromFunc && !ToFunc)
+    return false;
+
+  // Record call to continuation trace.
+  if (IsPreagg && FromFunc != ToFunc && (IsReturn || IsCallCont)) {
+    LBREntry First{ToOrig - 1, ToOrig - 1, false};
+    LBREntry Second{ToOrig, ToOrig, false};
+    return doTrace(First, Second, Count);
+  }
   // Ignore returns.
   if (IsReturn)
     return true;
-  BinaryFunction *ToFunc = handleAddress(To, /*IsFrom=*/false);
-  if (!FromFunc && !ToFunc)
-    return false;
 
   // Treat recursive control transfers as inter-branches.
   if (FromFunc == ToFunc && To != 0) {
@@ -830,10 +861,19 @@ bool DataAggregator::doTrace(const LBREntry &First, const LBREntry &Second,
   BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(Second.From);
   if (!FromFunc || !ToFunc) {
     LLVM_DEBUG({
-      dbgs() << "Out of range trace starting in " << FromFunc->getPrintName()
-             << formatv(" @ {0:x}", First.To - FromFunc->getAddress())
-             << " and ending in " << ToFunc->getPrintName()
-             << formatv(" @ {0:x}\n", Second.From - ToFunc->getAddress());
+      dbgs() << "Out of range trace starting in ";
+      if (FromFunc)
+        dbgs() << formatv("{0} @ {1:x}", *FromFunc,
+                          First.To - FromFunc->getAddress());
+      else
+        dbgs() << Twine::utohexstr(First.To);
+      dbgs() << " and ending in ";
+      if (ToFunc)
+        dbgs() << formatv("{0} @ {1:x}", *ToFunc,
+                          Second.From - ToFunc->getAddress());
+      else
+        dbgs() << Twine::utohexstr(Second.From);
+      dbgs() << '\n';
     });
     NumLongRangeTraces += Count;
     return false;
@@ -1620,7 +1660,8 @@ void DataAggregator::processBranchEvents() {
   for (const auto &AggrLBR : BranchLBRs) {
     const Trace &Loc = AggrLBR.first;
     const TakenBranchInfo &Info = AggrLBR.second;
-    doBranch(Loc.From, Loc.To, Info.TakenCount, Info.MispredCount);
+    doBranch(Loc.From, Loc.To, Info.TakenCount, Info.MispredCount,
+             /*IsPreagg*/ false);
   }
 }
 
@@ -1781,7 +1822,7 @@ void DataAggregator::processPreAggregated() {
     switch (AggrEntry.EntryType) {
     case AggregatedLBREntry::BRANCH:
       doBranch(AggrEntry.From.Offset, AggrEntry.To.Offset, AggrEntry.Count,
-               AggrEntry.Mispreds);
+               AggrEntry.Mispreds, /*IsPreagg=*/true);
       break;
     case AggregatedLBREntry::FT:
     case AggregatedLBREntry::FT_EXTERNAL_ORIGIN: {