[BOLT] Encode landing pads in BAT

bolt/docs/BAT.md

@@ -54,7 +54,7 @@ Functions table:
 |      table       |
 |                  |
 | Secondary entry  |
-|      points      |
+|  points and LPs  |
 |------------------|
 
 ```
@@ -80,7 +80,7 @@ Hot indices are delta encoded, implicitly starting at zero.
 | `HotIndex` | Delta, ULEB128 | Index of corresponding hot function in hot functions table | Cold |
 | `FuncHash` | 8b | Function hash for input function | Hot |
 | `NumBlocks` | ULEB128 | Number of basic blocks in the original function | Hot |
-| `NumSecEntryPoints` | ULEB128 | Number of secondary entry points in the original function | Hot |
+| `NumSecEntryPoints` | ULEB128 | Number of secondary entry points and landing pads in the original function | Hot |
 | `ColdInputSkew` | ULEB128 | Skew to apply to all input offsets | Cold |
 | `NumEntries` | ULEB128 | Number of address translation entries for a function | Both |
 | `EqualElems` | ULEB128 | Number of equal offsets in the beginning of a function | Both |
@@ -116,7 +116,11 @@ input basic block mapping.
 
 ### Secondary Entry Points table
 The table is emitted for hot fragments only. It contains `NumSecEntryPoints`
-offsets denoting secondary entry points, delta encoded, implicitly starting at zero.
+offsets denoting secondary entry points and landing pads, delta encoded,
+implicitly starting at zero.
 | Entry | Encoding | Description |
 | ----- | -------- | ----------- |
-| `SecEntryPoint` | Delta, ULEB128 | Secondary entry point offset |
+| `SecEntryPoint` | Delta, ULEB128 | Secondary entry point offset with `LPENTRY` LSB bit |
+
+`LPENTRY` bit denotes whether a given offset is a landing pad block. If not set,
+the offset is a secondary entry point.

bolt/include/bolt/Profile/BoltAddressTranslation.h

@@ -181,6 +181,9 @@ class BoltAddressTranslation {
   /// translation map entry
   const static uint32_t BRANCHENTRY = 0x1;
 
+  /// Identifies a landing pad in secondary entry point map entry.
+  const static uint32_t LPENTRY = 0x1;
+
 public:
   /// Map basic block input offset to a basic block index and hash pair.
   class BBHashMapTy {

bolt/lib/Profile/BoltAddressTranslation.cpp

@@ -86,21 +86,16 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
     if (Function.isIgnored() || (!BC.HasRelocations && !Function.isSimple()))
       continue;
 
-    uint32_t NumSecondaryEntryPoints = 0;
-    Function.forEachEntryPoint([&](uint64_t Offset, const MCSymbol *) {
-      if (!Offset)
-        return true;
-      ++NumSecondaryEntryPoints;
-      SecondaryEntryPointsMap[OutputAddress].push_back(Offset);
-      return true;
-    });
-
     LLVM_DEBUG(dbgs() << "Function name: " << Function.getPrintName() << "\n");
     LLVM_DEBUG(dbgs() << " Address reference: 0x"
                       << Twine::utohexstr(Function.getOutputAddress()) << "\n");
     LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBFHash(InputAddress)));
-    LLVM_DEBUG(dbgs() << " Secondary Entry Points: " << NumSecondaryEntryPoints
-                      << '\n');
+    LLVM_DEBUG({
+      uint32_t NumSecondaryEntryPoints = 0;
+      if (SecondaryEntryPointsMap.count(InputAddress))
+        NumSecondaryEntryPoints = SecondaryEntryPointsMap[InputAddress].size();
+      dbgs() << " Secondary Entry Points: " << NumSecondaryEntryPoints << '\n';
+    });
 
     MapTy Map;
     for (const BinaryBasicBlock *const BB :
@@ -207,10 +202,9 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
                       << Twine::utohexstr(Address) << ".\n");
     encodeULEB128(Address - PrevAddress, OS);
     PrevAddress = Address;
-    const uint32_t NumSecondaryEntryPoints =
-        SecondaryEntryPointsMap.count(Address)
-            ? SecondaryEntryPointsMap[Address].size()
-            : 0;
+    uint32_t NumSecondaryEntryPoints = 0;
+    if (SecondaryEntryPointsMap.count(HotInputAddress))
+      NumSecondaryEntryPoints = SecondaryEntryPointsMap[HotInputAddress].size();
     uint32_t Skew = 0;
     if (Cold) {
       auto HotEntryIt = Maps.find(ColdPartSource[Address]);
@@ -281,7 +275,7 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
     if (!Cold && NumSecondaryEntryPoints) {
       LLVM_DEBUG(dbgs() << "Secondary entry points: ");
       // Secondary entry point offsets, delta-encoded
-      for (uint32_t Offset : SecondaryEntryPointsMap[Address]) {
+      for (uint32_t Offset : SecondaryEntryPointsMap[HotInputAddress]) {
         encodeULEB128(Offset - PrevOffset, OS);
         LLVM_DEBUG(dbgs() << formatv("{0:x} ", Offset));
         PrevOffset = Offset;
@@ -471,8 +465,12 @@ void BoltAddressTranslation::dump(raw_ostream &OS) const {
       const std::vector<uint32_t> &SecondaryEntryPoints =
           SecondaryEntryPointsIt->second;
       OS << SecondaryEntryPoints.size() << " secondary entry points:\n";
-      for (uint32_t EntryPointOffset : SecondaryEntryPoints)
-        OS << formatv("{0:x}\n", EntryPointOffset);
+      for (uint32_t EntryPointOffset : SecondaryEntryPoints) {
+        OS << formatv("{0:x}", EntryPointOffset >> 1);
+        if (EntryPointOffset & LPENTRY)
+          OS << " (lp)";
+        OS << '\n';
+      }
     }
     OS << "\n";
   }
@@ -584,14 +582,21 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) {
     // changed
     if (BF.isIgnored() || (!BC.HasRelocations && !BF.isSimple()))
       continue;
+    const uint64_t FuncAddress = BF.getAddress();
     // Prepare function and block hashes
-    FuncHashes.addEntry(BF.getAddress(), BF.computeHash());
+    FuncHashes.addEntry(FuncAddress, BF.computeHash());
     BF.computeBlockHashes();
-    BBHashMapTy &BBHashMap = getBBHashMap(BF.getAddress());
+    BBHashMapTy &BBHashMap = getBBHashMap(FuncAddress);
+    std::vector<uint32_t> SecondaryEntryPoints;
     // Set BF/BB metadata
-    for (const BinaryBasicBlock &BB : BF)
+    for (const BinaryBasicBlock &BB : BF) {
       BBHashMap.addEntry(BB.getInputOffset(), BB.getIndex(), BB.getHash());
-    NumBasicBlocksMap.emplace(BF.getAddress(), BF.size());
+      bool IsLandingPad = BB.isLandingPad();
+      if (IsLandingPad || BF.getSecondaryEntryPointSymbol(BB))
+        SecondaryEntryPoints.emplace_back(BB.getOffset() << 1 | IsLandingPad);
+    }
+    SecondaryEntryPointsMap.emplace(FuncAddress, SecondaryEntryPoints);
+    NumBasicBlocksMap.emplace(FuncAddress, BF.size());
   }
 }
 
@@ -601,13 +606,20 @@ BoltAddressTranslation::getSecondaryEntryPointId(uint64_t Address,
   auto FunctionIt = SecondaryEntryPointsMap.find(Address);
   if (FunctionIt == SecondaryEntryPointsMap.end())
     return 0;
-  const std::vector<uint32_t> &Offsets = FunctionIt->second;
-  auto OffsetIt = std::find(Offsets.begin(), Offsets.end(), Offset);
-  if (OffsetIt == Offsets.end())
-    return 0;
-  // Adding one here because main entry point is not stored in BAT, and
-  // enumeration for secondary entry points starts with 1.
-  return OffsetIt - Offsets.begin() + 1;
+  unsigned EntryPoints = 0;
+  // Note we need to scan the vector to get the entry point id because it
+  // contains both entry points and landing pads.
+  for (uint32_t Off : FunctionIt->second) {
+    // Skip landing pads.
+    if (Off & LPENTRY)
+      continue;
+    // Adding one here because main entry point is not stored in BAT, and
+    // enumeration for secondary entry points starts with 1.
+    if (Off >> 1 == Offset)
+      return EntryPoints + 1;
+    ++EntryPoints;
+  }
+  return 0;
 }
 
 std::pair<const BinaryFunction *, unsigned>

bolt/test/X86/callcont-fallthru.s

@@ -22,9 +22,15 @@
 # RUN:   --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK3
 
 ## Check fallthrough to a landing pad case.
-# RUN: llvm-bolt %t.exe --pa -p %t.pa4 -o %t.out \
+# RUN: llvm-bolt %t.exe --pa -p %t.pa4 -o %t.out --enable-bat \
 # RUN:   --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK4
 
+## Check that a landing pad is emitted in BAT
+# RUN: llvm-bat-dump %t.out --dump-all | FileCheck %s --check-prefix=CHECK-BAT
+
+# CHECK-BAT:      1 secondary entry points:
+# CHECK-BAT-NEXT: 0x38 (lp)
+
   .globl foo
   .type foo, %function
 foo: