diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 817ad288aa34b..3b42c36541acd 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -54,7 +54,7 @@ Functions table: | table | | | | Secondary entry | -| points | +| points and LPs | |------------------| ``` @@ -80,7 +80,7 @@ Hot indices are delta encoded, implicitly starting at zero. | `HotIndex` | Delta, ULEB128 | Index of corresponding hot function in hot functions table | Cold | | `FuncHash` | 8b | Function hash for input function | Hot | | `NumBlocks` | ULEB128 | Number of basic blocks in the original function | Hot | -| `NumSecEntryPoints` | ULEB128 | Number of secondary entry points in the original function | Hot | +| `NumSecEntryPoints` | ULEB128 | Number of secondary entry points and landing pads in the original function | Hot | | `ColdInputSkew` | ULEB128 | Skew to apply to all input offsets | Cold | | `NumEntries` | ULEB128 | Number of address translation entries for a function | Both | | `EqualElems` | ULEB128 | Number of equal offsets in the beginning of a function | Both | @@ -116,7 +116,11 @@ input basic block mapping. ### Secondary Entry Points table The table is emitted for hot fragments only. It contains `NumSecEntryPoints` -offsets denoting secondary entry points, delta encoded, implicitly starting at zero. +offsets denoting secondary entry points and landing pads, delta encoded, +implicitly starting at zero. | Entry | Encoding | Description | | ----- | -------- | ----------- | -| `SecEntryPoint` | Delta, ULEB128 | Secondary entry point offset | +| `SecEntryPoint` | Delta, ULEB128 | Secondary entry point offset with `LPENTRY` LSB bit | + +`LPENTRY` bit denotes whether a given offset is a landing pad block. If not set, +the offset is a secondary entry point. diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 65b9ba874368f..62367ca3aebdc 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -181,6 +181,9 @@ class BoltAddressTranslation { /// translation map entry const static uint32_t BRANCHENTRY = 0x1; + /// Identifies a landing pad in secondary entry point map entry. + const static uint32_t LPENTRY = 0x1; + public: /// Map basic block input offset to a basic block index and hash pair. class BBHashMapTy { diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index ec7e303c0f52e..9ce62052653e3 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -86,21 +86,16 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { if (Function.isIgnored() || (!BC.HasRelocations && !Function.isSimple())) continue; - uint32_t NumSecondaryEntryPoints = 0; - Function.forEachEntryPoint([&](uint64_t Offset, const MCSymbol *) { - if (!Offset) - return true; - ++NumSecondaryEntryPoints; - SecondaryEntryPointsMap[OutputAddress].push_back(Offset); - return true; - }); - LLVM_DEBUG(dbgs() << "Function name: " << Function.getPrintName() << "\n"); LLVM_DEBUG(dbgs() << " Address reference: 0x" << Twine::utohexstr(Function.getOutputAddress()) << "\n"); LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBFHash(InputAddress))); - LLVM_DEBUG(dbgs() << " Secondary Entry Points: " << NumSecondaryEntryPoints - << '\n'); + LLVM_DEBUG({ + uint32_t NumSecondaryEntryPoints = 0; + if (SecondaryEntryPointsMap.count(InputAddress)) + NumSecondaryEntryPoints = SecondaryEntryPointsMap[InputAddress].size(); + dbgs() << " Secondary Entry Points: " << NumSecondaryEntryPoints << '\n'; + }); MapTy Map; for (const BinaryBasicBlock *const BB : @@ -207,10 +202,9 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, << Twine::utohexstr(Address) << ".\n"); encodeULEB128(Address - PrevAddress, OS); PrevAddress = Address; - const uint32_t NumSecondaryEntryPoints = - SecondaryEntryPointsMap.count(Address) - ? SecondaryEntryPointsMap[Address].size() - : 0; + uint32_t NumSecondaryEntryPoints = 0; + if (SecondaryEntryPointsMap.count(HotInputAddress)) + NumSecondaryEntryPoints = SecondaryEntryPointsMap[HotInputAddress].size(); uint32_t Skew = 0; if (Cold) { auto HotEntryIt = Maps.find(ColdPartSource[Address]); @@ -281,7 +275,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, if (!Cold && NumSecondaryEntryPoints) { LLVM_DEBUG(dbgs() << "Secondary entry points: "); // Secondary entry point offsets, delta-encoded - for (uint32_t Offset : SecondaryEntryPointsMap[Address]) { + for (uint32_t Offset : SecondaryEntryPointsMap[HotInputAddress]) { encodeULEB128(Offset - PrevOffset, OS); LLVM_DEBUG(dbgs() << formatv("{0:x} ", Offset)); PrevOffset = Offset; @@ -471,8 +465,12 @@ void BoltAddressTranslation::dump(raw_ostream &OS) const { const std::vector &SecondaryEntryPoints = SecondaryEntryPointsIt->second; OS << SecondaryEntryPoints.size() << " secondary entry points:\n"; - for (uint32_t EntryPointOffset : SecondaryEntryPoints) - OS << formatv("{0:x}\n", EntryPointOffset); + for (uint32_t EntryPointOffset : SecondaryEntryPoints) { + OS << formatv("{0:x}", EntryPointOffset >> 1); + if (EntryPointOffset & LPENTRY) + OS << " (lp)"; + OS << '\n'; + } } OS << "\n"; } @@ -584,14 +582,21 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { // changed if (BF.isIgnored() || (!BC.HasRelocations && !BF.isSimple())) continue; + const uint64_t FuncAddress = BF.getAddress(); // Prepare function and block hashes - FuncHashes.addEntry(BF.getAddress(), BF.computeHash()); + FuncHashes.addEntry(FuncAddress, BF.computeHash()); BF.computeBlockHashes(); - BBHashMapTy &BBHashMap = getBBHashMap(BF.getAddress()); + BBHashMapTy &BBHashMap = getBBHashMap(FuncAddress); + std::vector SecondaryEntryPoints; // Set BF/BB metadata - for (const BinaryBasicBlock &BB : BF) + for (const BinaryBasicBlock &BB : BF) { BBHashMap.addEntry(BB.getInputOffset(), BB.getIndex(), BB.getHash()); - NumBasicBlocksMap.emplace(BF.getAddress(), BF.size()); + bool IsLandingPad = BB.isLandingPad(); + if (IsLandingPad || BF.getSecondaryEntryPointSymbol(BB)) + SecondaryEntryPoints.emplace_back(BB.getOffset() << 1 | IsLandingPad); + } + SecondaryEntryPointsMap.emplace(FuncAddress, SecondaryEntryPoints); + NumBasicBlocksMap.emplace(FuncAddress, BF.size()); } } @@ -601,13 +606,20 @@ BoltAddressTranslation::getSecondaryEntryPointId(uint64_t Address, auto FunctionIt = SecondaryEntryPointsMap.find(Address); if (FunctionIt == SecondaryEntryPointsMap.end()) return 0; - const std::vector &Offsets = FunctionIt->second; - auto OffsetIt = std::find(Offsets.begin(), Offsets.end(), Offset); - if (OffsetIt == Offsets.end()) - return 0; - // Adding one here because main entry point is not stored in BAT, and - // enumeration for secondary entry points starts with 1. - return OffsetIt - Offsets.begin() + 1; + unsigned EntryPoints = 0; + // Note we need to scan the vector to get the entry point id because it + // contains both entry points and landing pads. + for (uint32_t Off : FunctionIt->second) { + // Skip landing pads. + if (Off & LPENTRY) + continue; + // Adding one here because main entry point is not stored in BAT, and + // enumeration for secondary entry points starts with 1. + if (Off >> 1 == Offset) + return EntryPoints + 1; + ++EntryPoints; + } + return 0; } std::pair diff --git a/bolt/test/X86/callcont-fallthru.s b/bolt/test/X86/callcont-fallthru.s index 31a7910d7fa3f..641beb79ecf2a 100644 --- a/bolt/test/X86/callcont-fallthru.s +++ b/bolt/test/X86/callcont-fallthru.s @@ -22,9 +22,15 @@ # RUN: --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK3 ## Check fallthrough to a landing pad case. -# RUN: llvm-bolt %t.exe --pa -p %t.pa4 -o %t.out \ +# RUN: llvm-bolt %t.exe --pa -p %t.pa4 -o %t.out --enable-bat \ # RUN: --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK4 +## Check that a landing pad is emitted in BAT +# RUN: llvm-bat-dump %t.out --dump-all | FileCheck %s --check-prefix=CHECK-BAT + +# CHECK-BAT: 1 secondary entry points: +# CHECK-BAT-NEXT: 0x38 (lp) + .globl foo .type foo, %function foo: