Skip to content

Commit 09eb8f3

Browse files
joaosaffranjoaosaffran
committed
Merge branch 'main' into frontend/adding-new-tokens
2 parents 11770c5 + e28a559 commit 09eb8f3

File tree

430 files changed

+49297
-41055
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

430 files changed

+49297
-41055
lines changed

‎bolt/include/bolt/Core/BinaryFunction.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -192,9 +192,6 @@ class BinaryFunction {
192192

193193
mutable MCSymbol *FunctionConstantIslandLabel{nullptr};
194194
mutable MCSymbol *FunctionColdConstantIslandLabel{nullptr};
195-
196-
// Returns constant island alignment
197-
uint16_t getAlignment() const { return sizeof(uint64_t); }
198195
};
199196

200197
static constexpr uint64_t COUNT_NO_PROFILE =
@@ -2114,9 +2111,7 @@ class BinaryFunction {
21142111
return *std::prev(CodeIter) <= *DataIter;
21152112
}
21162113

2117-
uint16_t getConstantIslandAlignment() const {
2118-
return Islands ? Islands->getAlignment() : 1;
2119-
}
2114+
uint16_t getConstantIslandAlignment() const;
21202115

21212116
/// If there is a constant island in the range [StartOffset, EndOffset),
21222117
/// return its address.
@@ -2168,6 +2163,11 @@ class BinaryFunction {
21682163
return Islands && !Islands->DataOffsets.empty();
21692164
}
21702165

2166+
/// Return true if the whole function is a constant island.
2167+
bool isDataObject() const {
2168+
return Islands && Islands->CodeOffsets.size() == 0;
2169+
}
2170+
21712171
bool isStartOfConstantIsland(uint64_t Offset) const {
21722172
return hasConstantIsland() && Islands->DataOffsets.count(Offset);
21732173
}

‎bolt/include/bolt/Core/MCPlusBuilder.h‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ class raw_ostream;
5151

5252
namespace bolt {
5353
class BinaryBasicBlock;
54+
class BinaryContext;
5455
class BinaryFunction;
5556

5657
/// Different types of indirect branches encountered during disassembly.
@@ -530,10 +531,15 @@ class MCPlusBuilder {
530531
return 0;
531532
}
532533

534+
/// Create a helper function to increment counter for Instrumentation
535+
virtual void createInstrCounterIncrFunc(BinaryContext &BC) {
536+
llvm_unreachable("not implemented");
537+
}
538+
533539
/// Create increment contents of target by 1 for Instrumentation
534-
virtual InstructionListType
535-
createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
536-
unsigned CodePointerSize) const {
540+
virtual InstructionListType createInstrIncMemory(const MCSymbol *Target,
541+
MCContext *Ctx, bool IsLeaf,
542+
unsigned CodePointerSize) {
537543
llvm_unreachable("not implemented");
538544
return InstructionListType();
539545
}

‎bolt/lib/Core/BinaryFunction.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,33 @@ BinaryFunction::getBasicBlockContainingOffset(uint64_t Offset) {
284284
return (Offset < BB->getOffset() + BB->getOriginalSize()) ? BB : nullptr;
285285
}
286286

287+
uint16_t BinaryFunction::getConstantIslandAlignment() const {
288+
if (Islands == nullptr)
289+
return 1;
290+
291+
// For constant island inside a function, the default 8-byte alignment is
292+
// probably good enough.
293+
const uint16_t DefaultAlignment = sizeof(uint64_t);
294+
if (!isDataObject())
295+
return DefaultAlignment;
296+
297+
// If the constant island itself is a binary function, get its alignment
298+
// based on its size, original address, and its owning section's alignment.
299+
const uint64_t MaxAlignment =
300+
std::min(uint64_t(1) << llvm::countr_zero(getAddress()),
301+
OriginSection->getAlignment());
302+
const uint64_t MinAlignment =
303+
std::max((uint64_t)DefaultAlignment,
304+
uint64_t(1) << (63 - llvm::countl_zero(getSize())));
305+
uint64_t Alignment = std::min(MinAlignment, MaxAlignment);
306+
if (Alignment >> 16) {
307+
BC.errs() << "BOLT-ERROR: the constant island's alignment is too big: 0x"
308+
<< Twine::utohexstr(Alignment) << "\n";
309+
exit(1);
310+
}
311+
return (uint16_t)Alignment;
312+
}
313+
287314
void BinaryFunction::markUnreachableBlocks() {
288315
std::stack<BinaryBasicBlock *> Stack;
289316

‎bolt/lib/Passes/Instrumentation.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,8 @@ void Instrumentation::createAuxiliaryFunctions(BinaryContext &BC) {
753753
createSimpleFunction("__bolt_fini_trampoline",
754754
BC.MIB->createReturnInstructionList(BC.Ctx.get()));
755755
}
756+
if (BC.isAArch64())
757+
BC.MIB->createInstrCounterIncrFunc(BC);
756758
}
757759
}
758760

‎bolt/lib/Rewrite/RewriteInstance.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1312,7 +1312,9 @@ void RewriteInstance::discoverFileObjects() {
13121312

13131313
// Annotate functions with code/data markers in AArch64
13141314
for (auto &[Address, Type] : MarkerSymbols) {
1315-
auto *BF = BC->getBinaryFunctionContainingAddress(Address, true, true);
1315+
auto *BF = BC->getBinaryFunctionContainingAddress(Address,
1316+
/*CheckPastEnd*/ false,
1317+
/*UseMaxSize*/ true);
13161318

13171319
if (!BF) {
13181320
// Stray marker

‎bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp‎

Lines changed: 130 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "llvm/MC/MCInstrInfo.h"
2727
#include "llvm/MC/MCRegister.h"
2828
#include "llvm/MC/MCRegisterInfo.h"
29+
#include "llvm/Support/CommandLine.h"
2930
#include "llvm/Support/DataExtractor.h"
3031
#include "llvm/Support/Debug.h"
3132
#include "llvm/Support/ErrorHandling.h"
@@ -35,6 +36,15 @@
3536
using namespace llvm;
3637
using namespace bolt;
3738

39+
namespace opts {
40+
extern cl::OptionCategory BoltInstrCategory;
41+
static cl::opt<bool> NoLSEAtomics(
42+
"no-lse-atomics",
43+
cl::desc("generate instrumentation code sequence without using LSE atomic "
44+
"instruction"),
45+
cl::init(false), cl::Optional, cl::cat(BoltInstrCategory));
46+
} // namespace opts
47+
3848
namespace {
3949

4050
static void getSystemFlag(MCInst &Inst, MCPhysReg RegName) {
@@ -106,7 +116,7 @@ static void storeReg(MCInst &Inst, MCPhysReg From, MCPhysReg To) {
106116
}
107117

108118
static void atomicAdd(MCInst &Inst, MCPhysReg RegTo, MCPhysReg RegCnt) {
109-
// NOTE: Supports only ARM with LSE extension
119+
assert(!opts::NoLSEAtomics && "Supports only ARM with LSE extension");
110120
Inst.setOpcode(AArch64::LDADDX);
111121
Inst.clear();
112122
Inst.addOperand(MCOperand::createReg(AArch64::XZR));
@@ -135,6 +145,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
135145
public:
136146
using MCPlusBuilder::MCPlusBuilder;
137147

148+
BinaryFunction *InstrCounterIncrFunc{nullptr};
149+
138150
std::unique_ptr<MCSymbolizer>
139151
createTargetSymbolizer(BinaryFunction &Function,
140152
bool CreateNewSymbols) const override {
@@ -2513,22 +2525,129 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
25132525
return Insts;
25142526
}
25152527

2516-
InstructionListType
2517-
createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
2518-
unsigned CodePointerSize) const override {
2528+
// Instrumentation code sequence using LSE atomic instruction has a total of
2529+
// 6 instructions:
2530+
//
2531+
// stp x0, x1, [sp, #-0x10]!
2532+
// adrp x0, page_address(counter)
2533+
// add x0, x0, page_offset(counter)
2534+
// mov x1, #0x1
2535+
// stadd x1, [x0]
2536+
// ldp x0, x1, [sp], #0x10
2537+
//
2538+
// Instrumentation code sequence without using LSE atomic instruction has
2539+
// 8 instructions at instrumentation place, with 6 instructions in the helper:
2540+
//
2541+
// stp x0, x30, [sp, #-0x10]!
2542+
// stp x1, x2, [sp, #-0x10]!
2543+
// adrp x0, page_address(counter)
2544+
// add x0, x0, page_offset(counter)
2545+
// adrp x1, page_address(helper)
2546+
// add x1, x1, page_offset(helper)
2547+
// blr x1
2548+
// ldp x0, x30, [sp], #0x10
2549+
//
2550+
// <helper>:
2551+
// ldaxr x1, [x0]
2552+
// add x1, x1, #0x1
2553+
// stlxr w2, x1, [x0]
2554+
// cbnz w2, <helper>
2555+
// ldp x1, x2, [sp], #0x10
2556+
// ret
2557+
2558+
void createInstrCounterIncrFunc(BinaryContext &BC) override {
2559+
assert(InstrCounterIncrFunc == nullptr &&
2560+
"helper function of counter increment for instrumentation "
2561+
"has already been created");
2562+
2563+
if (!opts::NoLSEAtomics)
2564+
return;
2565+
2566+
MCContext *Ctx = BC.Ctx.get();
2567+
InstrCounterIncrFunc = BC.createInjectedBinaryFunction(
2568+
"__bolt_instr_counter_incr", /*IsSimple*/ false);
2569+
std::vector<std::unique_ptr<BinaryBasicBlock>> BBs;
2570+
2571+
BBs.emplace_back(InstrCounterIncrFunc->createBasicBlock());
2572+
InstructionListType Instrs(4);
2573+
Instrs[0].setOpcode(AArch64::LDAXRX);
2574+
Instrs[0].clear();
2575+
Instrs[0].addOperand(MCOperand::createReg(AArch64::X1));
2576+
Instrs[0].addOperand(MCOperand::createReg(AArch64::X0));
2577+
Instrs[1].setOpcode(AArch64::ADDXri);
2578+
Instrs[1].clear();
2579+
Instrs[1].addOperand(MCOperand::createReg(AArch64::X1));
2580+
Instrs[1].addOperand(MCOperand::createReg(AArch64::X1));
2581+
Instrs[1].addOperand(MCOperand::createImm(1));
2582+
Instrs[1].addOperand(MCOperand::createImm(0));
2583+
Instrs[2].setOpcode(AArch64::STLXRX);
2584+
Instrs[2].clear();
2585+
Instrs[2].addOperand(MCOperand::createReg(AArch64::W2));
2586+
Instrs[2].addOperand(MCOperand::createReg(AArch64::X1));
2587+
Instrs[2].addOperand(MCOperand::createReg(AArch64::X0));
2588+
Instrs[3].setOpcode(AArch64::CBNZW);
2589+
Instrs[3].clear();
2590+
Instrs[3].addOperand(MCOperand::createReg(AArch64::W2));
2591+
Instrs[3].addOperand(MCOperand::createExpr(
2592+
MCSymbolRefExpr::create(BBs.back()->getLabel(), *Ctx)));
2593+
BBs.back()->addInstructions(Instrs.begin(), Instrs.end());
2594+
BBs.back()->setCFIState(0);
2595+
2596+
BBs.emplace_back(InstrCounterIncrFunc->createBasicBlock());
2597+
InstructionListType InstrsEpilog(2);
2598+
createPopRegisters(InstrsEpilog[0], AArch64::X1, AArch64::X2);
2599+
createReturn(InstrsEpilog[1]);
2600+
BBs.back()->addInstructions(InstrsEpilog.begin(), InstrsEpilog.end());
2601+
BBs.back()->setCFIState(0);
2602+
2603+
BBs[0]->addSuccessor(BBs[0].get());
2604+
BBs[0]->addSuccessor(BBs[1].get());
2605+
2606+
InstrCounterIncrFunc->insertBasicBlocks(nullptr, std::move(BBs),
2607+
/*UpdateLayout*/ true,
2608+
/*UpdateCFIState*/ false);
2609+
InstrCounterIncrFunc->updateState(BinaryFunction::State::CFG_Finalized);
2610+
2611+
LLVM_DEBUG({
2612+
dbgs() << "BOLT-DEBUG: instrumentation counter increment helper:\n";
2613+
InstrCounterIncrFunc->dump();
2614+
});
2615+
}
2616+
2617+
InstructionListType createInstrIncMemory(const MCSymbol *Target,
2618+
MCContext *Ctx, bool IsLeaf,
2619+
unsigned CodePointerSize) override {
25192620
unsigned int I = 0;
2520-
InstructionListType Instrs(6);
2621+
InstructionListType Instrs(opts::NoLSEAtomics ? 8 : 6);
2622+
2623+
if (opts::NoLSEAtomics) {
2624+
createPushRegisters(Instrs[I++], AArch64::X0, AArch64::LR);
2625+
createPushRegisters(Instrs[I++], AArch64::X1, AArch64::X2);
2626+
} else {
2627+
createPushRegisters(Instrs[I++], AArch64::X0, AArch64::X1);
2628+
}
25212629

2522-
createPushRegisters(Instrs[I++], AArch64::X0, AArch64::X1);
25232630
InstructionListType Addr = materializeAddress(Target, Ctx, AArch64::X0);
25242631
assert(Addr.size() == 2 && "Invalid Addr size");
25252632
std::copy(Addr.begin(), Addr.end(), Instrs.begin() + I);
25262633
I += Addr.size();
2527-
InstructionListType Insts = createIncMemory(AArch64::X0, AArch64::X1);
2528-
assert(Insts.size() == 2 && "Invalid Insts size");
2529-
std::copy(Insts.begin(), Insts.end(), Instrs.begin() + I);
2530-
I += Insts.size();
2531-
createPopRegisters(Instrs[I++], AArch64::X0, AArch64::X1);
2634+
2635+
if (opts::NoLSEAtomics) {
2636+
const MCSymbol *Helper = InstrCounterIncrFunc->getSymbol();
2637+
InstructionListType HelperAddr =
2638+
materializeAddress(Helper, Ctx, AArch64::X1);
2639+
assert(HelperAddr.size() == 2 && "Invalid HelperAddr size");
2640+
std::copy(HelperAddr.begin(), HelperAddr.end(), Instrs.begin() + I);
2641+
I += HelperAddr.size();
2642+
createIndirectCallInst(Instrs[I++], /*IsTailCall*/ false, AArch64::X1);
2643+
} else {
2644+
InstructionListType Insts = createIncMemory(AArch64::X0, AArch64::X1);
2645+
assert(Insts.size() == 2 && "Invalid Insts size");
2646+
std::copy(Insts.begin(), Insts.end(), Instrs.begin() + I);
2647+
I += Insts.size();
2648+
}
2649+
createPopRegisters(Instrs[I++], AArch64::X0,
2650+
opts::NoLSEAtomics ? AArch64::LR : AArch64::X1);
25322651
return Instrs;
25332652
}
25342653

‎bolt/lib/Target/AArch64/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ add_llvm_library(LLVMBOLTTargetAArch64
2828
AArch64CommonTableGen
2929
)
3030

31-
target_link_libraries(LLVMBOLTTargetAArch64 PRIVATE LLVMBOLTCore)
31+
target_link_libraries(LLVMBOLTTargetAArch64 PRIVATE LLVMBOLTCore LLVMBOLTUtils)
3232

3333
include_directories(
3434
${LLVM_MAIN_SRC_DIR}/lib/Target/AArch64

‎bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -626,9 +626,9 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
626626
return Insts;
627627
}
628628

629-
InstructionListType
630-
createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
631-
unsigned CodePointerSize) const override {
629+
InstructionListType createInstrIncMemory(const MCSymbol *Target,
630+
MCContext *Ctx, bool IsLeaf,
631+
unsigned CodePointerSize) override {
632632
// We need 2 scratch registers: one for the target address (x10), and one
633633
// for the increment value (x11).
634634
// addi sp, sp, -16

‎bolt/lib/Target/X86/X86MCPlusBuilder.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3053,9 +3053,9 @@ class X86MCPlusBuilder : public MCPlusBuilder {
30533053
Inst.clear();
30543054
}
30553055

3056-
InstructionListType
3057-
createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
3058-
unsigned CodePointerSize) const override {
3056+
InstructionListType createInstrIncMemory(const MCSymbol *Target,
3057+
MCContext *Ctx, bool IsLeaf,
3058+
unsigned CodePointerSize) override {
30593059
InstructionListType Instrs(IsLeaf ? 13 : 11);
30603060
unsigned int I = 0;
30613061

0 commit comments

Comments
 (0)