From a8180a8f4d2da467b44da52dd18f1638c8932232 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Mon, 31 Mar 2025 21:25:26 -0700 Subject: [PATCH 1/5] [RFC][BPF] Support Jump Table NOTE: We probably need cpu v5 or other flags to enable this feature. We can add it later when necessary. - Generate all jump tables in a single section named .jumptables. - Represent each jump table as a symbol: - value points to an offset within .jumptables; - size encodes jump table size in bytes. - Indirect jump is a gotox instruction: - dst register is an index within the table; - accompanied by a R_BPF_64_64 relocation pointing to a jump table symbol. clang -S: .LJTI0_0: .reloc 0, FK_SecRel_8, .BPF.JT.0.0 gotox r1 goto LBB0_2 LBB0_4: ... .section .jumptables,"",@progbits .L0_0_set_4 = ((LBB0_4-.LBPF.JX.0.0)>>3)-1 .L0_0_set_2 = ((LBB0_2-.LBPF.JX.0.0)>>3)-1 ... .BPF.JT.0.0: .long .L0_0_set_4 .long .L0_0_set_2 ... llvm-readelf -r --sections --symbols: Section Headers: [Nr] Name Type Address Off Size ES Flg Lk Inf Al ... [ 4] .jumptables PROGBITS 0000000000000000 000118 000100 00 0 0 1 ... Relocation section '.rel.text' at offset 0x2a8 contains 2 entries: Offset Info Type Symbol's Value Symbol's Name 0000000000000010 0000000300000001 R_BPF_64_64 0000000000000000 .BPF.JT.0.0 ... Symbol table '.symtab' contains 6 entries: Num: Value Size Type Bind Vis Ndx Name ... 2: 0000000000000000 112 FUNC GLOBAL DEFAULT 2 foo 3: 0000000000000000 128 NOTYPE GLOBAL DEFAULT 4 .BPF.JT.0.0 ... llvm-objdump -Sdr: 0000000000000000 : ... 2: gotox r1 0000000000000010: R_BPF_64_64 .BPF.JT.0.0 An option -bpf-min-jump-table-entries is implemented to control the minimum number of entries to use a jump table on BPF. The default value 4, but it can be changed with the following clang option clang ... -mllvm -bpf-min-jump-table-entries=6 where the number of jump table cases needs to be >= 6 in order to use jump table. --- llvm/include/llvm/CodeGen/AsmPrinter.h | 3 +- .../lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 1 + llvm/lib/Target/BPF/BPFAsmPrinter.cpp | 77 +++++++++++++++++++ llvm/lib/Target/BPF/BPFISelLowering.cpp | 42 +++++++++- llvm/lib/Target/BPF/BPFISelLowering.h | 29 ++++++- llvm/lib/Target/BPF/BPFInstrInfo.cpp | 12 +++ llvm/lib/Target/BPF/BPFInstrInfo.h | 3 + llvm/lib/Target/BPF/BPFInstrInfo.td | 30 ++++++++ llvm/lib/Target/BPF/BPFMCInstLower.cpp | 28 +++++++ llvm/lib/Target/BPF/BPFMCInstLower.h | 2 + .../BPF/BPFTargetLoweringObjectFile.cpp | 19 +++++ .../Target/BPF/BPFTargetLoweringObjectFile.h | 25 ++++++ llvm/lib/Target/BPF/BPFTargetMachine.cpp | 3 +- llvm/lib/Target/BPF/CMakeLists.txt | 1 + .../Target/BPF/MCTargetDesc/BPFAsmBackend.cpp | 13 ++++ 15 files changed, 283 insertions(+), 5 deletions(-) create mode 100644 llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp create mode 100644 llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index faab2503ced50..db328f4b2003e 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -663,7 +663,8 @@ class LLVM_ABI AsmPrinter : public MachineFunctionPass { MCSymbol *GetExternalSymbolSymbol(const Twine &Sym) const; /// Return the symbol for the specified jump table entry. - MCSymbol *GetJTISymbol(unsigned JTID, bool isLinkerPrivate = false) const; + virtual MCSymbol *GetJTISymbol(unsigned JTID, + bool isLinkerPrivate = false) const; /// Return the symbol for the specified jump table .set /// FIXME: privatize to AsmPrinter. diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index a347794a9a30c..d96f403d2f814 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -234,6 +234,7 @@ struct BPFOperand : public MCParsedAsmOperand { .Case("callx", true) .Case("goto", true) .Case("gotol", true) + .Case("gotox", true) .Case("may_goto", true) .Case("*", true) .Case("exit", true) diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp index e3843e0e112e2..d99b7bec4600a 100644 --- a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp +++ b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp @@ -17,18 +17,24 @@ #include "BTFDebug.h" #include "MCTargetDesc/BPFInstPrinter.h" #include "TargetInfo/BPFTargetInfo.h" +#include "llvm/BinaryFormat/ELF.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/TargetLowering.h" #include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetLoweringObjectFile.h" using namespace llvm; #define DEBUG_TYPE "asm-printer" @@ -49,6 +55,9 @@ class BPFAsmPrinter : public AsmPrinter { const char *ExtraCode, raw_ostream &O) override; void emitInstruction(const MachineInstr *MI) override; + virtual MCSymbol *GetJTISymbol(unsigned JTID, + bool isLinkerPrivate = false) const override; + virtual void emitJumpTableInfo() override; static char ID; @@ -150,6 +159,74 @@ void BPFAsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); } +MCSymbol *BPFAsmPrinter::GetJTISymbol(unsigned JTID, + bool isLinkerPrivate) const { + SmallString<60> Name; + raw_svector_ostream(Name) + << "BPF.JT." << MF->getFunctionNumber() << '.' << JTID; + MCSymbol *S = OutContext.getOrCreateSymbol(Name); + if (auto *ES = dyn_cast(S)) + ES->setBinding(ELF::STB_GLOBAL); + return S; +} + +void BPFAsmPrinter::emitJumpTableInfo() { + const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); + if (!MJTI) + return; + + const std::vector &JT = MJTI->getJumpTables(); + if (JT.empty()) + return; + + const TargetLoweringObjectFile &TLOF = getObjFileLowering(); + const Function &F = MF->getFunction(); + MCSection *JTS = TLOF.getSectionForJumpTable(F, TM); + assert(MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32); + unsigned EntrySize = MJTI->getEntrySize(getDataLayout()); + OutStreamer->switchSection(JTS); + for (unsigned JTI = 0; JTI < JT.size(); JTI++) { + ArrayRef JTBBs = JT[JTI].MBBs; + if (JTBBs.empty()) + continue; + + SmallPtrSet EmittedSets; + const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); + const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext); + for (const MachineBasicBlock *MBB : JTBBs) { + if (!EmittedSets.insert(MBB).second) + continue; + + // Offset from gotox to target basic block expressed in number + // of instructions, e.g.: + // + // .L0_0_set_4 = ((LBB0_4 - .LBPF.JX.0.0) >> 3) - 1 + const MCExpr *LHS = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext); + OutStreamer->emitAssignment( + GetJTSetSymbol(JTI, MBB->getNumber()), + MCBinaryExpr::createSub( + MCBinaryExpr::createAShr( + MCBinaryExpr::createSub(LHS, Base, OutContext), + MCConstantExpr::create(3, OutContext), OutContext), + MCConstantExpr::create(1, OutContext), OutContext)); + } + // BPF.JT.0.0: + // .long .L0_0_set_4 + // .long .L0_0_set_2 + // ... + // .size BPF.JT.0.0, 128 + MCSymbol *JTStart = GetJTISymbol(JTI); + OutStreamer->emitLabel(JTStart); + for (const MachineBasicBlock *MBB : JTBBs) { + MCSymbol *SetSymbol = GetJTSetSymbol(JTI, MBB->getNumber()); + const MCExpr *V = MCSymbolRefExpr::create(SetSymbol, OutContext); + OutStreamer->emitValue(V, EntrySize); + } + const MCExpr *JTSize = MCConstantExpr::create(JTBBs.size() * 4, OutContext); + OutStreamer->emitELFSize(JTStart, JTSize); + } +} + char BPFAsmPrinter::ID = 0; INITIALIZE_PASS(BPFAsmPrinter, "bpf-asm-printer", "BPF Assembly Printer", false, diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp index f4f414d192df0..5af456ea2e054 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.cpp +++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp @@ -18,6 +18,7 @@ #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/CodeGen/ValueTypes.h" @@ -25,6 +26,7 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/Module.h" +#include "llvm/MC/MCAsmInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -38,6 +40,10 @@ static cl::opt BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order")); +static cl::opt BPFMinimumJumpTableEntries( + "bpf-min-jump-table-entries", cl::init(4), cl::Hidden, + cl::desc("Set minimum number of entries to use a jump table on BPF")); + static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val = {}) { std::string Str; @@ -66,8 +72,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, setStackPointerRegisterToSaveRestore(BPF::R11); setOperationAction(ISD::BR_CC, MVT::i64, Custom); - setOperationAction(ISD::BR_JT, MVT::Other, Expand); - setOperationAction(ISD::BRIND, MVT::Other, Expand); + setOperationAction(ISD::BR_JT, MVT::Other, Custom); setOperationAction(ISD::BRCOND, MVT::Other, Expand); setOperationAction(ISD::TRAP, MVT::Other, Custom); @@ -159,6 +164,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, setBooleanContents(ZeroOrOneBooleanContent); setMaxAtomicSizeInBitsSupported(64); + setMinimumJumpTableEntries(BPFMinimumJumpTableEntries); // Function alignments setMinFunctionAlignment(Align(8)); @@ -332,6 +338,8 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { return LowerATOMIC_LOAD_STORE(Op, DAG); case ISD::TRAP: return LowerTRAP(Op, DAG); + case ISD::BR_JT: + return LowerBR_JT(Op, DAG); } } @@ -780,6 +788,16 @@ SDValue BPFTargetLowering::LowerTRAP(SDValue Op, SelectionDAG &DAG) const { return LowerCall(CLI, InVals); } +SDValue BPFTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const { + SDValue Chain = Op->getOperand(0); + SDValue Table = Op->getOperand(1); + SDValue Index = Op->getOperand(2); + JumpTableSDNode *JT = cast(Table); + SDLoc DL(Op); + SDValue TargetJT = DAG.getTargetJumpTable(JT->getIndex(), MVT::i32); + return DAG.getNode(BPFISD::BPF_BR_JT, DL, MVT::Other, Chain, TargetJT, Index); +} + const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const { switch ((BPFISD::NodeType)Opcode) { case BPFISD::FIRST_NUMBER: @@ -796,6 +814,8 @@ const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const { return "BPFISD::Wrapper"; case BPFISD::MEMCPY: return "BPFISD::MEMCPY"; + case BPFISD::BPF_BR_JT: + return "BPFISD::BPF_BR_JT"; } return nullptr; } @@ -1069,3 +1089,21 @@ bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL, return true; } + +MCSymbol *BPFTargetLowering::getJXAnchorSymbol(const MachineFunction *MF, + unsigned JTI) { + const MCAsmInfo *MAI = MF->getContext().getAsmInfo(); + SmallString<60> Name; + raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix() << "BPF.JX." + << MF->getFunctionNumber() << '.' << JTI; + return MF->getContext().getOrCreateSymbol(Name); +} + +unsigned BPFTargetLowering::getJumpTableEncoding() const { + return MachineJumpTableInfo::EK_LabelDifference32; +} + +const MCExpr *BPFTargetLowering::getPICJumpTableRelocBaseExpr( + const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const { + return MCSymbolRefExpr::create(getJXAnchorSymbol(MF, JTI), Ctx); +} diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h index 8f60261c10e9e..f533cd03b0c61 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.h +++ b/llvm/lib/Target/BPF/BPFISelLowering.h @@ -28,7 +28,8 @@ enum NodeType : unsigned { SELECT_CC, BR_CC, Wrapper, - MEMCPY + MEMCPY, + BPF_BR_JT, }; } @@ -66,6 +67,31 @@ class BPFTargetLowering : public TargetLowering { MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override; + // Always emit EK_LabelDifference32, computed as difference between + // JX instruction location and target basic block label. + virtual unsigned getJumpTableEncoding() const override; + + // This is a label for JX instructions, used for jump table offsets + // computation, e.g.: + // + // .LBPF.JX.0.0: <------- this is the anchor + // .reloc 0, FK_SecRel_8, BPF.JT.0.0 + // gotox r1 + // ... + // .section .jumptables,"",@progbits + // .L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1 + // ... + // BPF.JT.0.0: <------- JT definition + // .long .L0_0_set_7 + // ... + static MCSymbol *getJXAnchorSymbol(const MachineFunction *MF, unsigned JTI); + + // Refers to a symbol returned by getJXAnchorSymbol(), used by + // AsmPrinter::emitJumpTableInfo() to define the .L0_0_set_7 etc above. + virtual const MCExpr * + getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, + MCContext &Ctx) const override; + private: // Control Instruction Selection Features bool HasAlu32; @@ -81,6 +107,7 @@ class BPFTargetLowering : public TargetLowering { SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerTRAP(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const; template SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.cpp b/llvm/lib/Target/BPF/BPFInstrInfo.cpp index 70bc163615f61..e9f65b0517f95 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.cpp +++ b/llvm/lib/Target/BPF/BPFInstrInfo.cpp @@ -181,6 +181,10 @@ bool BPFInstrInfo::analyzeBranch(MachineBasicBlock &MBB, if (!isUnpredicatedTerminator(*I)) break; + // If a JX insn, we're done. + if (I->getOpcode() == BPF::JX) + break; + // A terminator that isn't a branch can't easily be handled // by this analysis. if (!I->isBranch()) @@ -259,3 +263,11 @@ unsigned BPFInstrInfo::removeBranch(MachineBasicBlock &MBB, return Count; } + +int BPFInstrInfo::getJumpTableIndex(const MachineInstr &MI) const { + if (MI.getOpcode() != BPF::JX) + return -1; + const MachineOperand &MO = MI.getOperand(1); + assert(MO.isJTI() && "JX operand #0 should be isJTI"); + return MO.getIndex(); +} diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.h b/llvm/lib/Target/BPF/BPFInstrInfo.h index d8bbad44e314e..d88e37975980a 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.h +++ b/llvm/lib/Target/BPF/BPFInstrInfo.h @@ -58,6 +58,9 @@ class BPFInstrInfo : public BPFGenInstrInfo { MachineBasicBlock *FBB, ArrayRef Cond, const DebugLoc &DL, int *BytesAdded = nullptr) const override; + + int getJumpTableIndex(const MachineInstr &MI) const override; + private: void expandMEMCPY(MachineBasicBlock::iterator) const; diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index b21f1a0eee3b0..581e49419c37f 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -31,6 +31,8 @@ def SDT_BPFMEMCPY : SDTypeProfile<0, 4, [SDTCisVT<0, i64>, SDTCisVT<1, i64>, SDTCisVT<2, i64>, SDTCisVT<3, i64>]>; +def SDT_BPFBrJt : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, // jump table + SDTCisVT<1, i64>]>; // index def BPFcall : SDNode<"BPFISD::CALL", SDT_BPFCall, [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, @@ -49,6 +51,9 @@ def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>; def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY, [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore, SDNPMayLoad]>; +def BPFBrJt : SDNode<"BPFISD::BPF_BR_JT", SDT_BPFBrJt, + [SDNPHasChain]>; + def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">; def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">; def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">; @@ -183,6 +188,15 @@ class TYPE_LD_ST mode, bits<2> size, let Inst{60-59} = size; } +// For indirect jump +class TYPE_IND_JMP op, bits<1> srctype, + dag outs, dag ins, string asmstr, list pattern> + : InstBPF { + + let Inst{63-60} = op; + let Inst{59} = srctype; +} + // jump instructions class JMP_RR : TYPE_ALU_JMP let BPFClass = BPF_JMP; } +class JMP_IND Pattern> + : TYPE_ALU_JMP { + bits<4> dst; + + let Inst{51-48} = dst; + let BPFClass = BPF_JMP; +} + class JMP_JCOND Pattern> : TYPE_ALU_JMP; defm JSLE : J; def JCOND : JMP_JCOND; + +let isIndirectBranch = 1 in { + def JX : JMP_IND; +} } // ALU instructions diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/llvm/lib/Target/BPF/BPFMCInstLower.cpp index 040a1fb750702..8d18ff58a92e0 100644 --- a/llvm/lib/Target/BPF/BPFMCInstLower.cpp +++ b/llvm/lib/Target/BPF/BPFMCInstLower.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "BPFMCInstLower.h" +#include "BPFISelLowering.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineInstr.h" @@ -19,6 +20,7 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" +#include "llvm/MC/MCStreamer.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -44,6 +46,29 @@ MCOperand BPFMCInstLower::LowerSymbolOperand(const MachineOperand &MO, return MCOperand::createExpr(Expr); } +MCOperand BPFMCInstLower::LowerJTIOperand(const MachineInstr &MI, + const MachineOperand &MO, + int JTI) const { + // Emit relocation entry referencing jump table symbol plus a label + // for JX anchor, e.g.: + // + // .LBPF.JX.0.0: + // .reloc 0, FK_SecRel_8, BPF.JT.0.0 + // gotox r1 + assert((MI.getOpcode() == BPF::JX) && + "Jump Table Index operands are expected only for JX instructions"); + const MachineFunction *MF = MI.getMF(); + Printer.OutStreamer->emitLabel(BPFTargetLowering::getJXAnchorSymbol(MF, JTI)); + MCSymbol *JT = Printer.GetJTISymbol(JTI); + const MCExpr *Zero = MCConstantExpr::create(0, Ctx); + Printer.OutStreamer->emitRelocDirective(*Zero, "FK_SecRel_8", + MCSymbolRefExpr::create(JT, Ctx), {}, + *Ctx.getSubtargetInfo()); + // JTI parameter is used only to emit relocation and is not a part + // of JX instruction encoding, so this operand is not really used. + return MCOperand::createImm(JTI); +} + void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { OutMI.setOpcode(MI->getOpcode()); @@ -77,6 +102,9 @@ void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { case MachineOperand::MO_ConstantPoolIndex: MCOp = LowerSymbolOperand(MO, Printer.GetCPISymbol(MO.getIndex())); break; + case MachineOperand::MO_JumpTableIndex: + MCOp = LowerJTIOperand(*MI, MO, MO.getIndex()); + break; } OutMI.addOperand(MCOp); diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.h b/llvm/lib/Target/BPF/BPFMCInstLower.h index 4bd0f1f0bf1cf..927e9dfaf8c68 100644 --- a/llvm/lib/Target/BPF/BPFMCInstLower.h +++ b/llvm/lib/Target/BPF/BPFMCInstLower.h @@ -32,6 +32,8 @@ class LLVM_LIBRARY_VISIBILITY BPFMCInstLower { void Lower(const MachineInstr *MI, MCInst &OutMI) const; MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const; + MCOperand LowerJTIOperand(const MachineInstr &MI, const MachineOperand &MO, + int JTI) const; MCSymbol *GetGlobalAddressSymbol(const MachineOperand &MO) const; MCSymbol *GetExternalSymbolSymbol(const MachineOperand &MO) const; diff --git a/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp new file mode 100644 index 0000000000000..997f09870bad6 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp @@ -0,0 +1,19 @@ +//===------------------ BPFTargetLoweringObjectFile.cpp -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "BPFTargetLoweringObjectFile.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCSectionELF.h" + +using namespace llvm; + +MCSection *BPFTargetLoweringObjectFileELF::getSectionForJumpTable( + const Function &F, const TargetMachine &TM, + const MachineJumpTableEntry *JTE) const { + return getContext().getELFSection(".jumptables", ELF::SHT_PROGBITS, 0); +} diff --git a/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h new file mode 100644 index 0000000000000..f3064c0c8cb8a --- /dev/null +++ b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h @@ -0,0 +1,25 @@ +//===============- BPFTargetLoweringObjectFile.h -*- C++ -*-================// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE +#define LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE + +#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" +#include "llvm/Target/TargetLoweringObjectFile.h" + +namespace llvm { +class BPFTargetLoweringObjectFileELF : public TargetLoweringObjectFileELF { + +public: + virtual MCSection * + getSectionForJumpTable(const Function &F, const TargetMachine &TM, + const MachineJumpTableEntry *JTE) const override; +}; +} // namespace llvm + +#endif // LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 527a480354571..d538b6fe11675 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -12,6 +12,7 @@ #include "BPFTargetMachine.h" #include "BPF.h" +#include "BPFTargetLoweringObjectFile.h" #include "BPFTargetTransformInfo.h" #include "MCTargetDesc/BPFMCAsmInfo.h" #include "TargetInfo/BPFTargetInfo.h" @@ -80,7 +81,7 @@ BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT, : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options, getEffectiveRelocModel(RM), getEffectiveCodeModel(CM, CodeModel::Small), OL), - TLOF(std::make_unique()), + TLOF(std::make_unique()), Subtarget(TT, std::string(CPU), std::string(FS), *this) { if (!DisableCheckUnreachable) { this->Options.TrapUnreachable = true; diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index eade4cacb7100..3678f1335ca36 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -37,6 +37,7 @@ add_llvm_target(BPFCodeGen BPFRegisterInfo.cpp BPFSelectionDAGInfo.cpp BPFSubtarget.cpp + BPFTargetLoweringObjectFile.cpp BPFTargetMachine.cpp BPFMIPeephole.cpp BPFMIChecking.cpp diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp index 958790d49d087..7165a88fb17e7 100644 --- a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp +++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp @@ -34,6 +34,7 @@ class BPFAsmBackend : public MCAsmBackend { createObjectTargetWriter() const override; MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override; + std::optional getFixupKind(StringRef Name) const override; bool writeNopData(raw_ostream &OS, uint64_t Count, const MCSubtargetInfo *STI) const override; @@ -54,6 +55,18 @@ MCFixupKindInfo BPFAsmBackend::getFixupKindInfo(MCFixupKind Kind) const { return Infos[Kind - FirstTargetFixupKind]; } +std::optional BPFAsmBackend::getFixupKind(StringRef Name) const { + if (Name == "FK_SecRel_8") + return FK_SecRel_8; + if (Name == "FK_BPF_PCRel_4") + return BPF::FK_BPF_PCRel_4; + if (Name == "FK_Data_8") + return FK_Data_8; + if (Name == "FK_Data_4") + return FK_Data_4; + return std::nullopt; +} + bool BPFAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, const MCSubtargetInfo *STI) const { if ((Count % 8) != 0) From 024f351f9c5680e471e1cbbd1b47715890b6711b Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Thu, 10 Jul 2025 14:14:29 -0700 Subject: [PATCH 2/5] [BPF] Don't insert unreachable 'goto' after JX instructions Update BPFInstrInfo::analyzeBranch() to comply with TargetInstrInfo::analyzeBranch() requirements for JX instruction: if branch instruction can't be categorized as a conditional with true/false branches -- return true. Because of this bug MachineBlockPlacement transformation inserted an additional unreachabe jump after JX, e.g.: bb.1.entry: ... JX killed $r1, %jump-table.0 JMP %bb.2 Additionally, isNotDuplicable annotation is necessary to avoid machine level transformations creating several JX instruction copies. Such copies would refer to the same jump table and would make it not possible to calculate jump offsets inside the table. Files triggering such duplication are present in kernel selftests. --- llvm/lib/Target/BPF/BPFInstrInfo.cpp | 7 ++++--- llvm/lib/Target/BPF/BPFInstrInfo.td | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.cpp b/llvm/lib/Target/BPF/BPFInstrInfo.cpp index e9f65b0517f95..6275995ce9ba2 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.cpp +++ b/llvm/lib/Target/BPF/BPFInstrInfo.cpp @@ -181,9 +181,10 @@ bool BPFInstrInfo::analyzeBranch(MachineBasicBlock &MBB, if (!isUnpredicatedTerminator(*I)) break; - // If a JX insn, we're done. - if (I->getOpcode() == BPF::JX) - break; + // From base method doc: ... returning true if it cannot be understood ... + // Indirect branch has multiple destinations and no true/false concepts. + if (I->isIndirectBranch()) + return true; // A terminator that isn't a branch can't easily be handled // by this analysis. diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index 581e49419c37f..83dccb132c519 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -308,7 +308,7 @@ defm JSLE : J; def JCOND : JMP_JCOND; -let isIndirectBranch = 1 in { +let isIndirectBranch = 1, isBarrier = 1, isNotDuplicable = 1 in { def JX : JMP_IND; } } From a6fecc9c1e8ed869ab0444c447d9523a2b3566d0 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Thu, 10 Jul 2025 17:41:30 -0700 Subject: [PATCH 3/5] [BPF] add test cases for jump tables - one testing a general structure of the generated code; - another testing that several jump tables within the same functions are generated independently. --- llvm/test/CodeGen/BPF/jump-table-multi.ll | 78 ++++++++++++++++++++ llvm/test/CodeGen/BPF/jump-table-simple.ll | 86 ++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 llvm/test/CodeGen/BPF/jump-table-multi.ll create mode 100644 llvm/test/CodeGen/BPF/jump-table-simple.ll diff --git a/llvm/test/CodeGen/BPF/jump-table-multi.ll b/llvm/test/CodeGen/BPF/jump-table-multi.ll new file mode 100644 index 0000000000000..4f8b9ad428804 --- /dev/null +++ b/llvm/test/CodeGen/BPF/jump-table-multi.ll @@ -0,0 +1,78 @@ +; RUN: llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < %s | FileCheck %s + +; Check that two jump tables of different size are generated + +define i64 @foo(i64 %v1, i64 %v2) { +; CHECK: .LBPF.JX.0.0: +; CHECK-NEXT: .reloc 0, FK_SecRel_8, BPF.JT.0.0 +; CHECK-NEXT: gotox r1 + +; CHECK: .LBPF.JX.0.1: +; CHECK-NEXT: .reloc 0, FK_SecRel_8, BPF.JT.0.1 +; CHECK-NEXT: gotox r2 + +; CHECK: .section .jumptables,"",@progbits + +; CHECK-NEXT: [[m1:.*]] = (({{.*}}-.LBPF.JX.0.0)>>3)-1 +; CHECK-NEXT: [[m2:.*]] = (({{.*}}-.LBPF.JX.0.0)>>3)-1 +; CHECK-NEXT: BPF.JT.0.0: +; CHECK-NEXT: .long [[m1]] +; CHECK-NEXT: .long [[m2]] +; CHECK-NEXT: .size BPF.JT.0.0, 8 + +; CHECK-NEXT: [[m1:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1 +; CHECK-NEXT: [[m2:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1 +; CHECK-NEXT: [[m3:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1 +; CHECK-NEXT: BPF.JT.0.1: +; CHECK-NEXT: .long [[m1]] +; CHECK-NEXT: .long [[m2]] +; CHECK-NEXT: .long [[m3]] +; CHECK-NEXT: .size BPF.JT.0.1, 12 + +entry: + switch i64 %v1, label %sw.default [ + i64 0, label %sw.bb1 + i64 1, label %sw.bb2 + ] + +sw.bb1: + br label %sw.epilog + +sw.bb2: + br label %sw.epilog + +sw.default: + br label %sw.epilog + +sw.epilog: + %ret = phi i64 [ 42, %sw.default ], [ 3, %sw.bb1 ], [ 5, %sw.bb2 ] + switch i64 %v2, label %sw.default.1 [ + i64 0, label %sw.bb1.1 + i64 1, label %sw.bb2.1 + i64 2, label %sw.bb3.1 + ] + +sw.bb1.1: + br label %sw.epilog.1 + +sw.bb2.1: + br label %sw.epilog.1 + +sw.bb3.1: + br label %sw.epilog.1 + +sw.default.1: + br label %sw.epilog.1 + +sw.epilog.1: + %ret.1 = phi i64 [ 42, %sw.default.1 ], [ 3, %sw.bb1.1 ], [ 5, %sw.bb2.1 ], [ 7, %sw.bb3.1 ] + %ret.2 = add i64 %ret, %ret.1 + ret i64 %ret.2 +} + +!llvm.module.flags = !{!0, !1} +!llvm.ident = !{!2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 7, !"frame-pointer", i32 2} +!2 = !{!"clang some version"} diff --git a/llvm/test/CodeGen/BPF/jump-table-simple.ll b/llvm/test/CodeGen/BPF/jump-table-simple.ll new file mode 100644 index 0000000000000..8c048223dbc0b --- /dev/null +++ b/llvm/test/CodeGen/BPF/jump-table-simple.ll @@ -0,0 +1,86 @@ +; Checks generated using command: +; +; llvm/utils/update_test_body.py llvm/test/CodeGen/BPF/jump-table-simple.ll +; +; RUN: rm -rf %t && split-file %s %t && cd %t +; RUN: llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < test.ll | FileCheck %s +; +; Check general program structure generated for a jump table + +.ifdef GEN +;--- test.ll +define i64 @foo(i64 %v) { +entry: + switch i64 %v, label %sw.default [ + i64 0, label %sw.epilog + i64 1, label %sw.bb1 + i64 2, label %sw.bb1 + i64 3, label %sw.bb2 + ] + +sw.bb1: + br label %sw.epilog + +sw.bb2: + br label %sw.epilog + +sw.default: + br label %sw.epilog + +sw.epilog: + %ret = phi i64 [ 42, %sw.default ], [ 3, %sw.bb1 ], [ 5, %sw.bb2 ], [ 7, %entry ] + ret i64 %ret +} + +!llvm.module.flags = !{!0, !1} +!llvm.ident = !{!2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 7, !"frame-pointer", i32 2} +!2 = !{!"clang some version"} + +;--- gen +echo "" +echo "; Generated checks follow" +echo ";" +llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < test.ll \ + | awk '/# -- End function/ {p=0} /@function/ {p=1} p {print "; CHECK" ": " $0}' +.endif + +; Generated checks follow +; +; CHECK: .type foo,@function +; CHECK: foo: # @foo +; CHECK: .cfi_startproc +; CHECK: # %bb.0: # %entry +; CHECK: if r1 > 3 goto LBB0_5 +; CHECK: # %bb.1: # %entry +; CHECK: .LBPF.JX.0.0: +; CHECK: .reloc 0, FK_SecRel_8, BPF.JT.0.0 +; CHECK: gotox r1 +; CHECK: LBB0_3: # %sw.bb1 +; CHECK: r0 = 3 +; CHECK: goto LBB0_6 +; CHECK: LBB0_2: +; CHECK: r0 = 7 +; CHECK: goto LBB0_6 +; CHECK: LBB0_4: # %sw.bb2 +; CHECK: r0 = 5 +; CHECK: goto LBB0_6 +; CHECK: LBB0_5: # %sw.default +; CHECK: r0 = 42 +; CHECK: LBB0_6: # %sw.epilog +; CHECK: exit +; CHECK: .Lfunc_end0: +; CHECK: .size foo, .Lfunc_end0-foo +; CHECK: .cfi_endproc +; CHECK: .section .jumptables,"",@progbits +; CHECK: .L0_0_set_2 = ((LBB0_2-.LBPF.JX.0.0)>>3)-1 +; CHECK: .L0_0_set_3 = ((LBB0_3-.LBPF.JX.0.0)>>3)-1 +; CHECK: .L0_0_set_4 = ((LBB0_4-.LBPF.JX.0.0)>>3)-1 +; CHECK: BPF.JT.0.0: +; CHECK: .long .L0_0_set_2 +; CHECK: .long .L0_0_set_3 +; CHECK: .long .L0_0_set_3 +; CHECK: .long .L0_0_set_4 +; CHECK: .size BPF.JT.0.0, 16 From 9aec188a0a916a37cec809e02c0cdcb7369ffdaf Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Thu, 10 Jul 2025 21:27:55 -0700 Subject: [PATCH 4/5] [BPF] generate gotox only for cpuv4 Coincidentally this fixes two test failures: - LLVM :: CodeGen/BPF/CORE/offset-reloc-fieldinfo-2-bpfeb.ll - LLVM :: CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll These tests invoke llc with -mcpuv1 and have a switch statement in the IR. Both tests failed with assertion in SelectionDAGLegalize::LegalizeOp(): for (const SDValue &Op : Node->op_values()) assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && "Unexpected illegal type!"); At the moment of the failure: Op.getOpcode() == BPFISD::BPF_BR_JT The error happened because one of the BPFBrJt parameters has i32 type: def SDT_BPFBrJt : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, // jump table SDTCisVT<1, i64>]>; // index def BPFBrJt : SDNode<"BPFISD::BPF_BR_JT", SDT_BPFBrJt, [SDNPHasChain]>; --- llvm/lib/Target/BPF/BPFISelLowering.cpp | 4 +++- llvm/lib/Target/BPF/BPFSubtarget.cpp | 4 ++++ llvm/lib/Target/BPF/BPFSubtarget.h | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp index 5af456ea2e054..742690238aa8e 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.cpp +++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp @@ -72,8 +72,10 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, setStackPointerRegisterToSaveRestore(BPF::R11); setOperationAction(ISD::BR_CC, MVT::i64, Custom); - setOperationAction(ISD::BR_JT, MVT::Other, Custom); setOperationAction(ISD::BRCOND, MVT::Other, Expand); + LegalizeAction IndirectBrAction = STI.hasGotox() ? Custom : Expand; + setOperationAction(ISD::BR_JT, MVT::Other, IndirectBrAction); + setOperationAction(ISD::BRIND, MVT::Other, IndirectBrAction); setOperationAction(ISD::TRAP, MVT::Other, Custom); diff --git a/llvm/lib/Target/BPF/BPFSubtarget.cpp b/llvm/lib/Target/BPF/BPFSubtarget.cpp index 4167547680b12..a11aa6933147c 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.cpp +++ b/llvm/lib/Target/BPF/BPFSubtarget.cpp @@ -43,6 +43,8 @@ static cl::opt static cl::opt Disable_load_acq_store_rel( "disable-load-acq-store-rel", cl::Hidden, cl::init(false), cl::desc("Disable load-acquire and store-release insns")); +static cl::opt Disable_gotox("disable-gotox", cl::Hidden, cl::init(false), + cl::desc("Disable gotox insn")); void BPFSubtarget::anchor() {} @@ -66,6 +68,7 @@ void BPFSubtarget::initializeEnvironment() { HasGotol = false; HasStoreImm = false; HasLoadAcqStoreRel = false; + HasGotox = false; } void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { @@ -96,6 +99,7 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { HasGotol = !Disable_gotol; HasStoreImm = !Disable_StoreImm; HasLoadAcqStoreRel = !Disable_load_acq_store_rel; + HasGotox = !Disable_gotox; return; } } diff --git a/llvm/lib/Target/BPF/BPFSubtarget.h b/llvm/lib/Target/BPF/BPFSubtarget.h index aed2211265e23..e870dfdc85ec9 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.h +++ b/llvm/lib/Target/BPF/BPFSubtarget.h @@ -65,7 +65,7 @@ class BPFSubtarget : public BPFGenSubtargetInfo { // whether cpu v4 insns are enabled. bool HasLdsx, HasMovsx, HasBswap, HasSdivSmod, HasGotol, HasStoreImm, - HasLoadAcqStoreRel; + HasLoadAcqStoreRel, HasGotox; std::unique_ptr CallLoweringInfo; std::unique_ptr InstSelector; @@ -94,6 +94,7 @@ class BPFSubtarget : public BPFGenSubtargetInfo { bool hasGotol() const { return HasGotol; } bool hasStoreImm() const { return HasStoreImm; } bool hasLoadAcqStoreRel() const { return HasLoadAcqStoreRel; } + bool hasGotox() const { return HasGotox; } bool isLittleEndian() const { return IsLittleEndian; } From ddc6c1f26e128fd8810180be70b4d00b844a1b61 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Thu, 10 Jul 2025 23:56:32 -0700 Subject: [PATCH 5/5] [BPF] consolidate jump table emit code in BPFAsmPrinter The requirement to emit jump table entries as offsets measured in instructions, e.g. as follows: .L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1 Makes it impossible to use generic AsmPrinter::emitJumpTableInfo() function. Merge request used this generic function before (and incorrect offsets were generated). This generic function required two overloads: - AsmPrinter::GetJTISymbol() - TargetLowering::getPICJumpTableRelocBaseExpr() Now all jump table emission logic is located in the BPFAsmPrinter::emitJumpTableInfo(), which does not require above overloads. Hence, remove the overloads and move corresponding code to BPFAsmPrinter to keep it in one place. --- llvm/include/llvm/CodeGen/AsmPrinter.h | 3 +- llvm/lib/Target/BPF/BPFAsmPrinter.cpp | 59 +++++++++++-------------- llvm/lib/Target/BPF/BPFAsmPrinter.h | 44 ++++++++++++++++++ llvm/lib/Target/BPF/BPFISelLowering.cpp | 14 ------ llvm/lib/Target/BPF/BPFISelLowering.h | 21 --------- llvm/lib/Target/BPF/BPFMCInstLower.cpp | 6 +-- llvm/lib/Target/BPF/BPFMCInstLower.h | 6 +-- 7 files changed, 77 insertions(+), 76 deletions(-) create mode 100644 llvm/lib/Target/BPF/BPFAsmPrinter.h diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index db328f4b2003e..faab2503ced50 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -663,8 +663,7 @@ class LLVM_ABI AsmPrinter : public MachineFunctionPass { MCSymbol *GetExternalSymbolSymbol(const Twine &Sym) const; /// Return the symbol for the specified jump table entry. - virtual MCSymbol *GetJTISymbol(unsigned JTID, - bool isLinkerPrivate = false) const; + MCSymbol *GetJTISymbol(unsigned JTID, bool isLinkerPrivate = false) const; /// Return the symbol for the specified jump table .set /// FIXME: privatize to AsmPrinter. diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp index d99b7bec4600a..a6d8976a734dc 100644 --- a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp +++ b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp @@ -11,6 +11,7 @@ // //===----------------------------------------------------------------------===// +#include "BPFAsmPrinter.h" #include "BPF.h" #include "BPFInstrInfo.h" #include "BPFMCInstLower.h" @@ -39,33 +40,6 @@ using namespace llvm; #define DEBUG_TYPE "asm-printer" -namespace { -class BPFAsmPrinter : public AsmPrinter { -public: - explicit BPFAsmPrinter(TargetMachine &TM, - std::unique_ptr Streamer) - : AsmPrinter(TM, std::move(Streamer), ID), BTF(nullptr) {} - - StringRef getPassName() const override { return "BPF Assembly Printer"; } - bool doInitialization(Module &M) override; - void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O); - bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, - const char *ExtraCode, raw_ostream &O) override; - bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, - const char *ExtraCode, raw_ostream &O) override; - - void emitInstruction(const MachineInstr *MI) override; - virtual MCSymbol *GetJTISymbol(unsigned JTID, - bool isLinkerPrivate = false) const override; - virtual void emitJumpTableInfo() override; - - static char ID; - -private: - BTFDebug *BTF; -}; -} // namespace - bool BPFAsmPrinter::doInitialization(Module &M) { AsmPrinter::doInitialization(M); @@ -159,11 +133,31 @@ void BPFAsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); } -MCSymbol *BPFAsmPrinter::GetJTISymbol(unsigned JTID, - bool isLinkerPrivate) const { +// This is a label for JX instructions, used for jump table offsets +// computation, e.g.: +// +// .LBPF.JX.0.0: <------- this is the anchor +// .reloc 0, FK_SecRel_8, BPF.JT.0.0 +// gotox r1 +// ... +// .section .jumptables,"",@progbits +// .L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1 +// ... +// BPF.JT.0.0: <------- JT definition +// .long .L0_0_set_7 +// ... +MCSymbol *BPFAsmPrinter::getJXAnchorSymbol(unsigned JTI) { + const MCAsmInfo *MAI = MF->getContext().getAsmInfo(); + SmallString<60> Name; + raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix() << "BPF.JX." + << MF->getFunctionNumber() << '.' << JTI; + return MF->getContext().getOrCreateSymbol(Name); +} + +MCSymbol *BPFAsmPrinter::getJTPublicSymbol(unsigned JTI) { SmallString<60> Name; raw_svector_ostream(Name) - << "BPF.JT." << MF->getFunctionNumber() << '.' << JTID; + << "BPF.JT." << MF->getFunctionNumber() << '.' << JTI; MCSymbol *S = OutContext.getOrCreateSymbol(Name); if (auto *ES = dyn_cast(S)) ES->setBinding(ELF::STB_GLOBAL); @@ -191,8 +185,7 @@ void BPFAsmPrinter::emitJumpTableInfo() { continue; SmallPtrSet EmittedSets; - const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); - const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext); + auto *Base = MCSymbolRefExpr::create(getJXAnchorSymbol(JTI), OutContext); for (const MachineBasicBlock *MBB : JTBBs) { if (!EmittedSets.insert(MBB).second) continue; @@ -215,7 +208,7 @@ void BPFAsmPrinter::emitJumpTableInfo() { // .long .L0_0_set_2 // ... // .size BPF.JT.0.0, 128 - MCSymbol *JTStart = GetJTISymbol(JTI); + MCSymbol *JTStart = getJTPublicSymbol(JTI); OutStreamer->emitLabel(JTStart); for (const MachineBasicBlock *MBB : JTBBs) { MCSymbol *SetSymbol = GetJTSetSymbol(JTI, MBB->getNumber()); diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.h b/llvm/lib/Target/BPF/BPFAsmPrinter.h new file mode 100644 index 0000000000000..d46d789274a9e --- /dev/null +++ b/llvm/lib/Target/BPF/BPFAsmPrinter.h @@ -0,0 +1,44 @@ +//===-- BPFFrameLowering.h - Define frame lowering for BPF -----*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H +#define LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H + +#include "BTFDebug.h" +#include "llvm/CodeGen/AsmPrinter.h" + +namespace llvm { + +class BPFAsmPrinter : public AsmPrinter { +public: + explicit BPFAsmPrinter(TargetMachine &TM, + std::unique_ptr Streamer) + : AsmPrinter(TM, std::move(Streamer), ID), BTF(nullptr) {} + + StringRef getPassName() const override { return "BPF Assembly Printer"; } + bool doInitialization(Module &M) override; + void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O); + bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, + const char *ExtraCode, raw_ostream &O) override; + bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, + const char *ExtraCode, raw_ostream &O) override; + + void emitInstruction(const MachineInstr *MI) override; + MCSymbol *getJTPublicSymbol(unsigned JTI); + MCSymbol *getJXAnchorSymbol(unsigned JTI); + virtual void emitJumpTableInfo() override; + + static char ID; + +private: + BTFDebug *BTF; +}; + +} // namespace llvm + +#endif /* LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H */ diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp index 742690238aa8e..98c9b2ed56959 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.cpp +++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp @@ -1092,20 +1092,6 @@ bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL, return true; } -MCSymbol *BPFTargetLowering::getJXAnchorSymbol(const MachineFunction *MF, - unsigned JTI) { - const MCAsmInfo *MAI = MF->getContext().getAsmInfo(); - SmallString<60> Name; - raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix() << "BPF.JX." - << MF->getFunctionNumber() << '.' << JTI; - return MF->getContext().getOrCreateSymbol(Name); -} - unsigned BPFTargetLowering::getJumpTableEncoding() const { return MachineJumpTableInfo::EK_LabelDifference32; } - -const MCExpr *BPFTargetLowering::getPICJumpTableRelocBaseExpr( - const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const { - return MCSymbolRefExpr::create(getJXAnchorSymbol(MF, JTI), Ctx); -} diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h index f533cd03b0c61..6e5bb2c814032 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.h +++ b/llvm/lib/Target/BPF/BPFISelLowering.h @@ -71,27 +71,6 @@ class BPFTargetLowering : public TargetLowering { // JX instruction location and target basic block label. virtual unsigned getJumpTableEncoding() const override; - // This is a label for JX instructions, used for jump table offsets - // computation, e.g.: - // - // .LBPF.JX.0.0: <------- this is the anchor - // .reloc 0, FK_SecRel_8, BPF.JT.0.0 - // gotox r1 - // ... - // .section .jumptables,"",@progbits - // .L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1 - // ... - // BPF.JT.0.0: <------- JT definition - // .long .L0_0_set_7 - // ... - static MCSymbol *getJXAnchorSymbol(const MachineFunction *MF, unsigned JTI); - - // Refers to a symbol returned by getJXAnchorSymbol(), used by - // AsmPrinter::emitJumpTableInfo() to define the .L0_0_set_7 etc above. - virtual const MCExpr * - getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, - MCContext &Ctx) const override; - private: // Control Instruction Selection Features bool HasAlu32; diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/llvm/lib/Target/BPF/BPFMCInstLower.cpp index 8d18ff58a92e0..6b18a1468d57f 100644 --- a/llvm/lib/Target/BPF/BPFMCInstLower.cpp +++ b/llvm/lib/Target/BPF/BPFMCInstLower.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "BPFMCInstLower.h" +#include "BPFAsmPrinter.h" #include "BPFISelLowering.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -57,9 +58,8 @@ MCOperand BPFMCInstLower::LowerJTIOperand(const MachineInstr &MI, // gotox r1 assert((MI.getOpcode() == BPF::JX) && "Jump Table Index operands are expected only for JX instructions"); - const MachineFunction *MF = MI.getMF(); - Printer.OutStreamer->emitLabel(BPFTargetLowering::getJXAnchorSymbol(MF, JTI)); - MCSymbol *JT = Printer.GetJTISymbol(JTI); + Printer.OutStreamer->emitLabel(Printer.getJXAnchorSymbol(JTI)); + MCSymbol *JT = Printer.getJTPublicSymbol(JTI); const MCExpr *Zero = MCConstantExpr::create(0, Ctx); Printer.OutStreamer->emitRelocDirective(*Zero, "FK_SecRel_8", MCSymbolRefExpr::create(JT, Ctx), {}, diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.h b/llvm/lib/Target/BPF/BPFMCInstLower.h index 927e9dfaf8c68..46256bcfc83e9 100644 --- a/llvm/lib/Target/BPF/BPFMCInstLower.h +++ b/llvm/lib/Target/BPF/BPFMCInstLower.h @@ -12,7 +12,7 @@ #include "llvm/Support/Compiler.h" namespace llvm { -class AsmPrinter; +class BPFAsmPrinter; class MCContext; class MCInst; class MCOperand; @@ -24,10 +24,10 @@ class MachineOperand; class LLVM_LIBRARY_VISIBILITY BPFMCInstLower { MCContext &Ctx; - AsmPrinter &Printer; + BPFAsmPrinter &Printer; public: - BPFMCInstLower(MCContext &ctx, AsmPrinter &printer) + BPFMCInstLower(MCContext &ctx, BPFAsmPrinter &printer) : Ctx(ctx), Printer(printer) {} void Lower(const MachineInstr *MI, MCInst &OutMI) const;