diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp index a347794a9a30c..d96f403d2f814 100644 --- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp +++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp @@ -234,6 +234,7 @@ struct BPFOperand : public MCParsedAsmOperand { .Case("callx", true) .Case("goto", true) .Case("gotol", true) + .Case("gotox", true) .Case("may_goto", true) .Case("*", true) .Case("exit", true) diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp index e3843e0e112e2..a6d8976a734dc 100644 --- a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp +++ b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp @@ -11,52 +11,35 @@ // //===----------------------------------------------------------------------===// +#include "BPFAsmPrinter.h" #include "BPF.h" #include "BPFInstrInfo.h" #include "BPFMCInstLower.h" #include "BTFDebug.h" #include "MCTargetDesc/BPFInstPrinter.h" #include "TargetInfo/BPFTargetInfo.h" +#include "llvm/BinaryFormat/ELF.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/TargetLowering.h" #include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetLoweringObjectFile.h" using namespace llvm; #define DEBUG_TYPE "asm-printer" -namespace { -class BPFAsmPrinter : public AsmPrinter { -public: - explicit BPFAsmPrinter(TargetMachine &TM, - std::unique_ptr Streamer) - : AsmPrinter(TM, std::move(Streamer), ID), BTF(nullptr) {} - - StringRef getPassName() const override { return "BPF Assembly Printer"; } - bool doInitialization(Module &M) override; - void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O); - bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, - const char *ExtraCode, raw_ostream &O) override; - bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, - const char *ExtraCode, raw_ostream &O) override; - - void emitInstruction(const MachineInstr *MI) override; - - static char ID; - -private: - BTFDebug *BTF; -}; -} // namespace - bool BPFAsmPrinter::doInitialization(Module &M) { AsmPrinter::doInitialization(M); @@ -150,6 +133,93 @@ void BPFAsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); } +// This is a label for JX instructions, used for jump table offsets +// computation, e.g.: +// +// .LBPF.JX.0.0: <------- this is the anchor +// .reloc 0, FK_SecRel_8, BPF.JT.0.0 +// gotox r1 +// ... +// .section .jumptables,"",@progbits +// .L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1 +// ... +// BPF.JT.0.0: <------- JT definition +// .long .L0_0_set_7 +// ... +MCSymbol *BPFAsmPrinter::getJXAnchorSymbol(unsigned JTI) { + const MCAsmInfo *MAI = MF->getContext().getAsmInfo(); + SmallString<60> Name; + raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix() << "BPF.JX." + << MF->getFunctionNumber() << '.' << JTI; + return MF->getContext().getOrCreateSymbol(Name); +} + +MCSymbol *BPFAsmPrinter::getJTPublicSymbol(unsigned JTI) { + SmallString<60> Name; + raw_svector_ostream(Name) + << "BPF.JT." << MF->getFunctionNumber() << '.' << JTI; + MCSymbol *S = OutContext.getOrCreateSymbol(Name); + if (auto *ES = dyn_cast(S)) + ES->setBinding(ELF::STB_GLOBAL); + return S; +} + +void BPFAsmPrinter::emitJumpTableInfo() { + const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); + if (!MJTI) + return; + + const std::vector &JT = MJTI->getJumpTables(); + if (JT.empty()) + return; + + const TargetLoweringObjectFile &TLOF = getObjFileLowering(); + const Function &F = MF->getFunction(); + MCSection *JTS = TLOF.getSectionForJumpTable(F, TM); + assert(MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32); + unsigned EntrySize = MJTI->getEntrySize(getDataLayout()); + OutStreamer->switchSection(JTS); + for (unsigned JTI = 0; JTI < JT.size(); JTI++) { + ArrayRef JTBBs = JT[JTI].MBBs; + if (JTBBs.empty()) + continue; + + SmallPtrSet EmittedSets; + auto *Base = MCSymbolRefExpr::create(getJXAnchorSymbol(JTI), OutContext); + for (const MachineBasicBlock *MBB : JTBBs) { + if (!EmittedSets.insert(MBB).second) + continue; + + // Offset from gotox to target basic block expressed in number + // of instructions, e.g.: + // + // .L0_0_set_4 = ((LBB0_4 - .LBPF.JX.0.0) >> 3) - 1 + const MCExpr *LHS = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext); + OutStreamer->emitAssignment( + GetJTSetSymbol(JTI, MBB->getNumber()), + MCBinaryExpr::createSub( + MCBinaryExpr::createAShr( + MCBinaryExpr::createSub(LHS, Base, OutContext), + MCConstantExpr::create(3, OutContext), OutContext), + MCConstantExpr::create(1, OutContext), OutContext)); + } + // BPF.JT.0.0: + // .long .L0_0_set_4 + // .long .L0_0_set_2 + // ... + // .size BPF.JT.0.0, 128 + MCSymbol *JTStart = getJTPublicSymbol(JTI); + OutStreamer->emitLabel(JTStart); + for (const MachineBasicBlock *MBB : JTBBs) { + MCSymbol *SetSymbol = GetJTSetSymbol(JTI, MBB->getNumber()); + const MCExpr *V = MCSymbolRefExpr::create(SetSymbol, OutContext); + OutStreamer->emitValue(V, EntrySize); + } + const MCExpr *JTSize = MCConstantExpr::create(JTBBs.size() * 4, OutContext); + OutStreamer->emitELFSize(JTStart, JTSize); + } +} + char BPFAsmPrinter::ID = 0; INITIALIZE_PASS(BPFAsmPrinter, "bpf-asm-printer", "BPF Assembly Printer", false, diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.h b/llvm/lib/Target/BPF/BPFAsmPrinter.h new file mode 100644 index 0000000000000..d46d789274a9e --- /dev/null +++ b/llvm/lib/Target/BPF/BPFAsmPrinter.h @@ -0,0 +1,44 @@ +//===-- BPFFrameLowering.h - Define frame lowering for BPF -----*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H +#define LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H + +#include "BTFDebug.h" +#include "llvm/CodeGen/AsmPrinter.h" + +namespace llvm { + +class BPFAsmPrinter : public AsmPrinter { +public: + explicit BPFAsmPrinter(TargetMachine &TM, + std::unique_ptr Streamer) + : AsmPrinter(TM, std::move(Streamer), ID), BTF(nullptr) {} + + StringRef getPassName() const override { return "BPF Assembly Printer"; } + bool doInitialization(Module &M) override; + void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O); + bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, + const char *ExtraCode, raw_ostream &O) override; + bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, + const char *ExtraCode, raw_ostream &O) override; + + void emitInstruction(const MachineInstr *MI) override; + MCSymbol *getJTPublicSymbol(unsigned JTI); + MCSymbol *getJXAnchorSymbol(unsigned JTI); + virtual void emitJumpTableInfo() override; + + static char ID; + +private: + BTFDebug *BTF; +}; + +} // namespace llvm + +#endif /* LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H */ diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp index f4f414d192df0..98c9b2ed56959 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.cpp +++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp @@ -18,6 +18,7 @@ #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/CodeGen/ValueTypes.h" @@ -25,6 +26,7 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/Module.h" +#include "llvm/MC/MCAsmInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" @@ -38,6 +40,10 @@ static cl::opt BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order")); +static cl::opt BPFMinimumJumpTableEntries( + "bpf-min-jump-table-entries", cl::init(4), cl::Hidden, + cl::desc("Set minimum number of entries to use a jump table on BPF")); + static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val = {}) { std::string Str; @@ -66,9 +72,10 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, setStackPointerRegisterToSaveRestore(BPF::R11); setOperationAction(ISD::BR_CC, MVT::i64, Custom); - setOperationAction(ISD::BR_JT, MVT::Other, Expand); - setOperationAction(ISD::BRIND, MVT::Other, Expand); setOperationAction(ISD::BRCOND, MVT::Other, Expand); + LegalizeAction IndirectBrAction = STI.hasGotox() ? Custom : Expand; + setOperationAction(ISD::BR_JT, MVT::Other, IndirectBrAction); + setOperationAction(ISD::BRIND, MVT::Other, IndirectBrAction); setOperationAction(ISD::TRAP, MVT::Other, Custom); @@ -159,6 +166,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM, setBooleanContents(ZeroOrOneBooleanContent); setMaxAtomicSizeInBitsSupported(64); + setMinimumJumpTableEntries(BPFMinimumJumpTableEntries); // Function alignments setMinFunctionAlignment(Align(8)); @@ -332,6 +340,8 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { return LowerATOMIC_LOAD_STORE(Op, DAG); case ISD::TRAP: return LowerTRAP(Op, DAG); + case ISD::BR_JT: + return LowerBR_JT(Op, DAG); } } @@ -780,6 +790,16 @@ SDValue BPFTargetLowering::LowerTRAP(SDValue Op, SelectionDAG &DAG) const { return LowerCall(CLI, InVals); } +SDValue BPFTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const { + SDValue Chain = Op->getOperand(0); + SDValue Table = Op->getOperand(1); + SDValue Index = Op->getOperand(2); + JumpTableSDNode *JT = cast(Table); + SDLoc DL(Op); + SDValue TargetJT = DAG.getTargetJumpTable(JT->getIndex(), MVT::i32); + return DAG.getNode(BPFISD::BPF_BR_JT, DL, MVT::Other, Chain, TargetJT, Index); +} + const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const { switch ((BPFISD::NodeType)Opcode) { case BPFISD::FIRST_NUMBER: @@ -796,6 +816,8 @@ const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const { return "BPFISD::Wrapper"; case BPFISD::MEMCPY: return "BPFISD::MEMCPY"; + case BPFISD::BPF_BR_JT: + return "BPFISD::BPF_BR_JT"; } return nullptr; } @@ -1069,3 +1091,7 @@ bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL, return true; } + +unsigned BPFTargetLowering::getJumpTableEncoding() const { + return MachineJumpTableInfo::EK_LabelDifference32; +} diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h index 8f60261c10e9e..6e5bb2c814032 100644 --- a/llvm/lib/Target/BPF/BPFISelLowering.h +++ b/llvm/lib/Target/BPF/BPFISelLowering.h @@ -28,7 +28,8 @@ enum NodeType : unsigned { SELECT_CC, BR_CC, Wrapper, - MEMCPY + MEMCPY, + BPF_BR_JT, }; } @@ -66,6 +67,10 @@ class BPFTargetLowering : public TargetLowering { MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override; + // Always emit EK_LabelDifference32, computed as difference between + // JX instruction location and target basic block label. + virtual unsigned getJumpTableEncoding() const override; + private: // Control Instruction Selection Features bool HasAlu32; @@ -81,6 +86,7 @@ class BPFTargetLowering : public TargetLowering { SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerTRAP(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const; template SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const; diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.cpp b/llvm/lib/Target/BPF/BPFInstrInfo.cpp index 70bc163615f61..6275995ce9ba2 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.cpp +++ b/llvm/lib/Target/BPF/BPFInstrInfo.cpp @@ -181,6 +181,11 @@ bool BPFInstrInfo::analyzeBranch(MachineBasicBlock &MBB, if (!isUnpredicatedTerminator(*I)) break; + // From base method doc: ... returning true if it cannot be understood ... + // Indirect branch has multiple destinations and no true/false concepts. + if (I->isIndirectBranch()) + return true; + // A terminator that isn't a branch can't easily be handled // by this analysis. if (!I->isBranch()) @@ -259,3 +264,11 @@ unsigned BPFInstrInfo::removeBranch(MachineBasicBlock &MBB, return Count; } + +int BPFInstrInfo::getJumpTableIndex(const MachineInstr &MI) const { + if (MI.getOpcode() != BPF::JX) + return -1; + const MachineOperand &MO = MI.getOperand(1); + assert(MO.isJTI() && "JX operand #0 should be isJTI"); + return MO.getIndex(); +} diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.h b/llvm/lib/Target/BPF/BPFInstrInfo.h index d8bbad44e314e..d88e37975980a 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.h +++ b/llvm/lib/Target/BPF/BPFInstrInfo.h @@ -58,6 +58,9 @@ class BPFInstrInfo : public BPFGenInstrInfo { MachineBasicBlock *FBB, ArrayRef Cond, const DebugLoc &DL, int *BytesAdded = nullptr) const override; + + int getJumpTableIndex(const MachineInstr &MI) const override; + private: void expandMEMCPY(MachineBasicBlock::iterator) const; diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index b21f1a0eee3b0..83dccb132c519 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -31,6 +31,8 @@ def SDT_BPFMEMCPY : SDTypeProfile<0, 4, [SDTCisVT<0, i64>, SDTCisVT<1, i64>, SDTCisVT<2, i64>, SDTCisVT<3, i64>]>; +def SDT_BPFBrJt : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, // jump table + SDTCisVT<1, i64>]>; // index def BPFcall : SDNode<"BPFISD::CALL", SDT_BPFCall, [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, @@ -49,6 +51,9 @@ def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>; def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY, [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore, SDNPMayLoad]>; +def BPFBrJt : SDNode<"BPFISD::BPF_BR_JT", SDT_BPFBrJt, + [SDNPHasChain]>; + def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">; def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">; def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">; @@ -183,6 +188,15 @@ class TYPE_LD_ST mode, bits<2> size, let Inst{60-59} = size; } +// For indirect jump +class TYPE_IND_JMP op, bits<1> srctype, + dag outs, dag ins, string asmstr, list pattern> + : InstBPF { + + let Inst{63-60} = op; + let Inst{59} = srctype; +} + // jump instructions class JMP_RR : TYPE_ALU_JMP let BPFClass = BPF_JMP; } +class JMP_IND Pattern> + : TYPE_ALU_JMP { + bits<4> dst; + + let Inst{51-48} = dst; + let BPFClass = BPF_JMP; +} + class JMP_JCOND Pattern> : TYPE_ALU_JMP; defm JSLE : J; def JCOND : JMP_JCOND; + +let isIndirectBranch = 1, isBarrier = 1, isNotDuplicable = 1 in { + def JX : JMP_IND; +} } // ALU instructions diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/llvm/lib/Target/BPF/BPFMCInstLower.cpp index 040a1fb750702..6b18a1468d57f 100644 --- a/llvm/lib/Target/BPF/BPFMCInstLower.cpp +++ b/llvm/lib/Target/BPF/BPFMCInstLower.cpp @@ -12,6 +12,8 @@ //===----------------------------------------------------------------------===// #include "BPFMCInstLower.h" +#include "BPFAsmPrinter.h" +#include "BPFISelLowering.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineInstr.h" @@ -19,6 +21,7 @@ #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" +#include "llvm/MC/MCStreamer.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -44,6 +47,28 @@ MCOperand BPFMCInstLower::LowerSymbolOperand(const MachineOperand &MO, return MCOperand::createExpr(Expr); } +MCOperand BPFMCInstLower::LowerJTIOperand(const MachineInstr &MI, + const MachineOperand &MO, + int JTI) const { + // Emit relocation entry referencing jump table symbol plus a label + // for JX anchor, e.g.: + // + // .LBPF.JX.0.0: + // .reloc 0, FK_SecRel_8, BPF.JT.0.0 + // gotox r1 + assert((MI.getOpcode() == BPF::JX) && + "Jump Table Index operands are expected only for JX instructions"); + Printer.OutStreamer->emitLabel(Printer.getJXAnchorSymbol(JTI)); + MCSymbol *JT = Printer.getJTPublicSymbol(JTI); + const MCExpr *Zero = MCConstantExpr::create(0, Ctx); + Printer.OutStreamer->emitRelocDirective(*Zero, "FK_SecRel_8", + MCSymbolRefExpr::create(JT, Ctx), {}, + *Ctx.getSubtargetInfo()); + // JTI parameter is used only to emit relocation and is not a part + // of JX instruction encoding, so this operand is not really used. + return MCOperand::createImm(JTI); +} + void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { OutMI.setOpcode(MI->getOpcode()); @@ -77,6 +102,9 @@ void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { case MachineOperand::MO_ConstantPoolIndex: MCOp = LowerSymbolOperand(MO, Printer.GetCPISymbol(MO.getIndex())); break; + case MachineOperand::MO_JumpTableIndex: + MCOp = LowerJTIOperand(*MI, MO, MO.getIndex()); + break; } OutMI.addOperand(MCOp); diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.h b/llvm/lib/Target/BPF/BPFMCInstLower.h index 4bd0f1f0bf1cf..46256bcfc83e9 100644 --- a/llvm/lib/Target/BPF/BPFMCInstLower.h +++ b/llvm/lib/Target/BPF/BPFMCInstLower.h @@ -12,7 +12,7 @@ #include "llvm/Support/Compiler.h" namespace llvm { -class AsmPrinter; +class BPFAsmPrinter; class MCContext; class MCInst; class MCOperand; @@ -24,14 +24,16 @@ class MachineOperand; class LLVM_LIBRARY_VISIBILITY BPFMCInstLower { MCContext &Ctx; - AsmPrinter &Printer; + BPFAsmPrinter &Printer; public: - BPFMCInstLower(MCContext &ctx, AsmPrinter &printer) + BPFMCInstLower(MCContext &ctx, BPFAsmPrinter &printer) : Ctx(ctx), Printer(printer) {} void Lower(const MachineInstr *MI, MCInst &OutMI) const; MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const; + MCOperand LowerJTIOperand(const MachineInstr &MI, const MachineOperand &MO, + int JTI) const; MCSymbol *GetGlobalAddressSymbol(const MachineOperand &MO) const; MCSymbol *GetExternalSymbolSymbol(const MachineOperand &MO) const; diff --git a/llvm/lib/Target/BPF/BPFSubtarget.cpp b/llvm/lib/Target/BPF/BPFSubtarget.cpp index 4167547680b12..a11aa6933147c 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.cpp +++ b/llvm/lib/Target/BPF/BPFSubtarget.cpp @@ -43,6 +43,8 @@ static cl::opt static cl::opt Disable_load_acq_store_rel( "disable-load-acq-store-rel", cl::Hidden, cl::init(false), cl::desc("Disable load-acquire and store-release insns")); +static cl::opt Disable_gotox("disable-gotox", cl::Hidden, cl::init(false), + cl::desc("Disable gotox insn")); void BPFSubtarget::anchor() {} @@ -66,6 +68,7 @@ void BPFSubtarget::initializeEnvironment() { HasGotol = false; HasStoreImm = false; HasLoadAcqStoreRel = false; + HasGotox = false; } void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { @@ -96,6 +99,7 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { HasGotol = !Disable_gotol; HasStoreImm = !Disable_StoreImm; HasLoadAcqStoreRel = !Disable_load_acq_store_rel; + HasGotox = !Disable_gotox; return; } } diff --git a/llvm/lib/Target/BPF/BPFSubtarget.h b/llvm/lib/Target/BPF/BPFSubtarget.h index aed2211265e23..e870dfdc85ec9 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.h +++ b/llvm/lib/Target/BPF/BPFSubtarget.h @@ -65,7 +65,7 @@ class BPFSubtarget : public BPFGenSubtargetInfo { // whether cpu v4 insns are enabled. bool HasLdsx, HasMovsx, HasBswap, HasSdivSmod, HasGotol, HasStoreImm, - HasLoadAcqStoreRel; + HasLoadAcqStoreRel, HasGotox; std::unique_ptr CallLoweringInfo; std::unique_ptr InstSelector; @@ -94,6 +94,7 @@ class BPFSubtarget : public BPFGenSubtargetInfo { bool hasGotol() const { return HasGotol; } bool hasStoreImm() const { return HasStoreImm; } bool hasLoadAcqStoreRel() const { return HasLoadAcqStoreRel; } + bool hasGotox() const { return HasGotox; } bool isLittleEndian() const { return IsLittleEndian; } diff --git a/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp new file mode 100644 index 0000000000000..997f09870bad6 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp @@ -0,0 +1,19 @@ +//===------------------ BPFTargetLoweringObjectFile.cpp -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "BPFTargetLoweringObjectFile.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCSectionELF.h" + +using namespace llvm; + +MCSection *BPFTargetLoweringObjectFileELF::getSectionForJumpTable( + const Function &F, const TargetMachine &TM, + const MachineJumpTableEntry *JTE) const { + return getContext().getELFSection(".jumptables", ELF::SHT_PROGBITS, 0); +} diff --git a/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h new file mode 100644 index 0000000000000..f3064c0c8cb8a --- /dev/null +++ b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h @@ -0,0 +1,25 @@ +//===============- BPFTargetLoweringObjectFile.h -*- C++ -*-================// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE +#define LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE + +#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" +#include "llvm/Target/TargetLoweringObjectFile.h" + +namespace llvm { +class BPFTargetLoweringObjectFileELF : public TargetLoweringObjectFileELF { + +public: + virtual MCSection * + getSectionForJumpTable(const Function &F, const TargetMachine &TM, + const MachineJumpTableEntry *JTE) const override; +}; +} // namespace llvm + +#endif // LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 527a480354571..d538b6fe11675 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -12,6 +12,7 @@ #include "BPFTargetMachine.h" #include "BPF.h" +#include "BPFTargetLoweringObjectFile.h" #include "BPFTargetTransformInfo.h" #include "MCTargetDesc/BPFMCAsmInfo.h" #include "TargetInfo/BPFTargetInfo.h" @@ -80,7 +81,7 @@ BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT, : CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options, getEffectiveRelocModel(RM), getEffectiveCodeModel(CM, CodeModel::Small), OL), - TLOF(std::make_unique()), + TLOF(std::make_unique()), Subtarget(TT, std::string(CPU), std::string(FS), *this) { if (!DisableCheckUnreachable) { this->Options.TrapUnreachable = true; diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index eade4cacb7100..3678f1335ca36 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -37,6 +37,7 @@ add_llvm_target(BPFCodeGen BPFRegisterInfo.cpp BPFSelectionDAGInfo.cpp BPFSubtarget.cpp + BPFTargetLoweringObjectFile.cpp BPFTargetMachine.cpp BPFMIPeephole.cpp BPFMIChecking.cpp diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp index 958790d49d087..7165a88fb17e7 100644 --- a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp +++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp @@ -34,6 +34,7 @@ class BPFAsmBackend : public MCAsmBackend { createObjectTargetWriter() const override; MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override; + std::optional getFixupKind(StringRef Name) const override; bool writeNopData(raw_ostream &OS, uint64_t Count, const MCSubtargetInfo *STI) const override; @@ -54,6 +55,18 @@ MCFixupKindInfo BPFAsmBackend::getFixupKindInfo(MCFixupKind Kind) const { return Infos[Kind - FirstTargetFixupKind]; } +std::optional BPFAsmBackend::getFixupKind(StringRef Name) const { + if (Name == "FK_SecRel_8") + return FK_SecRel_8; + if (Name == "FK_BPF_PCRel_4") + return BPF::FK_BPF_PCRel_4; + if (Name == "FK_Data_8") + return FK_Data_8; + if (Name == "FK_Data_4") + return FK_Data_4; + return std::nullopt; +} + bool BPFAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, const MCSubtargetInfo *STI) const { if ((Count % 8) != 0) diff --git a/llvm/test/CodeGen/BPF/jump-table-multi.ll b/llvm/test/CodeGen/BPF/jump-table-multi.ll new file mode 100644 index 0000000000000..4f8b9ad428804 --- /dev/null +++ b/llvm/test/CodeGen/BPF/jump-table-multi.ll @@ -0,0 +1,78 @@ +; RUN: llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < %s | FileCheck %s + +; Check that two jump tables of different size are generated + +define i64 @foo(i64 %v1, i64 %v2) { +; CHECK: .LBPF.JX.0.0: +; CHECK-NEXT: .reloc 0, FK_SecRel_8, BPF.JT.0.0 +; CHECK-NEXT: gotox r1 + +; CHECK: .LBPF.JX.0.1: +; CHECK-NEXT: .reloc 0, FK_SecRel_8, BPF.JT.0.1 +; CHECK-NEXT: gotox r2 + +; CHECK: .section .jumptables,"",@progbits + +; CHECK-NEXT: [[m1:.*]] = (({{.*}}-.LBPF.JX.0.0)>>3)-1 +; CHECK-NEXT: [[m2:.*]] = (({{.*}}-.LBPF.JX.0.0)>>3)-1 +; CHECK-NEXT: BPF.JT.0.0: +; CHECK-NEXT: .long [[m1]] +; CHECK-NEXT: .long [[m2]] +; CHECK-NEXT: .size BPF.JT.0.0, 8 + +; CHECK-NEXT: [[m1:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1 +; CHECK-NEXT: [[m2:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1 +; CHECK-NEXT: [[m3:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1 +; CHECK-NEXT: BPF.JT.0.1: +; CHECK-NEXT: .long [[m1]] +; CHECK-NEXT: .long [[m2]] +; CHECK-NEXT: .long [[m3]] +; CHECK-NEXT: .size BPF.JT.0.1, 12 + +entry: + switch i64 %v1, label %sw.default [ + i64 0, label %sw.bb1 + i64 1, label %sw.bb2 + ] + +sw.bb1: + br label %sw.epilog + +sw.bb2: + br label %sw.epilog + +sw.default: + br label %sw.epilog + +sw.epilog: + %ret = phi i64 [ 42, %sw.default ], [ 3, %sw.bb1 ], [ 5, %sw.bb2 ] + switch i64 %v2, label %sw.default.1 [ + i64 0, label %sw.bb1.1 + i64 1, label %sw.bb2.1 + i64 2, label %sw.bb3.1 + ] + +sw.bb1.1: + br label %sw.epilog.1 + +sw.bb2.1: + br label %sw.epilog.1 + +sw.bb3.1: + br label %sw.epilog.1 + +sw.default.1: + br label %sw.epilog.1 + +sw.epilog.1: + %ret.1 = phi i64 [ 42, %sw.default.1 ], [ 3, %sw.bb1.1 ], [ 5, %sw.bb2.1 ], [ 7, %sw.bb3.1 ] + %ret.2 = add i64 %ret, %ret.1 + ret i64 %ret.2 +} + +!llvm.module.flags = !{!0, !1} +!llvm.ident = !{!2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 7, !"frame-pointer", i32 2} +!2 = !{!"clang some version"} diff --git a/llvm/test/CodeGen/BPF/jump-table-simple.ll b/llvm/test/CodeGen/BPF/jump-table-simple.ll new file mode 100644 index 0000000000000..8c048223dbc0b --- /dev/null +++ b/llvm/test/CodeGen/BPF/jump-table-simple.ll @@ -0,0 +1,86 @@ +; Checks generated using command: +; +; llvm/utils/update_test_body.py llvm/test/CodeGen/BPF/jump-table-simple.ll +; +; RUN: rm -rf %t && split-file %s %t && cd %t +; RUN: llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < test.ll | FileCheck %s +; +; Check general program structure generated for a jump table + +.ifdef GEN +;--- test.ll +define i64 @foo(i64 %v) { +entry: + switch i64 %v, label %sw.default [ + i64 0, label %sw.epilog + i64 1, label %sw.bb1 + i64 2, label %sw.bb1 + i64 3, label %sw.bb2 + ] + +sw.bb1: + br label %sw.epilog + +sw.bb2: + br label %sw.epilog + +sw.default: + br label %sw.epilog + +sw.epilog: + %ret = phi i64 [ 42, %sw.default ], [ 3, %sw.bb1 ], [ 5, %sw.bb2 ], [ 7, %entry ] + ret i64 %ret +} + +!llvm.module.flags = !{!0, !1} +!llvm.ident = !{!2} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{i32 7, !"frame-pointer", i32 2} +!2 = !{!"clang some version"} + +;--- gen +echo "" +echo "; Generated checks follow" +echo ";" +llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < test.ll \ + | awk '/# -- End function/ {p=0} /@function/ {p=1} p {print "; CHECK" ": " $0}' +.endif + +; Generated checks follow +; +; CHECK: .type foo,@function +; CHECK: foo: # @foo +; CHECK: .cfi_startproc +; CHECK: # %bb.0: # %entry +; CHECK: if r1 > 3 goto LBB0_5 +; CHECK: # %bb.1: # %entry +; CHECK: .LBPF.JX.0.0: +; CHECK: .reloc 0, FK_SecRel_8, BPF.JT.0.0 +; CHECK: gotox r1 +; CHECK: LBB0_3: # %sw.bb1 +; CHECK: r0 = 3 +; CHECK: goto LBB0_6 +; CHECK: LBB0_2: +; CHECK: r0 = 7 +; CHECK: goto LBB0_6 +; CHECK: LBB0_4: # %sw.bb2 +; CHECK: r0 = 5 +; CHECK: goto LBB0_6 +; CHECK: LBB0_5: # %sw.default +; CHECK: r0 = 42 +; CHECK: LBB0_6: # %sw.epilog +; CHECK: exit +; CHECK: .Lfunc_end0: +; CHECK: .size foo, .Lfunc_end0-foo +; CHECK: .cfi_endproc +; CHECK: .section .jumptables,"",@progbits +; CHECK: .L0_0_set_2 = ((LBB0_2-.LBPF.JX.0.0)>>3)-1 +; CHECK: .L0_0_set_3 = ((LBB0_3-.LBPF.JX.0.0)>>3)-1 +; CHECK: .L0_0_set_4 = ((LBB0_4-.LBPF.JX.0.0)>>3)-1 +; CHECK: BPF.JT.0.0: +; CHECK: .long .L0_0_set_2 +; CHECK: .long .L0_0_set_3 +; CHECK: .long .L0_0_set_3 +; CHECK: .long .L0_0_set_4 +; CHECK: .size BPF.JT.0.0, 16