Skip to content

Commit 64ef0b4

Browse files
author
Yonghong Song
committed
[BPF] Add jump table support with switch statements and computed goto
NOTE 1: We probably need cpu v5 or other flags to enable this feature. We can add it later when necessary. Let us use cpu v4 for now. NOTE 2: An option -bpf-min-jump-table-entries is implemented to control the minimum number of entries to use a jump table on BPF. The default value 5 and this is to make it easy to test. Eventually we will increase min jump table entries to be 13. This patch adds jump table support. A new insn 'gotox <reg>' is added to allow goto through a register. The register represents the address in the current section. Example 1 (switch statement): ============================= Code: struct simple_ctx { int x; int y; int z; }; int ret_user, ret_user2; void bar(void); int foo(struct simple_ctx *ctx, struct simple_ctx *ctx2) { switch (ctx->x) { case 1: ret_user = 18; break; case 20: ret_user = 6; break; case 16: ret_user = 9; break; case 6: ret_user = 16; break; case 8: ret_user = 14; break; case 30: ret_user = 2; break; default: ret_user = 1; break; } bar(); switch (ctx2->x) { case 0: ret_user2 = 8; break; case 31: ret_user2 = 5; break; case 13: ret_user2 = 8; break; case 1: ret_user2 = 3; break; case 11: ret_user2 = 4; break; default: ret_user2 = 29; break; } return 0; } Run: clang --target=bpf -mcpu=v4 -O2 -S test.c The assembly code: ... # %bb.1: # %entry r1 <<= 3 r2 = .LJTI0_0 ll r2 += r1 r1 = *(u64 *)(r2 + 0) gotox r1 LBB0_2: w1 = 18 goto LBB0_9 ... # %bb.10: # %sw.epilog r1 <<= 3 r2 = .LJTI0_1 ll r2 += r1 r1 = *(u64 *)(r2 + 0) gotox r1 LBB0_11: w1 = 8 goto LBB0_16 ... .section .rodata,"a",@progbits .p2align 3, 0x0 .LJTI0_0: .quad LBB0_2 .quad LBB0_8 ... .quad LBB0_7 .LJTI0_1: .quad LBB0_11 .quad LBB0_13 ... Although we do have labels .LJTI0_0 and .LJTI0_1, but since they have prefix '.L' so they won't appear in the .o file like other symbols. Run: llvm-objdump -Sr test.o ... 4: 67 01 00 00 03 00 00 00 r1 <<= 0x3 5: 18 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r2 = 0x0 ll 0000000000000028: R_BPF_64_64 .rodata 7: 0f 12 00 00 00 00 00 00 r2 += r1 ... 29: 67 01 00 00 03 00 00 00 r1 <<= 0x3 30: 18 02 00 00 f0 00 00 00 00 00 00 00 00 00 00 00 r2 = 0xf0 ll 00000000000000f0: R_BPF_64_64 .rodata 32: 0f 12 00 00 00 00 00 00 r2 += r1 The size of jump table is not obvious. The libbpf needs to check all relocations against .rodata section in order to get precise size in order to construct bpf maps. Example 2 (Simple computed goto): ================================= Code: int bar(int a) { __label__ l1, l2; void * volatile tgt; int ret = 0; if (a) tgt = &&l1; // synthetic jump table generated here else tgt = &&l2; // another synthetic jump table goto *tgt; l1: ret += 1; l2: ret += 2; return ret; } Compile: clang --target=bpf -mcpu=v4 -O2 -c test1.c Objdump: llvm-objdump -Sr test1.o 0: 18 02 00 00 50 00 00 00 00 00 00 00 00 00 00 00 r2 = 0x50 ll 0000000000000000: R_BPF_64_64 .text 2: 16 01 02 00 00 00 00 00 if w1 == 0x0 goto +0x2 <bar+0x28> 3: 18 02 00 00 40 00 00 00 00 00 00 00 00 00 00 00 r2 = 0x40 ll 0000000000000018: R_BPF_64_64 .text 5: 7b 2a f8 ff 00 00 00 00 *(u64 *)(r10 - 0x8) = r2 6: 79 a1 f8 ff 00 00 00 00 r1 = *(u64 *)(r10 - 0x8) 7: 0d 01 00 00 00 00 00 00 gotox r1 8: b4 00 00 00 03 00 00 00 w0 = 0x3 9: 05 00 01 00 00 00 00 00 goto +0x1 <bar+0x58> 10: b4 00 00 00 02 00 00 00 w0 = 0x2 11: 95 00 00 00 00 00 00 00 exit For this case, there is no jump table so it would be hard to track offset during verification esp. when offset needs adjustment. So practically we need to create two jump tables for '&&l1' and '&&l2' respectively. Example 3 (More complicated computed goto): =========================================== Code: int foo(int a, int b) { __label__ l1, l2, l3, l4; void *jt1[] = {[0]=&&l1, [1]=&&l2}; void *jt2[] = {[0]=&&l3, [1]=&&l4}; int ret = 0; goto *jt1[a % 2]; l1: ret += 1; l2: ret += 3; goto *jt2[b % 2]; l3: ret += 5; l4: ret += 7; return ret; } Compile: clang --target=bpf -mcpu=v4 -O2 -S test2.c Asm code: ... r3 = (s32)r2 r3 <<= 3 r2 = .L__const.foo.jt2 ll r2 += r3 r1 = (s32)r1 r1 <<= 3 r3 = .L__const.foo.jt1 ll r3 += r1 w0 = 0 r1 = *(u64 *)(r3 + 0) gotox r1 .Ltmp0: # Block address taken LBB0_1: # %l1 # =>This Inner Loop Header: Depth=1 w0 += 1 w0 += 3 r1 = *(u64 *)(r2 + 0) gotox r1 .Ltmp1: # Block address taken LBB0_2: # %l2 ... .type .L__const.foo.jt1,@object # @__const.foo.jt1 .section .rodata,"a",@progbits .p2align 3, 0x0 .L__const.foo.jt1: .quad .Ltmp0 .quad .Ltmp1 .size .L__const.foo.jt1, 16 .type .L__const.foo.jt2,@object # @__const.foo.jt2 .p2align 3, 0x0 .L__const.foo.jt2: .quad .Ltmp2 .quad .Ltmp3 .size .L__const.foo.jt2, 16 Similar to switch statement case, for the binary, the symbols .L__const.foo.jt* will not show up in the symbol table and jump table will be in .rodata section. We need to resolve Example 2 case. Also with more libbpf work (dealing with .rodata sections etc.), everything should work fine for Examples 1 and 3. But we could do better by - Replacing symbols like .L<...> with symbols appearing in symbol table. - Add jump tables to .jumptables section instead of .rodata section. This should make things easier for libbpf. User can also benefit from this as relocation/section will be easy to check. Next two patches will fix Example 2 and improve all of them as mentioned in the above.
1 parent 175bab3 commit 64ef0b4

File tree

7 files changed

+115
-1
lines changed

7 files changed

+115
-1
lines changed

‎llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ struct BPFOperand : public MCParsedAsmOperand {
234234
.Case("callx", true)
235235
.Case("goto", true)
236236
.Case("gotol", true)
237+
.Case("gotox", true)
237238
.Case("may_goto", true)
238239
.Case("*", true)
239240
.Case("exit", true)

‎llvm/lib/Target/BPF/BPFISelLowering.cpp‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "llvm/CodeGen/MachineFrameInfo.h"
1919
#include "llvm/CodeGen/MachineFunction.h"
2020
#include "llvm/CodeGen/MachineInstrBuilder.h"
21+
#include "llvm/CodeGen/MachineJumpTableInfo.h"
2122
#include "llvm/CodeGen/MachineRegisterInfo.h"
2223
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
2324
#include "llvm/CodeGen/ValueTypes.h"
@@ -38,6 +39,10 @@ static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
3839
cl::Hidden, cl::init(false),
3940
cl::desc("Expand memcpy into load/store pairs in order"));
4041

42+
static cl::opt<unsigned> BPFMinimumJumpTableEntries(
43+
"bpf-min-jump-table-entries", cl::init(5), cl::Hidden,
44+
cl::desc("Set minimum number of entries to use a jump table on BPF"));
45+
4146
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg,
4247
SDValue Val = {}) {
4348
std::string Str;
@@ -67,12 +72,12 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
6772

6873
setOperationAction(ISD::BR_CC, MVT::i64, Custom);
6974
setOperationAction(ISD::BR_JT, MVT::Other, Expand);
70-
setOperationAction(ISD::BRIND, MVT::Other, Expand);
7175
setOperationAction(ISD::BRCOND, MVT::Other, Expand);
7276

7377
setOperationAction(ISD::TRAP, MVT::Other, Custom);
7478

7579
setOperationAction({ISD::GlobalAddress, ISD::ConstantPool}, MVT::i64, Custom);
80+
setOperationAction({ISD::JumpTable, ISD::BlockAddress}, MVT::i64, Custom);
7681

7782
setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
7883
setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
@@ -159,6 +164,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
159164

160165
setBooleanContents(ZeroOrOneBooleanContent);
161166
setMaxAtomicSizeInBitsSupported(64);
167+
setMinimumJumpTableEntries(BPFMinimumJumpTableEntries);
162168

163169
// Function alignments
164170
setMinFunctionAlignment(Align(8));
@@ -246,6 +252,10 @@ bool BPFTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
246252
return TargetLoweringBase::isZExtFree(Val, VT2);
247253
}
248254

255+
unsigned BPFTargetLowering::getJumpTableEncoding() const {
256+
return MachineJumpTableInfo::EK_BlockAddress;
257+
}
258+
249259
BPFTargetLowering::ConstraintType
250260
BPFTargetLowering::getConstraintType(StringRef Constraint) const {
251261
if (Constraint.size() == 1) {
@@ -316,10 +326,14 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
316326
report_fatal_error("unimplemented opcode: " + Twine(Op.getOpcode()));
317327
case ISD::BR_CC:
318328
return LowerBR_CC(Op, DAG);
329+
case ISD::JumpTable:
330+
return LowerJumpTable(Op, DAG);
319331
case ISD::GlobalAddress:
320332
return LowerGlobalAddress(Op, DAG);
321333
case ISD::ConstantPool:
322334
return LowerConstantPool(Op, DAG);
335+
case ISD::BlockAddress:
336+
return LowerBlockAddress(Op, DAG);
323337
case ISD::SELECT_CC:
324338
return LowerSELECT_CC(Op, DAG);
325339
case ISD::SDIV:
@@ -780,6 +794,11 @@ SDValue BPFTargetLowering::LowerTRAP(SDValue Op, SelectionDAG &DAG) const {
780794
return LowerCall(CLI, InVals);
781795
}
782796

797+
SDValue BPFTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
798+
JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
799+
return getAddr(N, DAG);
800+
}
801+
783802
const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
784803
switch ((BPFISD::NodeType)Opcode) {
785804
case BPFISD::FIRST_NUMBER:
@@ -811,6 +830,17 @@ static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
811830
N->getOffset(), Flags);
812831
}
813832

833+
static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty,
834+
SelectionDAG &DAG, unsigned Flags) {
835+
return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
836+
Flags);
837+
}
838+
839+
static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
840+
SelectionDAG &DAG, unsigned Flags) {
841+
return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
842+
}
843+
814844
template <class NodeTy>
815845
SDValue BPFTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
816846
unsigned Flags) const {
@@ -837,6 +867,12 @@ SDValue BPFTargetLowering::LowerConstantPool(SDValue Op,
837867
return getAddr(N, DAG);
838868
}
839869

870+
SDValue BPFTargetLowering::LowerBlockAddress(SDValue Op,
871+
SelectionDAG &DAG) const {
872+
BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
873+
return getAddr(N, DAG);
874+
}
875+
840876
unsigned
841877
BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
842878
unsigned Reg, bool isSigned) const {

‎llvm/lib/Target/BPF/BPFISelLowering.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ class BPFTargetLowering : public TargetLowering {
6666

6767
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override;
6868

69+
unsigned getJumpTableEncoding() const override;
70+
6971
private:
7072
// Control Instruction Selection Features
7173
bool HasAlu32;
@@ -81,6 +83,8 @@ class BPFTargetLowering : public TargetLowering {
8183
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
8284
SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
8385
SDValue LowerTRAP(SDValue Op, SelectionDAG &DAG) const;
86+
SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
87+
SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
8488

8589
template <class NodeTy>
8690
SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;

‎llvm/lib/Target/BPF/BPFInstrInfo.cpp‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,11 @@ bool BPFInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
182182
if (!isUnpredicatedTerminator(*I))
183183
break;
184184

185+
// From base method doc: ... returning true if it cannot be understood ...
186+
// Indirect branch has multiple destinations and no true/false concepts.
187+
if (I->isIndirectBranch())
188+
return true;
189+
185190
// A terminator that isn't a branch can't easily be handled
186191
// by this analysis.
187192
if (!I->isBranch())
@@ -260,3 +265,43 @@ unsigned BPFInstrInfo::removeBranch(MachineBasicBlock &MBB,
260265

261266
return Count;
262267
}
268+
269+
int BPFInstrInfo::getJumpTableIndex(const MachineInstr &MI) const {
270+
if (MI.getOpcode() != BPF::JX)
271+
return -1;
272+
273+
// The pattern looks like:
274+
// %0 = LD_imm64 %jump-table.0 ; load jump-table address
275+
// %1 = ADD_rr %0, $another_reg ; address + offset
276+
// %2 = LDD %1, 0 ; load the actual label
277+
// JX %2
278+
const MachineFunction &MF = *MI.getParent()->getParent();
279+
const MachineRegisterInfo &MRI = MF.getRegInfo();
280+
281+
Register Reg = MI.getOperand(0).getReg();
282+
if (!Reg.isVirtual())
283+
return -1;
284+
MachineInstr *Ldd = MRI.getUniqueVRegDef(Reg);
285+
if (Ldd == nullptr || Ldd->getOpcode() != BPF::LDD)
286+
return -1;
287+
288+
Reg = Ldd->getOperand(1).getReg();
289+
if (!Reg.isVirtual())
290+
return -1;
291+
MachineInstr *Add = MRI.getUniqueVRegDef(Reg);
292+
if (Add == nullptr || Add->getOpcode() != BPF::ADD_rr)
293+
return -1;
294+
295+
Reg = Add->getOperand(1).getReg();
296+
if (!Reg.isVirtual())
297+
return -1;
298+
MachineInstr *LDimm64 = MRI.getUniqueVRegDef(Reg);
299+
if (LDimm64 == nullptr || LDimm64->getOpcode() != BPF::LD_imm64)
300+
return -1;
301+
302+
const MachineOperand &MO = LDimm64->getOperand(1);
303+
if (!MO.isJTI())
304+
return -1;
305+
306+
return MO.getIndex();
307+
}

‎llvm/lib/Target/BPF/BPFInstrInfo.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ class BPFInstrInfo : public BPFGenInstrInfo {
5959
MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
6060
const DebugLoc &DL,
6161
int *BytesAdded = nullptr) const override;
62+
63+
int getJumpTableIndex(const MachineInstr &MI) const override;
64+
6265
private:
6366
void expandMEMCPY(MachineBasicBlock::iterator) const;
6467

‎llvm/lib/Target/BPF/BPFInstrInfo.td‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,18 @@ class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
216216
let BPFClass = BPF_JMP;
217217
}
218218

219+
class JMP_IND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
220+
: TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
221+
(outs),
222+
(ins GPR:$dst),
223+
!strconcat(OpcodeStr, " $dst"),
224+
Pattern> {
225+
bits<4> dst;
226+
227+
let Inst{51-48} = dst;
228+
let BPFClass = BPF_JMP;
229+
}
230+
219231
class JMP_JCOND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
220232
: TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
221233
(outs),
@@ -281,6 +293,10 @@ defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>;
281293
defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;
282294
defm JSET : J<BPF_JSET, "&", NoCond, NoCond>;
283295
def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>;
296+
297+
let isIndirectBranch = 1, isBarrier = 1 in {
298+
def JX : JMP_IND<BPF_JA, "gotox", [(brind i64:$dst)]>;
299+
}
284300
}
285301

286302
// ALU instructions
@@ -851,6 +867,8 @@ let usesCustomInserter = 1, isCodeGenOnly = 1 in {
851867
// load 64-bit global addr into register
852868
def : Pat<(BPFWrapper tglobaladdr:$in), (LD_imm64 tglobaladdr:$in)>;
853869
def : Pat<(BPFWrapper tconstpool:$in), (LD_imm64 tconstpool:$in)>;
870+
def : Pat<(BPFWrapper tblockaddress:$in), (LD_imm64 tblockaddress:$in)>;
871+
def : Pat<(BPFWrapper tjumptable:$in), (LD_imm64 tjumptable:$in)>;
854872

855873
// 0xffffFFFF doesn't fit into simm32, optimize common case
856874
def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)),

‎llvm/lib/Target/BPF/BPFMCInstLower.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,13 @@ void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
7777
case MachineOperand::MO_ConstantPoolIndex:
7878
MCOp = LowerSymbolOperand(MO, Printer.GetCPISymbol(MO.getIndex()));
7979
break;
80+
case MachineOperand::MO_JumpTableIndex:
81+
MCOp = LowerSymbolOperand(MO, Printer.GetJTISymbol(MO.getIndex()));
82+
break;
83+
case MachineOperand::MO_BlockAddress:
84+
MCOp = LowerSymbolOperand(
85+
MO, Printer.GetBlockAddressSymbol(MO.getBlockAddress()));
86+
break;
8087
}
8188

8289
OutMI.addOperand(MCOp);

0 commit comments

Comments
 (0)