Skip to content

[BOLT][AArch64] Exclude JT pattern matching under assert failure #122298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[BOLT][AArch64] Exclude JT pattern matching under assert failure #122298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 169 additions & 36 deletions bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "bolt/Core/BinaryBasicBlock.h"
#include "bolt/Core/BinaryFunction.h"
#include "bolt/Core/MCPlusBuilder.h"
#include "bolt/Utils/CommandLineOpts.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCFixupKindInfo.h"
Expand Down Expand Up @@ -213,6 +214,34 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
Inst.getOpcode() == AArch64::ADDXrx64);
}

bool isSUB(const MCInst &Inst) const override {
const unsigned opcode = Inst.getOpcode();
switch (opcode) {
case AArch64::SUBSWri:
case AArch64::SUBSWrr:
case AArch64::SUBSWrs:
case AArch64::SUBSWrx:
case AArch64::SUBSXri:
case AArch64::SUBSXrr:
case AArch64::SUBSXrs:
case AArch64::SUBSXrx:
case AArch64::SUBSXrx64:
case AArch64::SUBWri:
case AArch64::SUBWrr:
case AArch64::SUBWrs:
case AArch64::SUBWrx:
case AArch64::SUBXri:
case AArch64::SUBXrr:
case AArch64::SUBXrs:
case AArch64::SUBXrx:
case AArch64::SUBXrx64:
return true;
default:
return false;
}
llvm_unreachable("");
}

bool isLDRB(const MCInst &Inst) const {
return (Inst.getOpcode() == AArch64::LDRBBpost ||
Inst.getOpcode() == AArch64::LDRBBpre ||
Expand Down Expand Up @@ -652,13 +681,45 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
/// # of this BB)
/// br x0 # Indirect jump instruction
///
/// adrp + ldr pair instructions of JT
/// adrp x3, :got:jump_table
/// ldr x1, [x1, #value]
/// ldrh w1, [x1, w3, uxtw #1]
/// adr x3, 573ae4
/// add x1, x3, w1, sxth #2
/// br x1
///
/// lld/test/ELF/aarch64-adrp-ldr-got.s
/// if .rodata and .text are sufficiently (<1M)
/// close to each other so that the adrp + ldr pair can be relaxed to
/// nop + adr.
/// nop #
/// adr x0, #6d8f8 # Get JT table address
/// ldrh w0, [x0, w4, uxtw #1] # Loads JT entry
/// adr x2, 1479b0 # Get PC first instruction for next BB
/// add x0, x2, w0, sxth #2 # Finish building branch target
/// # (entries in JT are relative to the end
/// # of this BB)
/// br x0 # Indirect jump instruction
///
/// sub + ldr pair instructions of JT, JT address on the stack and other BB
/// sub x1, x29, #0x4, lsl #12
/// ldr x1, [x1, #14352]
/// ldrh w1, [x1, w3, uxtw #1]
/// adr x3, 573ae4
/// add x1, x3, w1, sxth #2
/// br x1
bool analyzeIndirectBranchFragment(
const MCInst &Inst,
DenseMap<const MCInst *, SmallVector<MCInst *, 4>> &UDChain,
const MCExpr *&JumpTable, int64_t &Offset, int64_t &ScaleValue,
MCInst *&PCRelBase) const {
// Expect AArch64 BR
assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode");

if (Inst.getOpcode() != AArch64::BR) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: Unexpected opcode\n";
return false;
}

// Match the indirect branch pattern for aarch64
SmallVector<MCInst *, 4> &UsesRoot = UDChain[&Inst];
Expand Down Expand Up @@ -697,8 +758,12 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
// Parsed as ADDXrs reg:x8 reg:x8 reg:x12 imm:0
return false;
}
assert(DefAdd->getOpcode() == AArch64::ADDXrx &&
"Failed to match indirect branch!");

if (DefAdd->getOpcode() != AArch64::ADDXrx) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: Failed to match indirect branch!\n";
return false;
}

// Validate ADD operands
int64_t OperandExtension = DefAdd->getOperand(3).getImm();
Expand Down Expand Up @@ -737,61 +802,129 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
// This was observed in the wild in HHVM code (dispatchImpl).
return false;
}

MCInst *DefBaseAddr = UsesAdd[1];
assert(DefBaseAddr->getOpcode() == AArch64::ADR &&
"Failed to match indirect branch pattern! (fragment 3)");
if (DefBaseAddr->getOpcode() != AArch64::ADR) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: "
<< "Failed to match indirect branch pattern! (fragment 3)\n";
return false;
}

PCRelBase = DefBaseAddr;
// Match LOAD to load the jump table (relative) target
const MCInst *DefLoad = UsesAdd[2];
assert(mayLoad(*DefLoad) &&
"Failed to match indirect branch load pattern! (1)");
assert((ScaleValue != 1LL || isLDRB(*DefLoad)) &&
"Failed to match indirect branch load pattern! (2)");
assert((ScaleValue != 2LL || isLDRH(*DefLoad)) &&
"Failed to match indirect branch load pattern! (3)");
if (!mayLoad(*DefLoad)) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: "
<< "Failed to match indirect branch load pattern! (1)\n";
return false;
}

if (isLDRB(*DefLoad) && ScaleValue != 1LL) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: "
<< "Failed to match indirect branch load pattern! (2)\n";
return false;
} else if (isLDRH(*DefLoad) && ScaleValue != 2LL) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: "
<< "Failed to match indirect branch load pattern! (3)\n";
return false;
} else if (isLDRW(*DefLoad) && ScaleValue != 4LL) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: "
<< "Failed to match indirect branch load pattern! (4)\n";
return false;
}

// Match ADD that calculates the JumpTable Base Address (not the offset)
SmallVector<MCInst *, 4> &UsesLoad = UDChain[DefLoad];
const MCInst *DefJTBaseAdd = UsesLoad[1];
const MCInst *DefJTPageBias = UsesLoad[1];
MCPhysReg From, To;
if (DefJTBaseAdd == nullptr || isLoadFromStack(*DefJTBaseAdd) ||
isRegToRegMove(*DefJTBaseAdd, From, To)) {
JumpTable = nullptr;
if (DefJTPageBias == nullptr || isLoadFromStack(*DefJTPageBias) ||
isRegToRegMove(*DefJTPageBias, From, To)) {
// Sometimes base address may have been defined in another basic block
// (hoisted). Return with no jump table info.
JumpTable = nullptr;
return true;
}

if (DefJTBaseAdd->getOpcode() == AArch64::ADR) {
if (isAddXri(*DefJTPageBias)) {
if (DefJTPageBias->getOperand(2).isImm())
Offset = DefJTPageBias->getOperand(2).getImm();
SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTPageBias];
const MCInst *DefJTBasePage = UsesJTBaseAdd[1];
if (DefJTBasePage == nullptr || isLoadFromStack(*DefJTBasePage)) {
return true;
}

if (DefJTBasePage->getOpcode() != AArch64::ADRP) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: "
<< "Failed to match jump table base page pattern! (2)\n";
return false;
}

if (DefJTBasePage->getOperand(1).isExpr())
JumpTable = DefJTBasePage->getOperand(1).getExpr();
return true;
} else if (isADR(*DefJTPageBias)) {
// TODO: Handle the pattern where there is no adrp/add pair.
// It also occurs when the binary is static.
// adr x13, 0x215a18 <_nl_value_type_LC_COLLATE+0x50>
// nop
// *adr x13, 0x215a18 <_nl_value_type_LC_COLLATE+0x50>
// ldrh w13, [x13, w12, uxtw #1]
// adr x12, 0x247b30 <__gettextparse+0x5b0>
// add x13, x12, w13, sxth #2
// br x13
errs() << "BOLT-WARNING: Failed to match indirect branch: "
"nop/adr instead of adrp/add \n";
return false;
}
SmallVector<MCInst *, 4> &UsesJTNop = UDChain[DefJTPageBias];
if (UsesJTNop.size() == 1 && UsesJTNop[0] != nullptr) {
if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: "
<< "Failed to match jump table pattern! (2)\n";
return false;
}

assert(DefJTBaseAdd->getOpcode() == AArch64::ADDXri &&
"Failed to match jump table base address pattern! (1)");
if (DefJTPageBias->getOperand(1).isExpr()) {
JumpTable = DefJTPageBias->getOperand(1).getExpr();
return true;
}
} else if (mayLoad(*DefJTPageBias)) {
if (isLoadFromStack(*DefJTPageBias))
return true;

if (DefJTBaseAdd->getOperand(2).isImm())
Offset = DefJTBaseAdd->getOperand(2).getImm();
SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTBaseAdd];
const MCInst *DefJTBasePage = UsesJTBaseAdd[1];
if (DefJTBasePage == nullptr || isLoadFromStack(*DefJTBasePage)) {
JumpTable = nullptr;
return true;
SmallVector<MCInst *, 4> &UsesJTBase = UDChain[DefJTPageBias];
const MCInst *DefJTBasePage = UsesJTBase[1];
if (DefJTBasePage == nullptr)
return true;
if (DefJTBasePage->getOpcode() == AArch64::ADRP) {
// test jmp-table-pattern-matching.s (3)
if (DefJTBasePage->getOperand(1).isExpr())
JumpTable = DefJTBasePage->getOperand(1).getExpr();
return true;
} else {
// Base address may have been defined in another basic block
// and sub instruction can be used to get base page address
// jmp-table-pattern-matching.s (2)
if (isSUB(*DefJTBasePage)) {
for (const MCOperand &Operand : useOperands(*DefJTBasePage)) {
if (!Operand.isReg())
continue;
const unsigned Reg = Operand.getReg();
if (Reg == AArch64::SP || Reg == AArch64::WSP ||
Reg == AArch64::FP || Reg == AArch64::W29)
return true;
}
}
}
}
assert(DefJTBasePage->getOpcode() == AArch64::ADRP &&
"Failed to match jump table base page pattern! (2)");
if (DefJTBasePage->getOperand(1).isExpr())
JumpTable = DefJTBasePage->getOperand(1).getExpr();
return true;

if (opts::Verbosity > 1)
outs() << "BOLT-WARNING: "
<< "Failed to match jump table pattern! (4)\n";

return false;
}

DenseMap<const MCInst *, SmallVector<MCInst *, 4>>
Expand Down
127 changes: 127 additions & 0 deletions bolt/test/AArch64/jmp-table-pattern-matching.s
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
## This test checks that disassemble stage works properly
## JT with indirect branch
## 1) nop + adr pair instructions
## 2) sub + ldr pair instructions
## 3) adrp + ldr pair instructions

# REQUIRES: system-linux

# RUN: rm -rf %t && split-file %s %t

## Prepare binary (1)
# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %t/jt_nop_adr.s \
# RUN: -o %t/jt_nop_adr.o
# RUN: %clang %cflags --target=aarch64-unknown-linux %t/jt_nop_adr.o \
# RUN: -Wl,-q -Wl,-z,now, -Wl,-T,%t/within-adr-range.t -o %t/jt_nop_adr.exe
# RUN: llvm-objdump --no-show-raw-insn -d %t/jt_nop_adr.exe | FileCheck \
# RUN: --check-prefix=JT-RELAXED %s

# JT-RELAXED: <_start>:
# JT-RELAXED-NEXT: nop
# JT-RELAXED-NEXT: adr {{.*}}x3

# RUN: llvm-bolt %t/jt_nop_adr.exe -o %t/jt_nop_adr.bolt -v 2 2>&1 | FileCheck %s
# CHECK-NOT: Failed to match

## This linker script ensures that .rodata and .text are sufficiently (<1M)
## close to each other so that the adrp + ldr pair can be relaxed to nop + adr.
#--- within-adr-range.t
SECTIONS {
.rodata 0x1000: { *(.rodata) }
.text 0x2000: { *(.text) }
.rela.rodata : { *(.rela.rodata) }
}

## Prepare binary (2)
# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %t/jt_sub_ldr.s \
# RUN: -o %t/jt_sub_ldr.o
# RUN: %clang %cflags --target=aarch64-unknown-linux %t/jt_sub_ldr.o \
# RUN: -Wl,-q -Wl,-z,now -o %t/jt_sub_ldr.exe
# RUN: llvm-objdump --no-show-raw-insn -d %t/jt_sub_ldr.exe | FileCheck \
# RUN: --check-prefix=JT-SUB-LDR %s

# JT-SUB-LDR: <_start>:
# JT-SUB-LDR-NEXT: sub
# JT-SUB-LDR-NEXT: ldr

# RUN: llvm-bolt %t/jt_sub_ldr.exe -o %t/jt_sub_ldr.bolt -v 2 2>&1 | FileCheck \
# RUN: --check-prefix=JT-BOLT-SUBLDR %s
# JT-BOLT-SUBLDR-NOT: Failed to match

## Prepare binary (3)
# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %t/jt_adrp_ldr.s \
# RUN: -o %t/jt_adrp_ldr.o
# RUN: %clang %cflags --target=aarch64-unknown-linux %t/jt_adrp_ldr.o \
# RUN: -Wl,-q -Wl,-z,now -Wl,--no-relax -o %t/jt_adrp_ldr.exe
# RUN: llvm-objdump --no-show-raw-insn -d %t/jt_adrp_ldr.exe | FileCheck \
# RUN: --check-prefix=JT-ADRP-LDR %s

# JT-ADRP-LDR: <_start>:
# JT-ADRP-LDR-NEXT: adrp
# JT-ADRP-LDR-NEXT: ldr

# RUN: llvm-bolt %t/jt_adrp_ldr.exe -o %t/jt_adrp_ldr.bolt -v 2 2>&1 | FileCheck \
# RUN: --check-prefix=JT-BOLT-ADRP-LDR %s
# JT-BOLT-ADRP-LDR-NOT: Failed to match

#--- jt_nop_adr.s
.globl _start
.type _start, %function
_start:
adrp x3, :got:jump_table
ldr x3, [x3, #:got_lo12:jump_table]
ldrh w3, [x3, x1, lsl #1]
adr x1, test2_0
add x3, x1, w3, sxth #2
br x3
test2_0:
ret
test2_1:
ret

.section .rodata,"a",@progbits
jump_table:
.hword (test2_0-test2_0)>>2
.hword (test2_1-test2_0)>>2


#--- jt_sub_ldr.s
.globl _start
.type _start, %function
_start:
sub x1, x29, #0x4, lsl #12
ldr x1, [x1, #14352]
ldrh w1, [x1, w3, uxtw #1]
adr x3, test2_0
add x1, x3, w1, sxth #2
br x1
test2_0:
ret
test2_1:
ret

.section .rodata,"a",@progbits
jump_table:
.hword (test2_0-test2_0)>>2
.hword (test2_1-test2_0)>>2


#--- jt_adrp_ldr.s
.globl _start
.type _start, %function
_start:
adrp x3, :got:jump_table
ldr x3, [x3, #:got_lo12:jump_table]
ldrh w3, [x3, x1, lsl #1]
adr x1, test2_0
add x3, x1, w3, sxth #2
br x3
test2_0:
ret
test2_1:
ret

.section .rodata,"a",@progbits
jump_table:
.hword (test2_0-test2_0)>>2
.hword (test2_1-test2_0)>>2
Loading
Loading