Skip to content

Commit 01a1351

Browse files
nikicnikic
authored
[RelLookupTableConverter] Make GEP type independent (llvm#155404)
This makes the RelLookupTableConverter independent of the type used in the GEP. In particular, it removes the requirement to have a leading zero index.
1 parent 4d9578b commit 01a1351

File tree

2 files changed

+158
-44
lines changed

2 files changed

+158
-44
lines changed

‎llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp‎

Lines changed: 61 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -21,51 +21,61 @@
2121

2222
using namespace llvm;
2323

24-
static bool shouldConvertToRelLookupTable(Module &M, GlobalVariable &GV) {
24+
struct LookupTableInfo {
25+
Value *Index;
26+
SmallVector<Constant *> Ptrs;
27+
};
28+
29+
static bool shouldConvertToRelLookupTable(LookupTableInfo &Info, Module &M,
30+
GlobalVariable &GV) {
2531
// If lookup table has more than one user,
2632
// do not generate a relative lookup table.
2733
// This is to simplify the analysis that needs to be done for this pass.
2834
// TODO: Add support for lookup tables with multiple uses.
2935
// For ex, this can happen when a function that uses a lookup table gets
3036
// inlined into multiple call sites.
31-
if (!GV.hasInitializer() ||
32-
!GV.isConstant() ||
33-
!GV.hasOneUse())
34-
return false;
35-
36-
GetElementPtrInst *GEP =
37-
dyn_cast<GetElementPtrInst>(GV.use_begin()->getUser());
38-
if (!GEP || !GEP->hasOneUse() ||
39-
GV.getValueType() != GEP->getSourceElementType())
40-
return false;
41-
42-
LoadInst *Load = dyn_cast<LoadInst>(GEP->use_begin()->getUser());
43-
if (!Load || !Load->hasOneUse() ||
44-
Load->getType() != GEP->getResultElementType())
45-
return false;
46-
37+
//
4738
// If the original lookup table does not have local linkage and is
4839
// not dso_local, do not generate a relative lookup table.
4940
// This optimization creates a relative lookup table that consists of
5041
// offsets between the start of the lookup table and its elements.
5142
// To be able to generate these offsets, relative lookup table and
5243
// its elements should have internal linkage and be dso_local, which means
5344
// that they should resolve to symbols within the same linkage unit.
54-
if (!GV.hasLocalLinkage() ||
55-
!GV.isDSOLocal() ||
56-
!GV.isImplicitDSOLocal())
45+
if (!GV.hasInitializer() || !GV.isConstant() || !GV.hasOneUse() ||
46+
!GV.hasLocalLinkage() || !GV.isDSOLocal() || !GV.isImplicitDSOLocal())
5747
return false;
5848

59-
ConstantArray *Array = dyn_cast<ConstantArray>(GV.getInitializer());
60-
if (!Array)
49+
auto *GEP = dyn_cast<GetElementPtrInst>(GV.use_begin()->getUser());
50+
if (!GEP || !GEP->hasOneUse())
51+
return false;
52+
53+
auto *Load = dyn_cast<LoadInst>(GEP->use_begin()->getUser());
54+
if (!Load || !Load->hasOneUse())
6155
return false;
6256

6357
// If values are not 64-bit pointers, do not generate a relative lookup table.
6458
const DataLayout &DL = M.getDataLayout();
65-
Type *ElemType = Array->getType()->getElementType();
59+
Type *ElemType = Load->getType();
6660
if (!ElemType->isPointerTy() || DL.getPointerTypeSizeInBits(ElemType) != 64)
6761
return false;
6862

63+
// Make sure this is a gep of the form GV + scale*var.
64+
unsigned IndexWidth =
65+
DL.getIndexTypeSizeInBits(Load->getPointerOperand()->getType());
66+
SmallMapVector<Value *, APInt, 4> VarOffsets;
67+
APInt ConstOffset(IndexWidth, 0);
68+
if (!GEP->collectOffset(DL, IndexWidth, VarOffsets, ConstOffset) ||
69+
!ConstOffset.isZero() || VarOffsets.size() != 1)
70+
return false;
71+
72+
// This can't be a pointer lookup table if the stride is smaller than a
73+
// pointer.
74+
Info.Index = VarOffsets.front().first;
75+
const APInt &Stride = VarOffsets.front().second;
76+
if (Stride.ult(DL.getTypeStoreSize(ElemType)))
77+
return false;
78+
6979
SmallVector<GlobalVariable *, 4> GVOps;
7080
Triple TT = M.getTargetTriple();
7181
// FIXME: This should be removed in the future.
@@ -80,14 +90,20 @@ static bool shouldConvertToRelLookupTable(Module &M, GlobalVariable &GV) {
8090
// https://github.com/rust-lang/rust/issues/141306.
8191
|| (TT.isX86() && TT.isOSDarwin());
8292

83-
for (const Use &Op : Array->operands()) {
84-
Constant *ConstOp = cast<Constant>(&Op);
93+
APInt Offset(IndexWidth, 0);
94+
uint64_t GVSize = DL.getTypeAllocSize(GV.getValueType());
95+
for (; Offset.ult(GVSize); Offset += Stride) {
96+
Constant *C =
97+
ConstantFoldLoadFromConst(GV.getInitializer(), ElemType, Offset, DL);
98+
if (!C)
99+
return false;
100+
85101
GlobalValue *GVOp;
86-
APInt Offset;
102+
APInt GVOffset;
87103

88104
// If an operand is not a constant offset from a lookup table,
89105
// do not generate a relative lookup table.
90-
if (!IsConstantOffsetFromGlobal(ConstOp, GVOp, Offset, DL))
106+
if (!IsConstantOffsetFromGlobal(C, GVOp, GVOffset, DL))
91107
return false;
92108

93109
// If operand is mutable, do not generate a relative lookup table.
@@ -102,6 +118,8 @@ static bool shouldConvertToRelLookupTable(Module &M, GlobalVariable &GV) {
102118

103119
if (ShouldDropUnnamedAddr)
104120
GVOps.push_back(GlovalVarOp);
121+
122+
Info.Ptrs.push_back(C);
105123
}
106124

107125
if (ShouldDropUnnamedAddr)
@@ -111,14 +129,12 @@ static bool shouldConvertToRelLookupTable(Module &M, GlobalVariable &GV) {
111129
return true;
112130
}
113131

114-
static GlobalVariable *createRelLookupTable(Function &Func,
132+
static GlobalVariable *createRelLookupTable(LookupTableInfo &Info,
133+
Function &Func,
115134
GlobalVariable &LookupTable) {
116135
Module &M = *Func.getParent();
117-
ConstantArray *LookupTableArr =
118-
cast<ConstantArray>(LookupTable.getInitializer());
119-
unsigned NumElts = LookupTableArr->getType()->getNumElements();
120136
ArrayType *IntArrayTy =
121-
ArrayType::get(Type::getInt32Ty(M.getContext()), NumElts);
137+
ArrayType::get(Type::getInt32Ty(M.getContext()), Info.Ptrs.size());
122138

123139
GlobalVariable *RelLookupTable = new GlobalVariable(
124140
M, IntArrayTy, LookupTable.isConstant(), LookupTable.getLinkage(),
@@ -127,10 +143,9 @@ static GlobalVariable *createRelLookupTable(Function &Func,
127143
LookupTable.isExternallyInitialized());
128144

129145
uint64_t Idx = 0;
130-
SmallVector<Constant *, 64> RelLookupTableContents(NumElts);
146+
SmallVector<Constant *, 64> RelLookupTableContents(Info.Ptrs.size());
131147

132-
for (Use &Operand : LookupTableArr->operands()) {
133-
Constant *Element = cast<Constant>(Operand);
148+
for (Constant *Element : Info.Ptrs) {
134149
Type *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());
135150
Constant *Base = llvm::ConstantExpr::getPtrToInt(RelLookupTable, IntPtrTy);
136151
Constant *Target = llvm::ConstantExpr::getPtrToInt(Element, IntPtrTy);
@@ -148,7 +163,8 @@ static GlobalVariable *createRelLookupTable(Function &Func,
148163
return RelLookupTable;
149164
}
150165

151-
static void convertToRelLookupTable(GlobalVariable &LookupTable) {
166+
static void convertToRelLookupTable(LookupTableInfo &Info,
167+
GlobalVariable &LookupTable) {
152168
GetElementPtrInst *GEP =
153169
cast<GetElementPtrInst>(LookupTable.use_begin()->getUser());
154170
LoadInst *Load = cast<LoadInst>(GEP->use_begin()->getUser());
@@ -159,21 +175,21 @@ static void convertToRelLookupTable(GlobalVariable &LookupTable) {
159175
Function &Func = *BB->getParent();
160176

161177
// Generate an array that consists of relative offsets.
162-
GlobalVariable *RelLookupTable = createRelLookupTable(Func, LookupTable);
178+
GlobalVariable *RelLookupTable =
179+
createRelLookupTable(Info, Func, LookupTable);
163180

164181
// Place new instruction sequence before GEP.
165182
Builder.SetInsertPoint(GEP);
166-
Value *Index = GEP->getOperand(2);
167-
IntegerType *IntTy = cast<IntegerType>(Index->getType());
168-
Value *Offset =
169-
Builder.CreateShl(Index, ConstantInt::get(IntTy, 2), "reltable.shift");
183+
IntegerType *IntTy = cast<IntegerType>(Info.Index->getType());
184+
Value *Offset = Builder.CreateShl(Info.Index, ConstantInt::get(IntTy, 2),
185+
"reltable.shift");
170186

171187
// Insert the call to load.relative intrinsic before LOAD.
172188
// GEP might not be immediately followed by a LOAD, like it can be hoisted
173189
// outside the loop or another instruction might be inserted them in between.
174190
Builder.SetInsertPoint(Load);
175191
Function *LoadRelIntrinsic = llvm::Intrinsic::getOrInsertDeclaration(
176-
&M, Intrinsic::load_relative, {Index->getType()});
192+
&M, Intrinsic::load_relative, {Info.Index->getType()});
177193

178194
// Create a call to load.relative intrinsic that computes the target address
179195
// by adding base address (lookup table address) and relative offset.
@@ -205,10 +221,11 @@ static bool convertToRelativeLookupTables(
205221
bool Changed = false;
206222

207223
for (GlobalVariable &GV : llvm::make_early_inc_range(M.globals())) {
208-
if (!shouldConvertToRelLookupTable(M, GV))
224+
LookupTableInfo Info;
225+
if (!shouldConvertToRelLookupTable(Info, M, GV))
209226
continue;
210227

211-
convertToRelLookupTable(GV);
228+
convertToRelLookupTable(Info, GV);
212229

213230
// Remove the original lookup table.
214231
GV.eraseFromParent();

‎llvm/test/Transforms/RelLookupTableConverter/X86/relative_lookup_table.ll‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,35 @@ target triple = "x86_64-unknown-linux-gnu"
6969
ptr @.str.9
7070
], align 16
7171

72+
@table3 = internal constant [2 x ptr] [
73+
ptr @.str.8,
74+
ptr @.str.9
75+
], align 16
76+
77+
@table4 = internal constant [2 x ptr] [
78+
ptr @.str.8,
79+
ptr @.str.9
80+
], align 16
81+
82+
@table5 = internal constant [2 x ptr] [
83+
ptr @.str.8,
84+
ptr @.str.9
85+
], align 16
86+
87+
@skip.table = internal constant [4 x ptr] [
88+
ptr @.str.8,
89+
ptr null,
90+
ptr @.str.9,
91+
ptr null
92+
], align 16
93+
94+
@wrong.skip.table = internal constant [4 x ptr] [
95+
ptr null,
96+
ptr @.str.8,
97+
ptr null,
98+
ptr @.str.9
99+
], align 16
100+
72101
;.
73102
; CHECK: @.str = private unnamed_addr constant [5 x i8] c"zero\00", align 1
74103
; CHECK: @.str.1 = private unnamed_addr constant [4 x i8] c"one\00", align 1
@@ -97,6 +126,11 @@ target triple = "x86_64-unknown-linux-gnu"
97126
; CHECK: @user_defined_lookup_table.table.rel = internal unnamed_addr constant [3 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @.str to i64), i64 ptrtoint (ptr @user_defined_lookup_table.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.1 to i64), i64 ptrtoint (ptr @user_defined_lookup_table.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.2 to i64), i64 ptrtoint (ptr @user_defined_lookup_table.table.rel to i64)) to i32)], align 4
98127
; CHECK: @table.rel = internal unnamed_addr constant [2 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.8 to i64), i64 ptrtoint (ptr @table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.9 to i64), i64 ptrtoint (ptr @table.rel to i64)) to i32)], align 4
99128
; CHECK: @table2.rel = internal unnamed_addr constant [2 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.8 to i64), i64 ptrtoint (ptr @table2.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.9 to i64), i64 ptrtoint (ptr @table2.rel to i64)) to i32)], align 4
129+
; CHECK: @table3.rel = internal unnamed_addr constant [2 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.8 to i64), i64 ptrtoint (ptr @table3.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.9 to i64), i64 ptrtoint (ptr @table3.rel to i64)) to i32)], align 4
130+
; CHECK: @table4 = internal constant [2 x ptr] [ptr @.str.8, ptr @.str.9], align 16
131+
; CHECK: @table5 = internal constant [2 x ptr] [ptr @.str.8, ptr @.str.9], align 16
132+
; CHECK: @skip.table.rel = internal unnamed_addr constant [2 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.8 to i64), i64 ptrtoint (ptr @skip.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @.str.9 to i64), i64 ptrtoint (ptr @skip.table.rel to i64)) to i32)], align 4
133+
; CHECK: @wrong.skip.table = internal constant [4 x ptr] [ptr null, ptr @.str.8, ptr null, ptr @.str.9], align 16
100134
;.
101135
define ptr @external_linkage(i32 %cond) {
102136
; CHECK-LABEL: define ptr @external_linkage(
@@ -323,6 +357,69 @@ entry:
323357
ret ptr %1
324358
}
325359

360+
define ptr @gep_no_leading_zero(i64 %index) {
361+
; CHECK-LABEL: define ptr @gep_no_leading_zero(
362+
; CHECK-SAME: i64 [[INDEX:%.*]]) {
363+
; CHECK-NEXT: [[RELTABLE_SHIFT:%.*]] = shl i64 [[INDEX]], 2
364+
; CHECK-NEXT: [[LOAD:%.*]] = call ptr @llvm.load.relative.i64(ptr @table3.rel, i64 [[RELTABLE_SHIFT]])
365+
; CHECK-NEXT: ret ptr [[LOAD]]
366+
;
367+
%gep = getelementptr ptr, ptr @table3, i64 %index
368+
%load = load ptr, ptr %gep
369+
ret ptr %load
370+
}
371+
372+
define ptr @gep_wrong_stride(i64 %index) {
373+
; CHECK-LABEL: define ptr @gep_wrong_stride(
374+
; CHECK-SAME: i64 [[INDEX:%.*]]) {
375+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr @table4, i64 [[INDEX]]
376+
; CHECK-NEXT: [[LOAD:%.*]] = load ptr, ptr [[GEP]], align 8
377+
; CHECK-NEXT: ret ptr [[LOAD]]
378+
;
379+
%gep = getelementptr i8, ptr @table4, i64 %index
380+
%load = load ptr, ptr %gep
381+
ret ptr %load
382+
}
383+
384+
define ptr @gep_wrong_constant_offset(i64 %index) {
385+
; CHECK-LABEL: define ptr @gep_wrong_constant_offset(
386+
; CHECK-SAME: i64 [[INDEX:%.*]]) {
387+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr { ptr, i32 }, ptr @table5, i64 [[INDEX]], i32 1
388+
; CHECK-NEXT: [[LOAD:%.*]] = load ptr, ptr [[GEP]], align 8
389+
; CHECK-NEXT: ret ptr [[LOAD]]
390+
;
391+
%gep = getelementptr { ptr, i32 }, ptr @table5, i64 %index, i32 1
392+
%load = load ptr, ptr %gep
393+
ret ptr %load
394+
}
395+
396+
; This is intentionally advancing by two pointers.
397+
define ptr @table_with_skipped_elements(i64 %index) {
398+
; CHECK-LABEL: define ptr @table_with_skipped_elements(
399+
; CHECK-SAME: i64 [[INDEX:%.*]]) {
400+
; CHECK-NEXT: [[RELTABLE_SHIFT:%.*]] = shl i64 [[INDEX]], 2
401+
; CHECK-NEXT: [[LOAD:%.*]] = call ptr @llvm.load.relative.i64(ptr @skip.table.rel, i64 [[RELTABLE_SHIFT]])
402+
; CHECK-NEXT: ret ptr [[LOAD]]
403+
;
404+
%gep = getelementptr [2 x ptr], ptr @skip.table, i64 %index
405+
%load = load ptr, ptr %gep
406+
ret ptr %load
407+
}
408+
409+
; Same as previous test, but the elements are at the wrong position in the
410+
; table.
411+
define ptr @table_with_skipped_elements_wrong(i64 %index) {
412+
; CHECK-LABEL: define ptr @table_with_skipped_elements_wrong(
413+
; CHECK-SAME: i64 [[INDEX:%.*]]) {
414+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr [2 x ptr], ptr @wrong.skip.table, i64 [[INDEX]]
415+
; CHECK-NEXT: [[LOAD:%.*]] = load ptr, ptr [[GEP]], align 8
416+
; CHECK-NEXT: ret ptr [[LOAD]]
417+
;
418+
%gep = getelementptr [2 x ptr], ptr @wrong.skip.table, i64 %index
419+
%load = load ptr, ptr %gep
420+
ret ptr %load
421+
}
422+
326423
!llvm.module.flags = !{!0, !1}
327424
!0 = !{i32 7, !"PIC Level", i32 2}
328425
!1 = !{i32 1, !"Code Model", i32 1}

0 commit comments

Comments
 (0)