Skip to content

Commit a216702

Browse files
nikicnikic
authored
[InstCombine] Merge one-use GEP offsets during expansion (#147263)
When expanding a GEP chain, if there is a chain of one-use GEPs followed by a multi-use GEP, rewrite the multi-use GEP to include the one-use GEPs offsets. This means the offsets from the one-use GEPs can be reused by the offset expansion without additional cost (from computing them again with a different reassociation).
1 parent 4127458 commit a216702

File tree

3 files changed

+98
-38
lines changed

3 files changed

+98
-38
lines changed

‎llvm/lib/Transforms/InstCombine/InstructionCombining.cpp

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -219,18 +219,64 @@ Value *InstCombinerImpl::EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP) {
219219
Value *InstCombinerImpl::EmitGEPOffsets(ArrayRef<GEPOperator *> GEPs,
220220
GEPNoWrapFlags NW, Type *IdxTy,
221221
bool RewriteGEPs) {
222-
Value *Sum = nullptr;
223-
for (GEPOperator *GEP : reverse(GEPs)) {
224-
Value *Offset = EmitGEPOffset(GEP, RewriteGEPs);
225-
if (Offset->getType() != IdxTy)
226-
Offset = Builder.CreateVectorSplat(
227-
cast<VectorType>(IdxTy)->getElementCount(), Offset);
222+
auto Add = [&](Value *Sum, Value *Offset) -> Value * {
228223
if (Sum)
229-
Sum = Builder.CreateAdd(Sum, Offset, "", NW.hasNoUnsignedWrap(),
230-
NW.isInBounds());
224+
return Builder.CreateAdd(Sum, Offset, "", NW.hasNoUnsignedWrap(),
225+
NW.isInBounds());
231226
else
232-
Sum = Offset;
227+
return Offset;
228+
};
229+
230+
Value *Sum = nullptr;
231+
Value *OneUseSum = nullptr;
232+
Value *OneUseBase = nullptr;
233+
GEPNoWrapFlags OneUseFlags = GEPNoWrapFlags::all();
234+
for (GEPOperator *GEP : reverse(GEPs)) {
235+
Value *Offset;
236+
{
237+
// Expand the offset at the point of the previous GEP to enable rewriting.
238+
// However, use the original insertion point for calculating Sum.
239+
IRBuilderBase::InsertPointGuard Guard(Builder);
240+
auto *Inst = dyn_cast<Instruction>(GEP);
241+
if (RewriteGEPs && Inst)
242+
Builder.SetInsertPoint(Inst);
243+
244+
Offset = llvm::emitGEPOffset(&Builder, DL, GEP);
245+
if (Offset->getType() != IdxTy)
246+
Offset = Builder.CreateVectorSplat(
247+
cast<VectorType>(IdxTy)->getElementCount(), Offset);
248+
if (GEP->hasOneUse()) {
249+
// Offsets of one-use GEPs will be merged into the next multi-use GEP.
250+
OneUseSum = Add(OneUseSum, Offset);
251+
OneUseFlags = OneUseFlags.intersectForOffsetAdd(GEP->getNoWrapFlags());
252+
if (!OneUseBase)
253+
OneUseBase = GEP->getPointerOperand();
254+
continue;
255+
}
256+
257+
if (OneUseSum)
258+
Offset = Add(OneUseSum, Offset);
259+
260+
// Rewrite the GEP to reuse the computed offset. This also includes
261+
// offsets from preceding one-use GEPs.
262+
if (RewriteGEPs && Inst &&
263+
!(GEP->getSourceElementType()->isIntegerTy(8) &&
264+
GEP->getOperand(1) == Offset)) {
265+
replaceInstUsesWith(
266+
*Inst,
267+
Builder.CreatePtrAdd(
268+
OneUseBase ? OneUseBase : GEP->getPointerOperand(), Offset, "",
269+
OneUseFlags.intersectForOffsetAdd(GEP->getNoWrapFlags())));
270+
eraseInstFromFunction(*Inst);
271+
}
272+
}
273+
274+
Sum = Add(Sum, Offset);
275+
OneUseSum = OneUseBase = nullptr;
276+
OneUseFlags = GEPNoWrapFlags::all();
233277
}
278+
if (OneUseSum)
279+
Sum = Add(Sum, OneUseSum);
234280
if (!Sum)
235281
return Constant::getNullValue(IdxTy);
236282
return Sum;

‎llvm/test/Transforms/InstCombine/getelementptr.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -682,15 +682,15 @@ define i32 @test28() nounwind {
682682
; CHECK-NEXT: entry:
683683
; CHECK-NEXT: [[ORIENTATIONS:%.*]] = alloca [1 x [1 x %struct.x]], align 8
684684
; CHECK-NEXT: [[T3:%.*]] = call i32 @puts(ptr noundef nonnull dereferenceable(1) @.str) #[[ATTR0]]
685-
; CHECK-NEXT: [[T45:%.*]] = getelementptr inbounds nuw i8, ptr [[ORIENTATIONS]], i64 1
686685
; CHECK-NEXT: br label [[BB10:%.*]]
687686
; CHECK: bb10:
688687
; CHECK-NEXT: [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[BB10]] ]
689688
; CHECK-NEXT: [[T12_REC:%.*]] = xor i32 [[INDVAR]], -1
690689
; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[T12_REC]] to i64
691-
; CHECK-NEXT: [[T12:%.*]] = getelementptr inbounds i8, ptr [[T45]], i64 [[TMP0]]
690+
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
691+
; CHECK-NEXT: [[T12:%.*]] = getelementptr inbounds i8, ptr [[ORIENTATIONS]], i64 [[TMP1]]
692692
; CHECK-NEXT: [[T16:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str1, ptr nonnull [[T12]]) #[[ATTR0]]
693-
; CHECK-NEXT: [[T84:%.*]] = icmp eq i32 [[INDVAR]], 0
693+
; CHECK-NEXT: [[T84:%.*]] = icmp eq i64 [[TMP1]], 0
694694
; CHECK-NEXT: [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
695695
; CHECK-NEXT: br i1 [[T84]], label [[BB17:%.*]], label [[BB10]]
696696
; CHECK: bb17:

‎llvm/test/Transforms/InstCombine/sub-gep.ll

Lines changed: 40 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -945,19 +945,15 @@ define i64 @multiple_geps_two_chains_gep_base(ptr %base, i64 %base.idx, i64 %idx
945945

946946
define i64 @multiple_geps_two_chains_multi_use(ptr %base, i64 %idx1, i64 %idx2, i64 %idx3, i64 %idx4) {
947947
; CHECK-LABEL: @multiple_geps_two_chains_multi_use(
948-
; CHECK-NEXT: [[P2_IDX:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
949-
; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 [[P2_IDX]]
950-
; CHECK-NEXT: [[P4_IDX:%.*]] = shl nsw i64 [[IDX4:%.*]], 2
951-
; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX]]
952-
; CHECK-NEXT: [[P3_IDX:%.*]] = shl nsw i64 [[IDX3:%.*]], 2
953-
; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[P3_IDX]]
954-
; CHECK-NEXT: [[P4_IDX1:%.*]] = shl nsw i64 [[IDX5:%.*]], 2
955-
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[P4_IDX1]]
948+
; CHECK-NEXT: [[P1_IDX1:%.*]] = add i64 [[IDX1:%.*]], [[IDX2:%.*]]
949+
; CHECK-NEXT: [[P4_IDX:%.*]] = shl i64 [[P1_IDX1]], 2
950+
; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 [[P4_IDX]]
951+
; CHECK-NEXT: [[P3_IDX2:%.*]] = add i64 [[IDX3:%.*]], [[IDX4:%.*]]
952+
; CHECK-NEXT: [[P4_IDX1:%.*]] = shl i64 [[P3_IDX2]], 2
953+
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX1]]
956954
; CHECK-NEXT: call void @use(ptr [[P5]])
957955
; CHECK-NEXT: call void @use(ptr [[P4]])
958-
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[P2_IDX]], [[P4_IDX]]
959-
; CHECK-NEXT: [[TMP2:%.*]] = add nsw i64 [[P3_IDX]], [[P4_IDX1]]
960-
; CHECK-NEXT: [[GEPDIFF:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
956+
; CHECK-NEXT: [[GEPDIFF:%.*]] = sub nsw i64 [[P4_IDX]], [[P4_IDX1]]
961957
; CHECK-NEXT: ret i64 [[GEPDIFF]]
962958
;
963959
%p1 = getelementptr inbounds i32, ptr %base, i64 %idx1
@@ -974,23 +970,18 @@ define i64 @multiple_geps_two_chains_multi_use(ptr %base, i64 %idx1, i64 %idx2,
974970

975971
define i64 @multiple_geps_two_chains_partial_multi_use(ptr %base, i64 %idx1, i64 %idx2, i64 %idx3, i64 %idx4, i64 %idx5, i64 %idx6) {
976972
; CHECK-LABEL: @multiple_geps_two_chains_partial_multi_use(
977-
; CHECK-NEXT: [[P2_IDX:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
978-
; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 [[P2_IDX]]
979-
; CHECK-NEXT: [[P4_IDX:%.*]] = shl nsw i64 [[IDX4:%.*]], 2
980-
; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX]]
981-
; CHECK-NEXT: [[P3_IDX:%.*]] = shl nsw i64 [[IDX3:%.*]], 2
982-
; CHECK-NEXT: [[P4_IDX1:%.*]] = shl nsw i64 [[IDX7:%.*]], 2
983-
; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[P4_IDX1]]
984-
; CHECK-NEXT: [[P5_IDX:%.*]] = shl nsw i64 [[IDX5:%.*]], 2
985-
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P5]], i64 [[P5_IDX]]
986-
; CHECK-NEXT: [[P6_IDX:%.*]] = shl nsw i64 [[IDX6:%.*]], 2
973+
; CHECK-NEXT: [[P1_IDX1:%.*]] = add i64 [[IDX1:%.*]], [[IDX2:%.*]]
974+
; CHECK-NEXT: [[P4_IDX:%.*]] = shl i64 [[P1_IDX1]], 2
975+
; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 [[P4_IDX]]
976+
; CHECK-NEXT: [[P4_IDX2:%.*]] = add i64 [[IDX4:%.*]], [[IDX5:%.*]]
977+
; CHECK-NEXT: [[P5_IDX:%.*]] = shl i64 [[P4_IDX2]], 2
978+
; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P5_IDX]]
987979
; CHECK-NEXT: call void @use(ptr [[P3]])
988980
; CHECK-NEXT: call void @use(ptr [[P4]])
989-
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[P2_IDX]], [[P4_IDX]]
990-
; CHECK-NEXT: [[TMP2:%.*]] = add nsw i64 [[TMP1]], [[P3_IDX]]
991-
; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[P4_IDX1]], [[P5_IDX]]
992-
; CHECK-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP3]], [[P6_IDX]]
993-
; CHECK-NEXT: [[GEPDIFF:%.*]] = sub nsw i64 [[TMP2]], [[TMP4]]
981+
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[P1_IDX1]], [[IDX3:%.*]]
982+
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[P4_IDX2]], [[IDX6:%.*]]
983+
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP3]], [[TMP4]]
984+
; CHECK-NEXT: [[GEPDIFF:%.*]] = shl i64 [[TMP5]], 2
994985
; CHECK-NEXT: ret i64 [[GEPDIFF]]
995986
;
996987
%p1 = getelementptr inbounds i32, ptr %base, i64 %idx1
@@ -1007,6 +998,29 @@ define i64 @multiple_geps_two_chains_partial_multi_use(ptr %base, i64 %idx1, i64
1007998
ret i64 %d
1008999
}
10091000

1001+
define i64 @multiple_geps_two_chains_partial_multi_use_insert_point(ptr %p, i64 %idx1, i64 %idx2, i64 %idx3) {
1002+
; CHECK-LABEL: @multiple_geps_two_chains_partial_multi_use_insert_point(
1003+
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 8
1004+
; CHECK-NEXT: call void @use(ptr [[GEP2]])
1005+
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[IDX2:%.*]], [[IDX3:%.*]]
1006+
; CHECK-NEXT: [[GEP4:%.*]] = getelementptr i8, ptr [[GEP2]], i64 [[TMP1]]
1007+
; CHECK-NEXT: call void @use(ptr [[GEP4]])
1008+
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 8
1009+
; CHECK-NEXT: [[GEPDIFF:%.*]] = sub i64 [[IDX1:%.*]], [[TMP2]]
1010+
; CHECK-NEXT: ret i64 [[GEPDIFF]]
1011+
;
1012+
%gep1 = getelementptr i8, ptr %p, i64 %idx1
1013+
%gep2 = getelementptr i8, ptr %p, i64 8
1014+
call void @use(ptr %gep2)
1015+
%gep3 = getelementptr i8, ptr %gep2, i64 %idx2
1016+
%gep4 = getelementptr i8, ptr %gep3, i64 %idx3
1017+
call void @use(ptr %gep4)
1018+
%gep1.int = ptrtoint ptr %gep1 to i64
1019+
%gep4.int = ptrtoint ptr %gep4 to i64
1020+
%sub = sub i64 %gep1.int, %gep4.int
1021+
ret i64 %sub
1022+
}
1023+
10101024
define i64 @multiple_geps_inbounds(ptr %base, i64 %idx, i64 %idx2) {
10111025
; CHECK-LABEL: @multiple_geps_inbounds(
10121026
; CHECK-NEXT: [[D:%.*]] = add nsw i64 [[IDX:%.*]], [[IDX2:%.*]]

0 commit comments

Comments
 (0)