Skip to content

Commit 1ae4d37

Browse files
SamTebbs33SamTebbs33
committed
Use SmallPtrSet and remove placeholder map
1 parent 73559dd commit 1ae4d37

File tree

3 files changed

+128
-7
lines changed

3 files changed

+128
-7
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3041,7 +3041,7 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
30413041
}
30423042

30433043
~VPExpressionRecipe() override {
3044-
SmallSet<VPSingleDefRecipe *, 4> ExpressionRecipesSeen;
3044+
SmallPtrSet<VPSingleDefRecipe *, 4> ExpressionRecipesSeen;
30453045
for (auto *R : reverse(ExpressionRecipes)) {
30463046
if (ExpressionRecipesSeen.insert(R).second)
30473047
delete R;

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2795,22 +2795,19 @@ VPExpressionRecipe::VPExpressionRecipe(
27952795
// the expression. The original operands are added as operands of the
27962796
// VPExpressionRecipe itself.
27972797

2798-
SmallMapVector<VPValue *, VPValue *, 4> OperandPlaceholders;
27992798
for (auto *R : ExpressionRecipes) {
28002799
for (const auto &[Idx, Op] : enumerate(R->operands())) {
28012800
auto *Def = Op->getDefiningRecipe();
28022801
if (Def && ExpressionRecipesAsSetOfUsers.contains(Def))
28032802
continue;
28042803
addOperand(Op);
2805-
VPValue *Tmp = new VPValue();
2806-
OperandPlaceholders[Op] = Tmp;
2807-
LiveInPlaceholders.push_back(Tmp);
2804+
LiveInPlaceholders.push_back(new VPValue());
28082805
}
28092806
}
28102807

28112808
for (auto *R : ExpressionRecipes)
2812-
for (auto &Entry : OperandPlaceholders)
2813-
R->replaceUsesOfWith(Entry.first, Entry.second);
2809+
for (auto const &[LiveIn, Tmp] : zip(operands(), LiveInPlaceholders))
2810+
R->replaceUsesOfWith(LiveIn, Tmp);
28142811
}
28152812

28162813
void VPExpressionRecipe::decompose() {

‎llvm/test/Transforms/LoopVectorize/reduction-inloop.ll‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3351,6 +3351,130 @@ for.end: ; preds = %for.body, %entry
33513351
ret i32 %x.0.lcssa
33523352
}
33533353

3354+
; Test that bundling recipes that share an operand into an expression works.
3355+
; In this case the two extends are the recipes that share an operand.
3356+
define i64 @reduction_expression_same_operands(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
3357+
; CHECK-LABEL: @reduction_expression_same_operands(
3358+
; CHECK-NEXT: entry:
3359+
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 4
3360+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
3361+
; CHECK: vector.ph:
3362+
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -4
3363+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
3364+
; CHECK: vector.body:
3365+
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
3366+
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
3367+
; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
3368+
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[TMP0]]
3369+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 4
3370+
; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>
3371+
; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>
3372+
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i64> [[TMP2]], [[TMP3]]
3373+
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
3374+
; CHECK-NEXT: [[TMP6]] = add i64 [[VEC_PHI]], [[TMP5]]
3375+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
3376+
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
3377+
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
3378+
; CHECK: middle.block:
3379+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
3380+
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
3381+
; CHECK: scalar.ph:
3382+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
3383+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
3384+
; CHECK-NEXT: br label [[LOOP:%.*]]
3385+
; CHECK: loop:
3386+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
3387+
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[RDX_NEXT:%.*]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
3388+
; CHECK-NEXT: [[TMP8:%.*]] = sext i32 [[IV]] to i64
3389+
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[TMP8]]
3390+
; CHECK-NEXT: [[LOAD0:%.*]] = load i16, ptr [[ARRAYIDX]], align 4
3391+
; CHECK-NEXT: [[CONV0:%.*]] = sext i16 [[LOAD0]] to i64
3392+
; CHECK-NEXT: [[CONV1:%.*]] = sext i16 [[LOAD0]] to i64
3393+
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV0]], [[CONV1]]
3394+
; CHECK-NEXT: [[RDX_NEXT]] = add nsw i64 [[RDX]], [[MUL]]
3395+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
3396+
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
3397+
; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP34:![0-9]+]]
3398+
; CHECK: exit:
3399+
; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ [[RDX_NEXT]], [[LOOP]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
3400+
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
3401+
;
3402+
; CHECK-INTERLEAVED-LABEL: @reduction_expression_same_operands(
3403+
; CHECK-INTERLEAVED-NEXT: entry:
3404+
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 8
3405+
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
3406+
; CHECK-INTERLEAVED: vector.ph:
3407+
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -8
3408+
; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]]
3409+
; CHECK-INTERLEAVED: vector.body:
3410+
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
3411+
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
3412+
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
3413+
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
3414+
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[TMP0]]
3415+
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 8
3416+
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 4
3417+
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i16>, ptr [[TMP2]], align 4
3418+
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>
3419+
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>
3420+
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = mul nsw <4 x i64> [[TMP3]], [[TMP4]]
3421+
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
3422+
; CHECK-INTERLEAVED-NEXT: [[TMP7]] = add i64 [[VEC_PHI]], [[TMP6]]
3423+
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = sext <4 x i16> [[WIDE_LOAD2]] to <4 x i64>
3424+
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[WIDE_LOAD2]] to <4 x i64>
3425+
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = mul nsw <4 x i64> [[TMP8]], [[TMP9]]
3426+
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP10]])
3427+
; CHECK-INTERLEAVED-NEXT: [[TMP12]] = add i64 [[VEC_PHI1]], [[TMP11]]
3428+
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
3429+
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
3430+
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
3431+
; CHECK-INTERLEAVED: middle.block:
3432+
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i64 [[TMP12]], [[TMP7]]
3433+
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
3434+
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
3435+
; CHECK-INTERLEAVED: scalar.ph:
3436+
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
3437+
; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
3438+
; CHECK-INTERLEAVED-NEXT: br label [[LOOP:%.*]]
3439+
; CHECK-INTERLEAVED: loop:
3440+
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
3441+
; CHECK-INTERLEAVED-NEXT: [[RDX:%.*]] = phi i64 [ [[RDX_NEXT:%.*]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
3442+
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = sext i32 [[IV]] to i64
3443+
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[TMP14]]
3444+
; CHECK-INTERLEAVED-NEXT: [[LOAD0:%.*]] = load i16, ptr [[ARRAYIDX]], align 4
3445+
; CHECK-INTERLEAVED-NEXT: [[CONV0:%.*]] = sext i16 [[LOAD0]] to i64
3446+
; CHECK-INTERLEAVED-NEXT: [[CONV1:%.*]] = sext i16 [[LOAD0]] to i64
3447+
; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV0]], [[CONV1]]
3448+
; CHECK-INTERLEAVED-NEXT: [[RDX_NEXT]] = add nsw i64 [[RDX]], [[MUL]]
3449+
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
3450+
; CHECK-INTERLEAVED-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
3451+
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP34:![0-9]+]]
3452+
; CHECK-INTERLEAVED: exit:
3453+
; CHECK-INTERLEAVED-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ [[RDX_NEXT]], [[LOOP]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
3454+
; CHECK-INTERLEAVED-NEXT: ret i64 [[R_0_LCSSA]]
3455+
;
3456+
entry:
3457+
br label %loop
3458+
3459+
loop:
3460+
%iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
3461+
%rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ]
3462+
%arrayidx = getelementptr inbounds i16, ptr %x, i32 %iv
3463+
%load0 = load i16, ptr %arrayidx, align 4
3464+
%conv0 = sext i16 %load0 to i32
3465+
%conv1 = sext i16 %load0 to i32
3466+
%mul = mul nsw i32 %conv0, %conv1
3467+
%conv = sext i32 %mul to i64
3468+
%rdx.next = add nsw i64 %rdx, %conv
3469+
%iv.next = add nuw nsw i32 %iv, 1
3470+
%exitcond = icmp eq i32 %iv.next, %n
3471+
br i1 %exitcond, label %exit, label %loop
3472+
3473+
exit:
3474+
%r.0.lcssa = phi i64 [ %rdx.next, %loop ]
3475+
ret i64 %r.0.lcssa
3476+
}
3477+
33543478
declare float @llvm.fmuladd.f32(float, float, float)
33553479

33563480
!6 = distinct !{!6, !7, !8}

0 commit comments

Comments
 (0)