diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index dfef6eb300230..6629208139c3e 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -5269,8 +5269,11 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I, Type *ValTy = getLoadStoreType(I); auto *VectorTy = cast(toVectorTy(ValTy, VF)); const Align Alignment = getLoadStoreAlignment(I); - const Value *Ptr = getLoadStorePointerOperand(I); - Type *PtrTy = toVectorTy(Ptr->getType(), VF); + Value *Ptr = getLoadStorePointerOperand(I); + Type *PtrTy = Ptr->getType(); + + if (!Legal->isUniform(Ptr, VF)) + PtrTy = toVectorTy(PtrTy, VF); return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) + TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr, diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll new file mode 100644 index 0000000000000..ec4429fa3cd40 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll @@ -0,0 +1,186 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+rva23u64 -S | FileCheck %s -check-prefixes=CHECK,RVA23 +; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+rva23u64,+zvl1024b -S | FileCheck %s -check-prefixes=CHECK,RVA23ZVL1024B + +define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) { +; CHECK-LABEL: @predicated_uniform_load( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[IBOX:%.*]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1 +; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0) +; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[SMAX2]] to i32 +; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[TMP2]], 1 +; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]] +; CHECK: vector.scevcheck: +; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[IBOX]] to i64 +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 1 +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 0) +; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[SMAX]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0 +; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[SMAX]], 4294967295 +; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] +; CHECK-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]] +; CHECK: vector.memcheck: +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[NBRBOXES:%.*]], i64 4 +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BOXES:%.*]], i64 4 +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[NBRBOXES]], [[SCEVGEP1]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[BOXES]], [[SCEVGEP]] +; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i1 [[COND:%.*]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer +; CHECK-NEXT: [[TMP13:%.*]] = xor [[BROADCAST_SPLAT1]], splat (i1 true) +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[BOXES]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement poison, ptr [[NBRBOXES]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector [[BROADCAST_SPLATINSERT3]], poison, zeroinitializer +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true) +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.vp.gather.nxv4i32.nxv4p0( align 4 [[BROADCAST_SPLAT]], [[TMP13]], i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[BROADCAST_SPLAT1]], zeroinitializer, [[WIDE_MASKED_GATHER]] +; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0( [[PREDPHI]], align 4 [[BROADCAST_SPLAT4]], splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]] +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP10]] +; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: br label [[EXIT:%.*]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] +; CHECK-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]] +; CHECK: loop.then: +; CHECK-NEXT: br label [[LOOP_LATCH]] +; CHECK: loop.else: +; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[BOXES]], align 4 +; CHECK-NEXT: br label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[STORE:%.*]] = phi i32 [ [[TMP17]], [[LOOP_ELSE]] ], [ 0, [[LOOP_THEN]] ] +; CHECK-NEXT: store i32 [[STORE]], ptr [[NBRBOXES]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV]], [[IBOX]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + br i1 %cond, label %loop.then, label %loop.else + +loop.then: + br label %loop.latch + +loop.else: + %0 = load i32, ptr %src, align 4 + br label %loop.latch + +loop.latch: + %store = phi i32 [%0, %loop.else], [0, %loop.then] + store i32 %store, ptr %dst, align 4 + %iv.next = add i32 %iv, 1 + %exitcond = icmp sgt i32 %iv, %n + br i1 %exitcond, label %exit, label %loop + +exit: + ret void +} + +define void @predicated_strided_store(ptr %start) { +; RVA23-LABEL: @predicated_strided_store( +; RVA23-NEXT: entry: +; RVA23-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; RVA23: vector.ph: +; RVA23-NEXT: [[TMP0:%.*]] = call @llvm.stepvector.nxv16i64() +; RVA23-NEXT: [[TMP1:%.*]] = mul [[TMP0]], splat (i64 1) +; RVA23-NEXT: [[INDUCTION:%.*]] = add zeroinitializer, [[TMP1]] +; RVA23-NEXT: br label [[VECTOR_BODY:%.*]] +; RVA23: vector.body: +; RVA23-NEXT: [[VEC_IND:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; RVA23-NEXT: [[AVL:%.*]] = phi i64 [ 586, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] +; RVA23-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true) +; RVA23-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64 +; RVA23-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[TMP3]], i64 0 +; RVA23-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; RVA23-NEXT: [[TMP4:%.*]] = mul [[VEC_IND]], splat (i64 7) +; RVA23-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START:%.*]], [[TMP4]] +; RVA23-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( zeroinitializer, align 1 [[TMP5]], splat (i1 true), i32 [[TMP2]]) +; RVA23-NEXT: [[TMP6:%.*]] = zext i32 [[TMP2]] to i64 +; RVA23-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]] +; RVA23-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[BROADCAST_SPLAT]] +; RVA23-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; RVA23-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; RVA23: middle.block: +; RVA23-NEXT: br label [[EXIT:%.*]] +; RVA23: scalar.ph: +; RVA23-NEXT: br label [[LOOP:%.*]] +; RVA23: loop: +; RVA23-NEXT: [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] +; RVA23-NEXT: [[TMP8:%.*]] = mul i64 [[IV]], 7 +; RVA23-NEXT: [[ADD_PTR:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]] +; RVA23-NEXT: store i8 0, ptr [[ADD_PTR]], align 1 +; RVA23-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; RVA23-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 585 +; RVA23-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]] +; RVA23: exit: +; RVA23-NEXT: ret void +; +; RVA23ZVL1024B-LABEL: @predicated_strided_store( +; RVA23ZVL1024B-NEXT: entry: +; RVA23ZVL1024B-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; RVA23ZVL1024B: vector.ph: +; RVA23ZVL1024B-NEXT: [[TMP0:%.*]] = call @llvm.stepvector.nxv2i64() +; RVA23ZVL1024B-NEXT: [[TMP1:%.*]] = mul [[TMP0]], splat (i64 1) +; RVA23ZVL1024B-NEXT: [[INDUCTION:%.*]] = add zeroinitializer, [[TMP1]] +; RVA23ZVL1024B-NEXT: br label [[VECTOR_BODY:%.*]] +; RVA23ZVL1024B: vector.body: +; RVA23ZVL1024B-NEXT: [[VEC_IND:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; RVA23ZVL1024B-NEXT: [[AVL:%.*]] = phi i64 [ 586, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] +; RVA23ZVL1024B-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) +; RVA23ZVL1024B-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64 +; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[TMP3]], i64 0 +; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; RVA23ZVL1024B-NEXT: [[TMP4:%.*]] = mul [[VEC_IND]], splat (i64 7) +; RVA23ZVL1024B-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START:%.*]], [[TMP4]] +; RVA23ZVL1024B-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0( zeroinitializer, align 1 [[TMP5]], splat (i1 true), i32 [[TMP2]]) +; RVA23ZVL1024B-NEXT: [[TMP6:%.*]] = zext i32 [[TMP2]] to i64 +; RVA23ZVL1024B-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]] +; RVA23ZVL1024B-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[BROADCAST_SPLAT]] +; RVA23ZVL1024B-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; RVA23ZVL1024B-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; RVA23ZVL1024B: middle.block: +; RVA23ZVL1024B-NEXT: br label [[EXIT:%.*]] +; RVA23ZVL1024B: scalar.ph: +; RVA23ZVL1024B-NEXT: br label [[LOOP:%.*]] +; RVA23ZVL1024B: loop: +; RVA23ZVL1024B-NEXT: [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] +; RVA23ZVL1024B-NEXT: [[TMP8:%.*]] = mul i64 [[IV]], 7 +; RVA23ZVL1024B-NEXT: [[ADD_PTR:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]] +; RVA23ZVL1024B-NEXT: store i8 0, ptr [[ADD_PTR]], align 1 +; RVA23ZVL1024B-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; RVA23ZVL1024B-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 585 +; RVA23ZVL1024B-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]] +; RVA23ZVL1024B: exit: +; RVA23ZVL1024B-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %0 = mul i64 %iv, 7 + %add.ptr = getelementptr i8, ptr %start, i64 %0 + store i8 0, ptr %add.ptr, align 1 + %iv.next = add i64 %iv, 1 + %exitcond = icmp eq i64 %iv, 585 + br i1 %exitcond, label %exit, label %loop + +exit: + ret void +}