Skip to content

Commit 7f152fa

Browse files
ritter-x2aritter-x2a
committed
[LowerMemIntrinsics][NFC] Use Align in TTI::getMemcpyLoopLoweringType (llvm#100984)
...and also in TTI::getMemcpyLoopResidualLoweringType. (cherry picked from commit 9e462b7) Change-Id: Iee645ead746236ec28d31742f3eb4933af0c7747
1 parent 9fe6008 commit 7f152fa

File tree

6 files changed

+34
-35
lines changed

6 files changed

+34
-35
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1581,7 +1581,7 @@ class TargetTransformInfo {
15811581
/// \returns The type to use in a loop expansion of a memcpy call.
15821582
Type *getMemcpyLoopLoweringType(
15831583
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1584-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
1584+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
15851585
std::optional<uint32_t> AtomicElementSize = std::nullopt) const;
15861586

15871587
/// \param[out] OpsOut The operand types to copy RemainingBytes of memory.
@@ -1593,7 +1593,7 @@ class TargetTransformInfo {
15931593
void getMemcpyLoopResidualLoweringType(
15941594
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
15951595
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1596-
unsigned SrcAlign, unsigned DestAlign,
1596+
Align SrcAlign, Align DestAlign,
15971597
std::optional<uint32_t> AtomicCpySize = std::nullopt) const;
15981598

15991599
/// \returns True if the two functions have compatible attributes for inlining
@@ -2126,13 +2126,13 @@ class TargetTransformInfo::Concept {
21262126
Type *ExpectedType) = 0;
21272127
virtual Type *getMemcpyLoopLoweringType(
21282128
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
2129-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
2129+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
21302130
std::optional<uint32_t> AtomicElementSize) const = 0;
21312131

21322132
virtual void getMemcpyLoopResidualLoweringType(
21332133
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
21342134
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
2135-
unsigned SrcAlign, unsigned DestAlign,
2135+
Align SrcAlign, Align DestAlign,
21362136
std::optional<uint32_t> AtomicCpySize) const = 0;
21372137
virtual bool areInlineCompatible(const Function *Caller,
21382138
const Function *Callee) const = 0;
@@ -2828,7 +2828,7 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
28282828
}
28292829
Type *getMemcpyLoopLoweringType(
28302830
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
2831-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
2831+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
28322832
std::optional<uint32_t> AtomicElementSize) const override {
28332833
return Impl.getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
28342834
DestAddrSpace, SrcAlign, DestAlign,
@@ -2837,7 +2837,7 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
28372837
void getMemcpyLoopResidualLoweringType(
28382838
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
28392839
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
2840-
unsigned SrcAlign, unsigned DestAlign,
2840+
Align SrcAlign, Align DestAlign,
28412841
std::optional<uint32_t> AtomicCpySize) const override {
28422842
Impl.getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes,
28432843
SrcAddrSpace, DestAddrSpace,

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -837,7 +837,7 @@ class TargetTransformInfoImplBase {
837837
Type *
838838
getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
839839
unsigned SrcAddrSpace, unsigned DestAddrSpace,
840-
unsigned SrcAlign, unsigned DestAlign,
840+
Align SrcAlign, Align DestAlign,
841841
std::optional<uint32_t> AtomicElementSize) const {
842842
return AtomicElementSize ? Type::getIntNTy(Context, *AtomicElementSize * 8)
843843
: Type::getInt8Ty(Context);
@@ -846,7 +846,7 @@ class TargetTransformInfoImplBase {
846846
void getMemcpyLoopResidualLoweringType(
847847
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
848848
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
849-
unsigned SrcAlign, unsigned DestAlign,
849+
Align SrcAlign, Align DestAlign,
850850
std::optional<uint32_t> AtomicCpySize) const {
851851
unsigned OpSizeInBytes = AtomicCpySize ? *AtomicCpySize : 1;
852852
Type *OpType = Type::getIntNTy(Context, OpSizeInBytes * 8);

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1194,7 +1194,7 @@ Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
11941194

11951195
Type *TargetTransformInfo::getMemcpyLoopLoweringType(
11961196
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1197-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
1197+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
11981198
std::optional<uint32_t> AtomicElementSize) const {
11991199
return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
12001200
DestAddrSpace, SrcAlign, DestAlign,
@@ -1204,7 +1204,7 @@ Type *TargetTransformInfo::getMemcpyLoopLoweringType(
12041204
void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
12051205
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
12061206
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1207-
unsigned SrcAlign, unsigned DestAlign,
1207+
Align SrcAlign, Align DestAlign,
12081208
std::optional<uint32_t> AtomicCpySize) const {
12091209
TTIImpl->getMemcpyLoopResidualLoweringType(
12101210
OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,

‎llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -418,19 +418,19 @@ int64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
418418
// FIXME: This could use fine tuning and microbenchmarks.
419419
Type *GCNTTIImpl::getMemcpyLoopLoweringType(
420420
LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
421-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
421+
unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
422422
std::optional<uint32_t> AtomicElementSize) const {
423423

424424
if (AtomicElementSize)
425425
return Type::getIntNTy(Context, *AtomicElementSize * 8);
426426

427-
unsigned MinAlign = std::min(SrcAlign, DestAlign);
427+
Align MinAlign = std::min(SrcAlign, DestAlign);
428428

429429
// A (multi-)dword access at an address == 2 (mod 4) will be decomposed by the
430430
// hardware into byte accesses. If you assume all alignments are equally
431431
// probable, it's more efficient on average to use short accesses for this
432432
// case.
433-
if (MinAlign == 2)
433+
if (MinAlign == Align(2))
434434
return Type::getInt16Ty(Context);
435435

436436
// Not all subtargets have 128-bit DS instructions, and we currently don't
@@ -450,7 +450,7 @@ Type *GCNTTIImpl::getMemcpyLoopLoweringType(
450450
void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
451451
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
452452
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
453-
unsigned SrcAlign, unsigned DestAlign,
453+
Align SrcAlign, Align DestAlign,
454454
std::optional<uint32_t> AtomicCpySize) const {
455455
assert(RemainingBytes < 16);
456456

@@ -459,9 +459,9 @@ void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
459459
OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
460460
DestAlign, AtomicCpySize);
461461

462-
unsigned MinAlign = std::min(SrcAlign, DestAlign);
462+
Align MinAlign = std::min(SrcAlign, DestAlign);
463463

464-
if (MinAlign != 2) {
464+
if (MinAlign != Align(2)) {
465465
Type *I64Ty = Type::getInt64Ty(Context);
466466
while (RemainingBytes >= 8) {
467467
OpsOut.push_back(I64Ty);

‎llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,15 +137,16 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
137137
unsigned AddrSpace) const;
138138

139139
int64_t getMaxMemIntrinsicInlineSizeThreshold() const;
140-
Type *getMemcpyLoopLoweringType(
141-
LLVMContext & Context, Value * Length, unsigned SrcAddrSpace,
142-
unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
143-
std::optional<uint32_t> AtomicElementSize) const;
140+
Type *
141+
getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
142+
unsigned SrcAddrSpace, unsigned DestAddrSpace,
143+
Align SrcAlign, Align DestAlign,
144+
std::optional<uint32_t> AtomicElementSize) const;
144145

145146
void getMemcpyLoopResidualLoweringType(
146147
SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
147148
unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
148-
unsigned SrcAlign, unsigned DestAlign,
149+
Align SrcAlign, Align DestAlign,
149150
std::optional<uint32_t> AtomicCpySize) const;
150151
unsigned getMaxInterleaveFactor(ElementCount VF);
151152

‎llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ void llvm::createMemCpyLoopKnownSize(
4545

4646
Type *TypeOfCopyLen = CopyLen->getType();
4747
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
48-
Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value(),
49-
AtomicElementSize);
48+
Ctx, CopyLen, SrcAS, DstAS, SrcAlign, DstAlign, AtomicElementSize);
5049
assert((!AtomicElementSize || !LoopOpType->isVectorTy()) &&
5150
"Atomic memcpy lowering is not supported for vector operand type");
5251

@@ -111,8 +110,8 @@ void llvm::createMemCpyLoopKnownSize(
111110

112111
SmallVector<Type *, 5> RemainingOps;
113112
TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
114-
SrcAS, DstAS, SrcAlign.value(),
115-
DstAlign.value(), AtomicElementSize);
113+
SrcAS, DstAS, SrcAlign, DstAlign,
114+
AtomicElementSize);
116115

117116
for (auto *OpTy : RemainingOps) {
118117
Align PartSrcAlign(commonAlignment(SrcAlign, BytesCopied));
@@ -197,8 +196,7 @@ void llvm::createMemCpyLoopUnknownSize(
197196
unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
198197

199198
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
200-
Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value(),
201-
AtomicElementSize);
199+
Ctx, CopyLen, SrcAS, DstAS, SrcAlign, DstAlign, AtomicElementSize);
202200
assert((!AtomicElementSize || !LoopOpType->isVectorTy()) &&
203201
"Atomic memcpy lowering is not supported for vector operand type");
204202
unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
@@ -411,8 +409,8 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
411409
unsigned SrcAS = cast<PointerType>(SrcAddr->getType())->getAddressSpace();
412410
unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
413411

414-
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
415-
Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value());
412+
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAS, DstAS,
413+
SrcAlign, DstAlign);
416414
unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
417415
Type *Int8Type = Type::getInt8Ty(Ctx);
418416
bool LoopOpIsInt8 = LoopOpType == Int8Type;
@@ -668,8 +666,8 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
668666
unsigned SrcAS = cast<PointerType>(SrcAddr->getType())->getAddressSpace();
669667
unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
670668

671-
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
672-
Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value());
669+
Type *LoopOpType = TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAS, DstAS,
670+
SrcAlign, DstAlign);
673671
unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
674672

675673
// Calculate the loop trip count and remaining bytes to copy after the loop.
@@ -737,8 +735,8 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
737735
IRBuilder<> BwdResBuilder(CopyBackwardsBB->getFirstNonPHI());
738736
SmallVector<Type *, 5> RemainingOps;
739737
TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
740-
SrcAS, DstAS, PartSrcAlign.value(),
741-
PartDstAlign.value());
738+
SrcAS, DstAS, PartSrcAlign,
739+
PartDstAlign);
742740
for (auto *OpTy : RemainingOps) {
743741
// reverse the order of the emitted operations
744742
BwdResBuilder.SetInsertPoint(CopyBackwardsBB->getFirstNonPHI());
@@ -818,8 +816,8 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
818816
IRBuilder<> FwdResBuilder(FwdResidualBB->getTerminator());
819817
SmallVector<Type *, 5> RemainingOps;
820818
TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
821-
SrcAS, DstAS, PartSrcAlign.value(),
822-
PartDstAlign.value());
819+
SrcAS, DstAS, PartSrcAlign,
820+
PartDstAlign);
823821
for (auto *OpTy : RemainingOps)
824822
GenerateResidualLdStPair(OpTy, FwdResBuilder, BytesCopied);
825823
}

0 commit comments

Comments
 (0)