diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index d4bf0d2aa7dad..75f1c9b180bec 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -7828,6 +7828,13 @@ The attributes in this metadata are added to all followup loops of the loop distribution pass. See :ref:`Transformation Metadata ` for details. +'``llvm.loop.isdistributed``' Metadata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If a loop was successfully processed by the loop distribution pass, +this metadata is added (i.e. has been distributed). See +:ref:`Transformation Metadata ` for details. + '``llvm.licm.disable``' Metadata ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/docs/TransformMetadata.rst b/llvm/docs/TransformMetadata.rst index 3c0e10b3eb7a5..1f9f50f1f33bc 100644 --- a/llvm/docs/TransformMetadata.rst +++ b/llvm/docs/TransformMetadata.rst @@ -346,6 +346,18 @@ It is recommended to add ``llvm.loop.disable_nonforced`` to fallback version (which is likely never executed) is further optimized which would increase the code size. +Attributes defined in ``llvm.loop.isdistributed`` are added to successfully +distributed loops to prevent subsequent reprocessing. + +As an example, the following instructs a loop to be ignored during +loop distribution. + +.. code-block:: llvm + + !4 = distinct !{!4, !5, !6} + !5 = !{!"llvm.loop.mustprogress"} + !6 = !{!"llvm.loop.isdistributed", i32 1} + Versioning LICM --------------- diff --git a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp index 27d3004d81947..1099aa335e4c5 100644 --- a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp +++ b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp @@ -113,6 +113,8 @@ static cl::opt EnableLoopDistribute( cl::desc("Enable the new, experimental LoopDistribution Pass"), cl::init(false)); +static const char *DistributedMetaData = "llvm.loop.isdistributed"; + STATISTIC(NumLoopsDistributed, "Number of loops distributed"); namespace { @@ -826,6 +828,8 @@ class LoopDistributeForLoop { {LLVMLoopDistributeFollowupAll, LLVMLoopDistributeFollowupFallback}, "llvm.loop.distribute.", true); LVer.getNonVersionedLoop()->setLoopID(UnversionedLoopID); + addStringMetadataToLoop(LVer.getNonVersionedLoop(), DistributedMetaData, + true); } // Create identical copies of the original loop for each partition and hook @@ -986,6 +990,13 @@ static bool runImpl(Function &F, LoopInfo *LI, DominatorTree *DT, for (Loop *L : Worklist) { LoopDistributeForLoop LDL(L, &F, LI, DT, SE, LAIs, ORE); + // Do not reprocess loops we already distributed + if (getOptionalBoolLoopAttribute(L, DistributedMetaData).value_or(false)) { + LLVM_DEBUG( + dbgs() << "LDist: Distributed loop guarded for reprocessing\n"); + continue; + } + // If distribution was forced for the specific loop to be // enabled/disabled, follow that. Otherwise use the global flag. if (LDL.isForced().value_or(EnableLoopDistribute)) diff --git a/llvm/test/Transforms/LoopDistribute/cross-partition-access.ll b/llvm/test/Transforms/LoopDistribute/cross-partition-access.ll index 6e1106c3277a7..f9c1b365a6b2b 100644 --- a/llvm/test/Transforms/LoopDistribute/cross-partition-access.ll +++ b/llvm/test/Transforms/LoopDistribute/cross-partition-access.ll @@ -64,30 +64,30 @@ define dso_local void @_Z13distribution3PiS_S_S_i(ptr nocapture noundef %a, ptr ; CHECK: [[FOR_BODY_LDIST1]]: ; CHECK-NEXT: [[IDXPROM_LDIST1:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH_LDIST1]] ], [ [[I6_LDIST1:%.*]], %[[FOR_BODY_LDIST1]] ] ; CHECK-NEXT: [[ARRAYIDX_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IDXPROM_LDIST1]] -; CHECK-NEXT: [[I2_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META6:![0-9]+]] +; CHECK-NEXT: [[I2_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]] ; CHECK-NEXT: [[ADD4_LDIST1:%.*]] = add nsw i32 [[I2_LDIST1]], 1 ; CHECK-NEXT: [[ARRAYIDX8_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM_LDIST1]] -; CHECK-NEXT: store i32 [[ADD4_LDIST1]], ptr [[ARRAYIDX8_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]] +; CHECK-NEXT: store i32 [[ADD4_LDIST1]], ptr [[ARRAYIDX8_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META10:![0-9]+]], !noalias [[META12:![0-9]+]] ; CHECK-NEXT: [[I3_LDIST1:%.*]] = getelementptr i32, ptr [[C]], i64 [[IDXPROM_LDIST1]] ; CHECK-NEXT: [[ARRAYIDX17_LDIST1:%.*]] = getelementptr i8, ptr [[I3_LDIST1]], i64 -4 -; CHECK-NEXT: [[I4_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX17_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META14:![0-9]+]], !noalias [[META15:![0-9]+]] +; CHECK-NEXT: [[I4_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX17_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META15:![0-9]+]], !noalias [[META16:![0-9]+]] ; CHECK-NEXT: [[SUB18_LDIST1:%.*]] = sub nsw i32 [[ADD4_LDIST1]], [[I4_LDIST1]] -; CHECK-NEXT: store i32 [[SUB18_LDIST1]], ptr [[I3_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META14]], !noalias [[META15]] +; CHECK-NEXT: store i32 [[SUB18_LDIST1]], ptr [[I3_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META15]], !noalias [[META16]] ; CHECK-NEXT: [[I6_LDIST1]] = add i64 [[IDXPROM_LDIST1]], 1 ; CHECK-NEXT: [[CMP1_NOT_LDIST1:%.*]] = icmp eq i64 [[I6_LDIST1]], [[LEN]] -; CHECK-NEXT: br i1 [[CMP1_NOT_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP16:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP1_NOT_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP17:![0-9]+]] ; CHECK: [[FOR_BODY_PH]]: ; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_BODY]]: ; CHECK-NEXT: [[IDXPROM:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH]] ], [ [[I6:%.*]], %[[FOR_BODY]] ] ; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM]] -; CHECK-NEXT: [[I5:%.*]] = load i32, ptr [[ARRAYIDX8]], align 4, !tbaa [[TBAA0]], !alias.scope [[META9]], !noalias [[META11]] +; CHECK-NEXT: [[I5:%.*]] = load i32, ptr [[ARRAYIDX8]], align 4, !tbaa [[TBAA0]], !alias.scope [[META10]], !noalias [[META12]] ; CHECK-NEXT: [[ADD27:%.*]] = add nsw i32 [[I5]], 2 ; CHECK-NEXT: [[ARRAYIDX31:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[IDXPROM]] -; CHECK-NEXT: store i32 [[ADD27]], ptr [[ARRAYIDX31]], align 4, !tbaa [[TBAA0]], !alias.scope [[META15]], !noalias [[META6]] +; CHECK-NEXT: store i32 [[ADD27]], ptr [[ARRAYIDX31]], align 4, !tbaa [[TBAA0]], !alias.scope [[META16]], !noalias [[META7]] ; CHECK-NEXT: [[I6]] = add i64 [[IDXPROM]], 1 ; CHECK-NEXT: [[CMP1_NOT:%.*]] = icmp eq i64 [[I6]], [[LEN]] -; CHECK-NEXT: br i1 [[CMP1_NOT]], label %[[END_LOOPEXIT_LOOPEXIT20:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP16]] +; CHECK-NEXT: br i1 [[CMP1_NOT]], label %[[END_LOOPEXIT_LOOPEXIT20:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP17]] ; CHECK: [[END_LOOPEXIT_LOOPEXIT]]: ; CHECK-NEXT: br label %[[END_LOOPEXIT:.*]] ; CHECK: [[END_LOOPEXIT_LOOPEXIT20]]: @@ -143,17 +143,18 @@ end: ; preds = %end.loopexit, %entr ; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0} ; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0} ; CHECK: [[META3]] = !{!"Simple C++ TBAA"} -; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]} +; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]} ; CHECK: [[META5]] = !{!"llvm.loop.mustprogress"} -; CHECK: [[META6]] = !{[[META7:![0-9]+]]} -; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]} -; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"} -; CHECK: [[META9]] = !{[[META10:![0-9]+]]} -; CHECK: [[META10]] = distinct !{[[META10]], [[META8]]} -; CHECK: [[META11]] = !{[[META12:![0-9]+]], [[META13:![0-9]+]], [[META7]]} -; CHECK: [[META12]] = distinct !{[[META12]], [[META8]]} -; CHECK: [[META13]] = distinct !{[[META13]], [[META8]]} -; CHECK: [[META14]] = !{[[META12]]} +; CHECK: [[META6]] = !{!"llvm.loop.isdistributed", i32 1} +; CHECK: [[META7]] = !{[[META8:![0-9]+]]} +; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]} +; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"} +; CHECK: [[META10]] = !{[[META11:![0-9]+]]} +; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]} +; CHECK: [[META12]] = !{[[META13:![0-9]+]], [[META14:![0-9]+]], [[META8]]} +; CHECK: [[META13]] = distinct !{[[META13]], [[META9]]} +; CHECK: [[META14]] = distinct !{[[META14]], [[META9]]} ; CHECK: [[META15]] = !{[[META13]]} -; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META5]]} +; CHECK: [[META16]] = !{[[META14]]} +; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]} ;. diff --git a/llvm/test/Transforms/LoopDistribute/followup.ll b/llvm/test/Transforms/LoopDistribute/followup.ll index 55307bdf24991..ae418ad9e6b42 100644 --- a/llvm/test/Transforms/LoopDistribute/followup.ll +++ b/llvm/test/Transforms/LoopDistribute/followup.ll @@ -58,29 +58,29 @@ define void @f(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) { ; CHECK: [[FOR_BODY_LDIST1]]: ; CHECK-NEXT: [[IND_LDIST1:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH_LDIST1]] ], [ [[ADD_LDIST1:%.*]], %[[FOR_BODY_LDIST1]] ] ; CHECK-NEXT: [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IND_LDIST1]] -; CHECK-NEXT: [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]] +; CHECK-NEXT: [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]] ; CHECK-NEXT: [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IND_LDIST1]] -; CHECK-NEXT: [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4, !alias.scope [[META10:![0-9]+]] +; CHECK-NEXT: [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4, !alias.scope [[META11:![0-9]+]] ; CHECK-NEXT: [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]] ; CHECK-NEXT: [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1 ; CHECK-NEXT: [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]] -; CHECK-NEXT: store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META3]], !noalias [[META6]] +; CHECK-NEXT: store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META4]], !noalias [[META7]] ; CHECK-NEXT: [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], 20 -; CHECK-NEXT: br i1 [[EXITCOND_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP13:![0-9]+]] ; CHECK: [[FOR_BODY_PH]]: ; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_BODY]]: ; CHECK-NEXT: [[IND:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ] ; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[IND]], 1 ; CHECK-NEXT: [[ARRAYIDXD:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[IND]] -; CHECK-NEXT: [[LOADD:%.*]] = load i32, ptr [[ARRAYIDXD]], align 4, !alias.scope [[META14:![0-9]+]] +; CHECK-NEXT: [[LOADD:%.*]] = load i32, ptr [[ARRAYIDXD]], align 4, !alias.scope [[META15:![0-9]+]] ; CHECK-NEXT: [[ARRAYIDXE:%.*]] = getelementptr inbounds i32, ptr [[E]], i64 [[IND]] -; CHECK-NEXT: [[LOADE:%.*]] = load i32, ptr [[ARRAYIDXE]], align 4, !alias.scope [[META15:![0-9]+]] +; CHECK-NEXT: [[LOADE:%.*]] = load i32, ptr [[ARRAYIDXE]], align 4, !alias.scope [[META16:![0-9]+]] ; CHECK-NEXT: [[MULC:%.*]] = mul i32 [[LOADD]], [[LOADE]] ; CHECK-NEXT: [[ARRAYIDXC:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IND]] -; CHECK-NEXT: store i32 [[MULC]], ptr [[ARRAYIDXC]], align 4, !alias.scope [[META16:![0-9]+]], !noalias [[META10]] +; CHECK-NEXT: store i32 [[MULC]], ptr [[ARRAYIDXC]], align 4, !alias.scope [[META17:![0-9]+]], !noalias [[META11]] ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[ADD]], 20 -; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT16:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT16:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; CHECK: [[FOR_END_LOOPEXIT]]: ; CHECK-NEXT: br label %[[FOR_END:.*]] ; CHECK: [[FOR_END_LOOPEXIT16]]: @@ -131,23 +131,24 @@ for.end: !4 = !{!"llvm.loop.distribute.followup_sequential", !{!"FollowupSequential", i32 8}} !5 = !{!"llvm.loop.distribute.followup_fallback", !{!"FollowupFallback"}} ;. -; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]} ; CHECK: [[META1]] = !{!"FollowupAll"} ; CHECK: [[META2]] = !{!"FollowupFallback"} -; CHECK: [[META3]] = !{[[META4:![0-9]+]]} -; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]]} -; CHECK: [[META5]] = distinct !{[[META5]], !"LVerDomain"} -; CHECK: [[META6]] = !{[[META7:![0-9]+]], [[META8:![0-9]+]], [[META9:![0-9]+]]} -; CHECK: [[META7]] = distinct !{[[META7]], [[META5]]} -; CHECK: [[META8]] = distinct !{[[META8]], [[META5]]} -; CHECK: [[META9]] = distinct !{[[META9]], [[META5]]} -; CHECK: [[META10]] = !{[[META11:![0-9]+]]} -; CHECK: [[META11]] = distinct !{[[META11]], [[META5]]} -; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META13:![0-9]+]]} -; CHECK: [[META13]] = !{!"FollowupSequential", i32 8} -; CHECK: [[META14]] = !{[[META8]]} +; CHECK: [[META3]] = !{!"llvm.loop.isdistributed", i32 1} +; CHECK: [[META4]] = !{[[META5:![0-9]+]]} +; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]} +; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"} +; CHECK: [[META7]] = !{[[META8:![0-9]+]], [[META9:![0-9]+]], [[META10:![0-9]+]]} +; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]} +; CHECK: [[META9]] = distinct !{[[META9]], [[META6]]} +; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]} +; CHECK: [[META11]] = !{[[META12:![0-9]+]]} +; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]} +; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META14:![0-9]+]]} +; CHECK: [[META14]] = !{!"FollowupSequential", i32 8} ; CHECK: [[META15]] = !{[[META9]]} -; CHECK: [[META16]] = !{[[META7]]} -; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META18:![0-9]+]]} -; CHECK: [[META18]] = !{!"FollowupCoincident", i1 false} +; CHECK: [[META16]] = !{[[META10]]} +; CHECK: [[META17]] = !{[[META8]]} +; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META19:![0-9]+]]} +; CHECK: [[META19]] = !{!"FollowupCoincident", i1 false} ;. diff --git a/llvm/test/Transforms/LoopDistribute/laa-invalidation.ll b/llvm/test/Transforms/LoopDistribute/laa-invalidation.ll index ee42860cd250e..62c5627ac2d38 100644 --- a/llvm/test/Transforms/LoopDistribute/laa-invalidation.ll +++ b/llvm/test/Transforms/LoopDistribute/laa-invalidation.ll @@ -27,13 +27,13 @@ define void @test_pr50940(ptr %A, ptr %B) { ; CHECK-NEXT: store i16 1, ptr [[B]], align 1 ; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i16 [[IV_LVER_ORIG]], 1 ; CHECK-NEXT: [[C_1_LVER_ORIG:%.*]] = icmp ult i16 [[IV_LVER_ORIG]], 38 -; CHECK-NEXT: br i1 [[C_1_LVER_ORIG]], label [[INNER_LVER_ORIG]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK-NEXT: br i1 [[C_1_LVER_ORIG]], label [[INNER_LVER_ORIG]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: inner.ph3.ldist1: ; CHECK-NEXT: br label [[INNER_LDIST1:%.*]] ; CHECK: inner.ldist1: ; CHECK-NEXT: [[IV_LDIST1:%.*]] = phi i16 [ 0, [[INNER_PH3_LDIST1]] ], [ [[IV_NEXT_LDIST1:%.*]], [[INNER_LDIST1]] ] -; CHECK-NEXT: [[L_LDIST1:%.*]] = load <2 x i16>, ptr [[UGLYGEP]], align 1, !alias.scope !0, !noalias !3 -; CHECK-NEXT: store i16 0, ptr [[GEP_A_3]], align 1, !alias.scope !0, !noalias !3 +; CHECK-NEXT: [[L_LDIST1:%.*]] = load <2 x i16>, ptr [[UGLYGEP]], align 1, !alias.scope [[META2:![0-9]+]], !noalias [[META5:![0-9]+]] +; CHECK-NEXT: store i16 0, ptr [[GEP_A_3]], align 1, !alias.scope [[META2]], !noalias [[META5]] ; CHECK-NEXT: [[IV_NEXT_LDIST1]] = add nuw nsw i16 [[IV_LDIST1]], 1 ; CHECK-NEXT: [[C_1_LDIST1:%.*]] = icmp ult i16 [[IV_LDIST1]], 38 ; CHECK-NEXT: br i1 [[C_1_LDIST1]], label [[INNER_LDIST1]], label [[INNER_PH3:%.*]] @@ -41,7 +41,7 @@ define void @test_pr50940(ptr %A, ptr %B) { ; CHECK-NEXT: br label [[INNER:%.*]] ; CHECK: inner: ; CHECK-NEXT: [[IV:%.*]] = phi i16 [ 0, [[INNER_PH3]] ], [ [[IV_NEXT:%.*]], [[INNER]] ] -; CHECK-NEXT: store i16 1, ptr [[B]], align 1, !alias.scope !3 +; CHECK-NEXT: store i16 1, ptr [[B]], align 1, !alias.scope [[META5]] ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1 ; CHECK-NEXT: [[C_1:%.*]] = icmp ult i16 [[IV]], 38 ; CHECK-NEXT: br i1 [[C_1]], label [[INNER]], label [[EXIT_LOOPEXIT4:%.*]] diff --git a/llvm/test/Transforms/LoopDistribute/no-reprocess.ll b/llvm/test/Transforms/LoopDistribute/no-reprocess.ll new file mode 100644 index 0000000000000..2dbc77aa756f6 --- /dev/null +++ b/llvm/test/Transforms/LoopDistribute/no-reprocess.ll @@ -0,0 +1,130 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -aa-pipeline=basic-aa -passes=loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -debug-only=loop-distribute --disable-output -S < %s 2>&1 | FileCheck %s +; REQUIRES: asserts + +; Test if a loop already distributed will not reprocess because of metadata +; information marking it as processed. + +; CHECK: LDist: Distributed loop guarded for reprocessing +; CHECK: LDist: Skipping; + +define dso_local void @_Z13distribution3PiS_S_S_i(ptr noundef captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef captures(none) %c, ptr noundef writeonly captures(none) %d, i64 noundef signext %len) { +entry: + %cmp = icmp sgt i64 %len, 0 + br i1 %cmp, label %end, label %for.body.lver.check + +for.body.lver.check: ; preds = %entry + %0 = shl i64 %len, 2 + %scevgep = getelementptr i8, ptr %a, i64 %0 + %scevgep1 = getelementptr i8, ptr %c, i64 -4 + %scevgep2 = getelementptr i8, ptr %c, i64 %0 + %scevgep3 = getelementptr i8, ptr %d, i64 %0 + %scevgep4 = getelementptr i8, ptr %b, i64 %0 + %bound0 = icmp ult ptr %a, %scevgep2 + %bound1 = icmp ult ptr %scevgep1, %scevgep + %found.conflict = and i1 %bound0, %bound1 + %bound05 = icmp ult ptr %a, %scevgep3 + %bound16 = icmp ult ptr %d, %scevgep + %found.conflict7 = and i1 %bound05, %bound16 + %conflict.rdx = or i1 %found.conflict, %found.conflict7 + %bound08 = icmp ult ptr %a, %scevgep4 + %bound19 = icmp ult ptr %b, %scevgep + %found.conflict10 = and i1 %bound08, %bound19 + %conflict.rdx11 = or i1 %conflict.rdx, %found.conflict10 + %bound012 = icmp ult ptr %scevgep1, %scevgep3 + %bound113 = icmp ult ptr %d, %scevgep2 + %found.conflict14 = and i1 %bound012, %bound113 + %conflict.rdx15 = or i1 %conflict.rdx11, %found.conflict14 + %bound016 = icmp ult ptr %d, %scevgep4 + %bound117 = icmp ult ptr %b, %scevgep3 + %found.conflict18 = and i1 %bound016, %bound117 + %conflict.rdx19 = or i1 %conflict.rdx15, %found.conflict18 + br i1 %conflict.rdx19, label %for.body.ph.lver.orig, label %for.body.ph.ldist1 + +for.body.ph.lver.orig: ; preds = %for.body.lver.check + br label %for.body.lver.orig + +for.body.lver.orig: ; preds = %for.body.lver.orig, %for.body.ph.lver.orig + %indvars.iv.lver.orig = phi i64 [ 0, %for.body.ph.lver.orig ], [ %indvars.iv.next.lver.orig, %for.body.lver.orig ] + %arrayidx.lver.orig = getelementptr inbounds i32, ptr %b, i64 %indvars.iv.lver.orig + %i2.lver.orig = load i32, ptr %arrayidx.lver.orig, align 4, !tbaa !0 + %add4.lver.orig = add nsw i32 %i2.lver.orig, 1 + %arrayidx8.lver.orig = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.lver.orig + store i32 %add4.lver.orig, ptr %arrayidx8.lver.orig, align 4, !tbaa !0 + %i3.lver.orig = getelementptr i32, ptr %c, i64 %indvars.iv.lver.orig + %arrayidx17.lver.orig = getelementptr i8, ptr %i3.lver.orig, i64 -4 + %i4.lver.orig = load i32, ptr %arrayidx17.lver.orig, align 4, !tbaa !0 + %sub18.lver.orig = sub nsw i32 %add4.lver.orig, %i4.lver.orig + store i32 %sub18.lver.orig, ptr %i3.lver.orig, align 4, !tbaa !0 + %i5.lver.orig = load i32, ptr %arrayidx8.lver.orig, align 4, !tbaa !0 + %add27.lver.orig = add nsw i32 %i5.lver.orig, 2 + %arrayidx31.lver.orig = getelementptr inbounds i32, ptr %d, i64 %indvars.iv.lver.orig + store i32 %add27.lver.orig, ptr %arrayidx31.lver.orig, align 4, !tbaa !0 + %indvars.iv.next.lver.orig = add i64 %indvars.iv.lver.orig, 1 + %cmp1.not.lver.orig = icmp eq i64 %indvars.iv.next.lver.orig, %len + br i1 %cmp1.not.lver.orig, label %end.loopexit.loopexit, label %for.body.lver.orig, !llvm.loop !4 + +for.body.ph.ldist1: ; preds = %for.body.lver.check + br label %for.body.ldist1 + +for.body.ldist1: ; preds = %for.body.ldist1, %for.body.ph.ldist1 + %indvars.iv.ldist1 = phi i64 [ 0, %for.body.ph.ldist1 ], [ %indvars.iv.next.ldist1, %for.body.ldist1 ] + %arrayidx.ldist1 = getelementptr inbounds i32, ptr %b, i64 %indvars.iv.ldist1 + %i2.ldist1 = load i32, ptr %arrayidx.ldist1, align 4, !tbaa !0, !alias.scope !7 + %add4.ldist1 = add nsw i32 %i2.ldist1, 1 + %arrayidx8.ldist1 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.ldist1 + store i32 %add4.ldist1, ptr %arrayidx8.ldist1, align 4, !tbaa !0, !alias.scope !10, !noalias !12 + %i3.ldist1 = getelementptr i32, ptr %c, i64 %indvars.iv.ldist1 + %arrayidx17.ldist1 = getelementptr i8, ptr %i3.ldist1, i64 -4 + %i4.ldist1 = load i32, ptr %arrayidx17.ldist1, align 4, !tbaa !0, !alias.scope !15, !noalias !16 + %sub18.ldist1 = sub nsw i32 %add4.ldist1, %i4.ldist1 + store i32 %sub18.ldist1, ptr %i3.ldist1, align 4, !tbaa !0, !alias.scope !15, !noalias !16 + %indvars.iv.next.ldist1 = add i64 %indvars.iv.ldist1, 1 + %cmp1.not.ldist1 = icmp eq i64 %indvars.iv.next.ldist1, %len + br i1 %cmp1.not.ldist1, label %for.body.ph, label %for.body.ldist1, !llvm.loop !17 + +for.body.ph: ; preds = %for.body.ldist1 + br label %for.body + +for.body: ; preds = %for.body, %for.body.ph + %indvars.iv = phi i64 [ 0, %for.body.ph ], [ %indvars.iv.next, %for.body ] + %arrayidx8 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv + %i5 = load i32, ptr %arrayidx8, align 4, !tbaa !0, !alias.scope !10, !noalias !12 + %add27 = add nsw i32 %i5, 2 + %arrayidx31 = getelementptr inbounds i32, ptr %d, i64 %indvars.iv + store i32 %add27, ptr %arrayidx31, align 4, !tbaa !0, !alias.scope !16, !noalias !7 + %indvars.iv.next = add i64 %indvars.iv, 1 + %cmp1.not = icmp eq i64 %indvars.iv.next, %len + br i1 %cmp1.not, label %end.loopexit.loopexit20, label %for.body, !llvm.loop !17 + +end.loopexit.loopexit: ; preds = %for.body.lver.orig + br label %end.loopexit + +end.loopexit.loopexit20: ; preds = %for.body + br label %end.loopexit + +end.loopexit: ; preds = %end.loopexit.loopexit20, %end.loopexit.loopexit + br label %end + +end: ; preds = %end.loopexit, %entry + ret void +} + +!0 = !{!1, !1, i64 0} +!1 = !{!"int", !2, i64 0} +!2 = !{!"omnipotent char", !3, i64 0} +!3 = !{!"Simple C++ TBAA"} +!4 = distinct !{!4, !5, !6} +!5 = !{!"llvm.loop.mustprogress"} +!6 = !{!"llvm.loop.isdistributed", i32 1} +!7 = !{!8} +!8 = distinct !{!8, !9} +!9 = distinct !{!9, !"LVerDomain"} +!10 = !{!11} +!11 = distinct !{!11, !9} +!12 = !{!13, !14, !8} +!13 = distinct !{!13, !9} +!14 = distinct !{!14, !9} +!15 = !{!13} +!16 = !{!14} +!17 = distinct !{!17, !5} diff --git a/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll b/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll index 98b89abfeafda..dd23a1121ebcb 100644 --- a/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll +++ b/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll @@ -36,7 +36,7 @@ define void @f(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %d, p ; CHECK-NEXT: [[ARRAYIDXC_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[MUL_EXT_LVER_ORIG]] ; CHECK-NEXT: store i32 [[MULC_LVER_ORIG]], ptr [[ARRAYIDXC_LVER_ORIG]], align 4 ; CHECK-NEXT: [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[ADD_LVER_ORIG]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]] +; CHECK-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: for.body.ph.ldist1: ; CHECK-NEXT: br label [[FOR_BODY_LDIST1:%.*]] ; CHECK: for.body.ldist1: @@ -45,14 +45,14 @@ define void @f(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %d, p ; CHECK-NEXT: [[MUL_LDIST1:%.*]] = mul i32 [[IND1_LDIST1]], 2 ; CHECK-NEXT: [[MUL_EXT_LDIST1:%.*]] = zext i32 [[MUL_LDIST1]] to i64 ; CHECK-NEXT: [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_EXT_LDIST1]] -; CHECK-NEXT: [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META0:![0-9]+]] +; CHECK-NEXT: [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META2:![0-9]+]] ; CHECK-NEXT: [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[MUL_EXT_LDIST1]] ; CHECK-NEXT: [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4 ; CHECK-NEXT: [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]] ; CHECK-NEXT: [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1 ; CHECK-NEXT: [[INC1_LDIST1]] = add i32 [[IND1_LDIST1]], 1 ; CHECK-NEXT: [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]] -; CHECK-NEXT: store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META3:![0-9]+]] +; CHECK-NEXT: store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META5:![0-9]+]] ; CHECK-NEXT: [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], [[N]] ; CHECK-NEXT: br i1 [[EXITCOND_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]] ; CHECK: for.body.ph: @@ -161,7 +161,7 @@ define void @f_with_offset(ptr noalias %b, ptr noalias %c, ptr noalias %d, ptr n ; CHECK-NEXT: [[ARRAYIDXC_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[MUL_EXT_LVER_ORIG]] ; CHECK-NEXT: store i32 [[MULC_LVER_ORIG]], ptr [[ARRAYIDXC_LVER_ORIG]], align 4 ; CHECK-NEXT: [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[ADD_LVER_ORIG]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]] +; CHECK-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: for.body.ph.ldist1: ; CHECK-NEXT: br label [[FOR_BODY_LDIST1:%.*]] ; CHECK: for.body.ldist1: @@ -170,14 +170,14 @@ define void @f_with_offset(ptr noalias %b, ptr noalias %c, ptr noalias %d, ptr n ; CHECK-NEXT: [[MUL_LDIST1:%.*]] = mul i32 [[IND1_LDIST1]], 2 ; CHECK-NEXT: [[MUL_EXT_LDIST1:%.*]] = zext i32 [[MUL_LDIST1]] to i64 ; CHECK-NEXT: [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_EXT_LDIST1]] -; CHECK-NEXT: [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META5:![0-9]+]] +; CHECK-NEXT: [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META8:![0-9]+]] ; CHECK-NEXT: [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[MUL_EXT_LDIST1]] ; CHECK-NEXT: [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4 ; CHECK-NEXT: [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]] ; CHECK-NEXT: [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1 ; CHECK-NEXT: [[INC1_LDIST1]] = add i32 [[IND1_LDIST1]], 1 ; CHECK-NEXT: [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]] -; CHECK-NEXT: store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META8:![0-9]+]] +; CHECK-NEXT: store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META11:![0-9]+]] ; CHECK-NEXT: [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], [[N]] ; CHECK-NEXT: br i1 [[EXITCOND_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]] ; CHECK: for.body.ph: diff --git a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll index 80d6d361cf0d5..8a639166d41e4 100644 --- a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll +++ b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll @@ -42,8 +42,8 @@ for.body: ; preds = %for.body, %entry %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind -; CHECK: %loadA.ldist1 = {{.*}} !noalias !25 -; A noalias C: !25 -> { 17(15), 18(15), 19(15), 26(24) } +; CHECK: %loadA.ldist1 = {{.*}} !alias.scope !24, !noalias !27 +; A noalias C: !33 -> { 19(17), 20(17), 21(17), 28(26) } ; ^^^^^^ %loadA = load i32, ptr %arrayidxA, align 4 @@ -60,15 +60,15 @@ for.body: ; preds = %for.body, %entry %arrayidxD = getelementptr inbounds i32, ptr %d, i64 %ind -; CHECK: %loadD = {{.*}} !alias.scope !31 -; D's scope: !31 -> { 18(15), 32(33) } +; CHECK: %loadD = {{.*}} !alias.scope !33 +; D's scope: !33 -> { 20(17), 34(35) } ; ^^^^^^ %loadD = load i32, ptr %arrayidxD, align 4 %arrayidxE = getelementptr inbounds i32, ptr %e, i64 %ind -; CHECK: %loadE = {{.*}} !alias.scope !34 -; E's scope: !34 -> { 19(15), 35(33) } +; CHECK: %loadE = {{.*}} !alias.scope !36 +; E's scope: !36 -> { 21(17), 37(33) } ; ^^^^^^ %loadE = load i32, ptr %arrayidxE, align 4 @@ -76,8 +76,8 @@ for.body: ; preds = %for.body, %entry %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind -; CHECK: store i32 %mulC, {{.*}} !alias.scope !36, !noalias !38 -; C's scope: !36 -> { 17(15), 37(33) } +; CHECK: store i32 %mulC, {{.*}} !alias.scope !38, !noalias !40 +; C's scope: !38 -> { 19(17), 39(35) ; ^^^^^^ ; C noalias D and E: !38 -> { 21(15), 32(33), 35(33) } ; ^^^^^^ ^^^^^^ @@ -92,15 +92,19 @@ for.end: ; preds = %for.body ; Domain for the second loop versioning for the top loop after ; distribution. -; CHECK: !15 = distinct !{!15, !"LVerDomain"} -; CHECK: !17 = distinct !{!17, !15} -; CHECK: !25 = !{!17, !18, !19, !26} -; CHECK: !31 = !{!18, !32} -; CHECK: !32 = distinct !{!32, !33} +; CHECK: !17 = distinct !{!17, !"LVerDomain"} +; CHECK: !19 = distinct !{!19, !17} +; CHECK: !20 = distinct !{!20, !17} +; CHECK: !21 = distinct !{!21, !17} +; CHECK: !27 = !{!19, !20, !21, !28} +; CHECK: !28 = distinct !{!28, !26} +; CHECK: !33 = !{!20, !34} +; CHECK: !34 = distinct !{!34, !35} ; Domain for the second loop versioning for the bottom loop after ; distribution. -; CHECK: !33 = distinct !{!33, !"LVerDomain"} -; CHECK: !34 = !{!19, !35} -; CHECK: !35 = distinct !{!35, !33} -; CHECK: !36 = !{!17, !37} -; CHECK: !38 = !{!21, !32, !35} +; CHECK: !35 = distinct !{!35, !"LVerDomain"} +; CHECK: !36 = !{!21, !37} +; CHECK: !37 = distinct !{!37, !35} +; CHECK: !38 = !{!19, !39} +; CHECK: !39 = distinct !{!39, !35} +; CHECK: !40 = !{!23, !34, !37} diff --git a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected index 5751738f7bb3b..65904d13c1c96 100644 --- a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected @@ -72,12 +72,12 @@ define void @ldist(i1 %cond, ptr %A, ptr %B, ptr %C) { ; CHECK-NEXT: LDist: Partition 0: ; CHECK-NEXT: for.body.ldist1: ; preds = %if.end.ldist1, %for.body.ph.ldist1 ; CHECK-NEXT: %iv.ldist1 = phi i16 [ 0, %for.body.ph.ldist1 ], [ %iv.next.ldist1, %if.end.ldist1 ] -; CHECK-NEXT: %lv.ldist1 = load i16, ptr %A, align 1, !alias.scope !0, !noalias !3 -; CHECK-NEXT: store i16 %lv.ldist1, ptr %A, align 1, !alias.scope !0, !noalias !3 +; CHECK-NEXT: %lv.ldist1 = load i16, ptr %A, align 1, !alias.scope !2, !noalias !5 +; CHECK-NEXT: store i16 %lv.ldist1, ptr %A, align 1, !alias.scope !2, !noalias !5 ; CHECK-NEXT: br i1 %cond, label %if.then.ldist1, label %if.end.ldist1 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.ldist1: ; preds = %for.body.ldist1 -; CHECK-NEXT: %lv2.ldist1 = load i16, ptr %A, align 1, !alias.scope !0, !noalias !3 +; CHECK-NEXT: %lv2.ldist1 = load i16, ptr %A, align 1, !alias.scope !2, !noalias !5 ; CHECK-NEXT: br label %if.end.ldist1 ; CHECK-EMPTY: ; CHECK-NEXT: if.end.ldist1: ; preds = %if.then.ldist1, %for.body.ldist1