diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index fcc5eb1c05ba0..f9dca609f6828 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -2368,6 +2368,7 @@ static std::optional instCombineSVEUzp1(InstCombiner &IC, // uzp1(to_svbool(A), to_svbool(B)) --> // uzp1(from_svbool(to_svbool(A)), from_svbool(to_svbool(B))) --> + if ((match(II.getArgOperand(0), m_Intrinsic(m_Intrinsic(m_Value(A)))) && match(II.getArgOperand(1), @@ -2674,6 +2675,52 @@ static std::optional instCombinePTrue(InstCombiner &IC, return std::nullopt; } +static std::optional instCombineSVERev(InstCombiner &IC, + IntrinsicInst &II) { + // rev(rev(x)) -> x + switch (II.getIntrinsicID()) { + default: + return std::nullopt; + + case Intrinsic::aarch64_sve_rev: + case Intrinsic::aarch64_sve_rev_b16: + case Intrinsic::aarch64_sve_rev_b32: + case Intrinsic::aarch64_sve_rev_b64: { + Value *InnerArg = II.getArgOperand(0); + IntrinsicInst *InnerRev = dyn_cast(InnerArg); + // Fold rev(rev(x)) -> x, if intrinsic IDs match and InnerRev has one use + if (InnerRev && InnerRev->getIntrinsicID() == II.getIntrinsicID() && + InnerRev->hasOneUse()) + return IC.replaceInstUsesWith(II, InnerRev->getArgOperand(0)); + + return std::nullopt; + } + + case Intrinsic::aarch64_sve_revb: + case Intrinsic::aarch64_sve_revd: + case Intrinsic::aarch64_sve_revh: + case Intrinsic::aarch64_sve_revw: { + Value *InnerArg = II.getArgOperand(2); + IntrinsicInst *InnerRev = dyn_cast(InnerArg); + + // Early exit if InnerRev != outerId and doesn't only have one use + if (!InnerRev || InnerRev->getIntrinsicID() != II.getIntrinsicID() || + !InnerRev->hasOneUse()) + return std::nullopt; + + Value *OuterPred = II.getArgOperand(0); + Value *OuterPassThru = II.getArgOperand(1); + Value *InnerPred = InnerRev->getArgOperand(0); + Value *InnerPassThru = InnerRev->getArgOperand(1); + + // Fold rev(rev(x)) -> x, if predicates and pass-thrus match + return IC.replaceInstUsesWith(II, InnerRev->getArgOperand(2)); + + return std::nullopt; + } + } +} + std::optional AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { @@ -2773,6 +2820,15 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, return instCombineSVEInsr(IC, II); case Intrinsic::aarch64_sve_ptrue: return instCombinePTrue(IC, II); + case Intrinsic::aarch64_sve_rev: + case Intrinsic::aarch64_sve_rev_b16: + case Intrinsic::aarch64_sve_rev_b32: + case Intrinsic::aarch64_sve_rev_b64: + case Intrinsic::aarch64_sve_revb: + case Intrinsic::aarch64_sve_revd: + case Intrinsic::aarch64_sve_revh: + case Intrinsic::aarch64_sve_revw: + return instCombineSVERev(IC, II); } return std::nullopt; diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-rev-inv.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-rev-inv.ll new file mode 100644 index 0000000000000..e9541eda85d40 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-rev-inv.ll @@ -0,0 +1,261 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -S -passes=instcombine < %s | FileCheck %s + +target triple = "aarch64-unknown-linux-gnu" + +define @aarch64_sve_rev_inv( %0) #0{ +; CHECK-LABEL: define @aarch64_sve_rev_inv( +; CHECK-SAME: [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: ret [[TMP0]] +; +entry: + %1 = call @llvm.aarch64.sve.rev( %0) + %2 = call @llvm.aarch64.sve.rev( %1) + ret %2 +} + +define @aarch64_sve_rev_b16_inv( %0) #0{ +; CHECK-LABEL: define @aarch64_sve_rev_b16_inv( +; CHECK-SAME: [[TMP0:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: ret [[TMP0]] +; +entry: + %1 = call @llvm.aarch64.sve.rev.b16( %0) + %2 = call @llvm.aarch64.sve.rev.b16( %1) + ret %2 +} + +define @aarch64_sve_rev_b32_inv( %0) #0{ +; CHECK-LABEL: define @aarch64_sve_rev_b32_inv( +; CHECK-SAME: [[TMP0:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: ret [[TMP0]] +; +entry: + %1 = call @llvm.aarch64.sve.rev.b32( %0) + %2 = call @llvm.aarch64.sve.rev.b32( %1) + ret %2 +} + +define @aarch64_sve_rev_b64_inv( %0)#0 { +; CHECK-LABEL: define @aarch64_sve_rev_b64_inv( +; CHECK-SAME: [[TMP0:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: ret [[TMP0]] +; +entry: + %1 = call @llvm.aarch64.sve.rev.b64( %0) + %2 = call @llvm.aarch64.sve.rev.b64( %1) + ret %2 +} + + +define @aarch64_sve_revb_inv( %pred, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revb_inv( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: ret [[A]] +; + %1 = call @llvm.aarch64.sve.revb.nxv4i32( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revb.nxv4i32( %pred, %passthru, %1) + ret %2 +} + +define @aarch64_sve_revd_inv( %pred, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revd_inv( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: ret [[A]] +; + %1 = call @llvm.aarch64.sve.revd.nxv16i8( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revd.nxv16i8( %pred, %passthru, %1) + ret %2 +} + +define @aarch64_sve_revh_inv( %pred, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revh_inv( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: ret [[A]] +; + %1 = call @llvm.aarch64.sve.revh.nxv4i32( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revh.nxv4i32( %pred, %passthru, %1) + ret %2 +} + +define @aarch64_sve_revw_inv( %pred, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revw_inv( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: ret [[A]] +; + %1 = call @llvm.aarch64.sve.revw.nxv2i64( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revw.nxv2i64( %pred, %passthru, %1) + ret %2 +} + + +; negative test +define @aarch64_sve_revb_inv_pred_mismatch( %pred, %pred1, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revb_inv_pred_mismatch( +; CHECK-SAME: [[PRED:%.*]], [[PRED1:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revb.nxv4i32( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revb.nxv4i32( [[PRED1]], [[PASSTHRU]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revb.nxv4i32( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revb.nxv4i32( %pred1, %passthru, %1) + ret %2 +} + +; negative test +define @aarch64_sve_revd_inv_pred_mismatch( %pred, %pred1, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revd_inv_pred_mismatch( +; CHECK-SAME: [[PRED:%.*]], [[PRED1:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revd.nxv16i8( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revd.nxv16i8( [[PRED1]], [[PASSTHRU]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revd.nxv16i8( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revd.nxv16i8( %pred1, %passthru, %1) + ret %2 +} + +; negative test +define @aarch64_sve_revh_inv_pred_mismatch( %pred, %pred1, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revh_inv_pred_mismatch( +; CHECK-SAME: [[PRED:%.*]], [[PRED1:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revh.nxv4i32( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revh.nxv4i32( [[PRED1]], [[PASSTHRU]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revh.nxv4i32( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revh.nxv4i32( %pred1, %passthru, %1) + ret %2 +} + +; negative test +define @aarch64_sve_revw_inv_pred_mismatch( %pred, %pred1, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revw_inv_pred_mismatch( +; CHECK-SAME: [[PRED:%.*]], [[PRED1:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revw.nxv2i64( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revw.nxv2i64( [[PRED1]], [[PASSTHRU]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revw.nxv2i64( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revw.nxv2i64( %pred1, %passthru, %1) + ret %2 +} + +; negative test +define @aarch64_sve_revb_inv_passthru_mismatch( %pred, %passthru, %passthru1, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revb_inv_passthru_mismatch( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[PASSTHRU1:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revb.nxv4i32( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revb.nxv4i32( [[PRED]], [[PASSTHRU1]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revb.nxv4i32( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revb.nxv4i32( %pred, %passthru1, %1) + ret %2 +} + +; negative test +define @aarch64_sve_revd_inv_passthru_mismatch( %pred, %passthru, %passthru1, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revd_inv_passthru_mismatch( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[PASSTHRU1:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revd.nxv16i8( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revd.nxv16i8( [[PRED]], [[PASSTHRU1]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revd.nxv16i8( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revd.nxv16i8( %pred, %passthru1, %1) + ret %2 +} + +; negative test +define @aarch64_sve_revh_inv_passthru_mismatch( %pred, %passthru, %passthru1, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revh_inv_passthru_mismatch( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[PASSTHRU1:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revh.nxv4i32( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revh.nxv4i32( [[PRED]], [[PASSTHRU1]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revh.nxv4i32( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revh.nxv4i32( %pred, %passthru1, %1) + ret %2 +} + +; negative test +define @aarch64_sve_revw_inv_passthru_mismatch( %pred, %passthru, %passthru1, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revw_inv_passthru_mismatch( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[PASSTHRU1:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revw.nxv2i64( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revw.nxv2i64( [[PRED]], [[PASSTHRU1]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revw.nxv2i64( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revw.nxv2i64( %pred, %passthru1, %1) + ret %2 +} + +; negative test +define @aarch64_sve_rev_mismatch( %0) #0{ +; CHECK-LABEL: define @aarch64_sve_rev_mismatch( +; CHECK-SAME: [[TMP0:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.rev.b16( [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.rev.b32( [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; +entry: + %1 = call @llvm.aarch64.sve.rev.b16( %0) + %2 = call @llvm.aarch64.sve.rev.b32( %1) + ret %2 +} + +; negative test +define @aarch64_sve_rev_mismatch_1( %pred, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_rev_mismatch_1( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revw.nxv2i64( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revh.nxv2i64( [[PRED]], [[PASSTHRU]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.revw.nxv2i64( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revh.nxv2i64( %pred, %passthru, %1) + ret %2 +} + +; negative test +define @aarch64_sve_rev_inv_multi_use( %0) #0 { +; CHECK-LABEL: define @aarch64_sve_rev_inv_multi_use( +; CHECK-SAME: [[TMP0:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.rev.b16( [[TMP0]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.rev.b16( [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = or [[TMP1]], [[TMP2]] +; CHECK-NEXT: ret [[TMP3]] +; +entry: + %1 = call @llvm.aarch64.sve.rev.b16( %0) + %2 = call @llvm.aarch64.sve.rev.b16( %1) + %3 = or %1, %2 + ret %3 +} + + +; negative test +define @aarch64_sve_revw_inv_multi_use( %pred, %passthru, %a) #0 { +; CHECK-LABEL: define @aarch64_sve_revw_inv_multi_use( +; CHECK-SAME: [[PRED:%.*]], [[PASSTHRU:%.*]], [[A:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.revw.nxv2i64( [[PRED]], [[PASSTHRU]], [[A]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.revw.nxv2i64( [[PRED]], [[PASSTHRU]], [[TMP1]]) +; CHECK-NEXT: [[TMP3:%.*]] = or [[TMP1]], [[TMP2]] +; CHECK-NEXT: ret [[TMP3]] +; + %1 = call @llvm.aarch64.sve.revw.nxv2i64( %pred, %passthru, %a) + %2 = call @llvm.aarch64.sve.revw.nxv2i64( %pred, %passthru, %1) + %3 = or %1, %2 + ret %3 +} + +attributes #0 = { "target-features"="+sve" }