diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index fcc5eb1c05ba0..7b1d203560a27 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -1327,11 +1327,14 @@ static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II) { case Intrinsic::aarch64_sve_umulh: return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_umulh_u); case Intrinsic::aarch64_sve_asr: - return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_asr_u); + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_asr_u) + .setMatchingIROpcode(Instruction::AShr); case Intrinsic::aarch64_sve_lsl: - return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_lsl_u); + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_lsl_u) + .setMatchingIROpcode(Instruction::Shl); case Intrinsic::aarch64_sve_lsr: - return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_lsr_u); + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_lsr_u) + .setMatchingIROpcode(Instruction::LShr); case Intrinsic::aarch64_sve_and: return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_and_u) .setMatchingIROpcode(Instruction::And); @@ -1354,6 +1357,9 @@ static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II) { case Intrinsic::aarch64_sve_and_u: return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( Instruction::And); + case Intrinsic::aarch64_sve_asr_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::AShr); case Intrinsic::aarch64_sve_eor_u: return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( Instruction::Xor); @@ -1369,6 +1375,12 @@ static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II) { case Intrinsic::aarch64_sve_fsub_u: return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( Instruction::FSub); + case Intrinsic::aarch64_sve_lsl_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::Shl); + case Intrinsic::aarch64_sve_lsr_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::LShr); case Intrinsic::aarch64_sve_mul_u: return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( Instruction::Mul); @@ -1571,7 +1583,11 @@ simplifySVEIntrinsicBinOp(InstCombiner &IC, IntrinsicInst &II, else SimpleII = simplifyBinOp(Opc, Op1, Op2, DL); - if (!SimpleII) + // An SVE intrinsic's result is always defined. However, this is not the case + // for its equivalent IR instruction (e.g. when shifting by an amount more + // than the data's bitwidth). Simplifications to an undefined result must be + // ignored to preserve the intrinsic's expected behaviour. + if (!SimpleII || isa(SimpleII)) return std::nullopt; if (IInfo.inactiveLanesAreNotDefined()) diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-simplify-binop.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-simplify-binop.ll index bec5596907063..3aea04b702f4d 100644 --- a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-simplify-binop.ll +++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-simplify-binop.ll @@ -107,7 +107,8 @@ define @constant_mul_u_after_striping_inactive_lanes( @dont_propagate_poison( %pg, %a) #0 { ; CHECK-LABEL: define @dont_propagate_poison( ; CHECK-SAME: [[PG:%.*]], [[A:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: ret poison +; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.and.nxv4i32( [[PG]], poison, splat (i32 1)) +; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.and.nxv4i32( %pg, poison, splat (i32 1)) ret %r diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-simplify-shift.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-simplify-shift.ll index 400dc0dc8f8ee..39e79573628c3 100644 --- a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-simplify-shift.ll +++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-simplify-shift.ll @@ -6,8 +6,7 @@ target triple = "aarch64-unknown-linux-gnu" define @constant_asr_i8_shift_by_0( %pg) #0 { ; CHECK-LABEL: define @constant_asr_i8_shift_by_0( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.asr.nxv16i8( [[PG]], splat (i8 7), zeroinitializer) -; CHECK-NEXT: ret [[R]] +; CHECK-NEXT: ret splat (i8 7) ; %r = call @llvm.aarch64.sve.asr.nxv16i8( %pg, splat (i8 7), splat (i8 0)) ret %r @@ -16,7 +15,7 @@ define @constant_asr_i8_shift_by_0( %pg) #0 define @constant_asr_i8_shift_by_1( %pg) #0 { ; CHECK-LABEL: define @constant_asr_i8_shift_by_1( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.asr.nxv16i8( [[PG]], splat (i8 -63), splat (i8 1)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i8 -32), splat (i8 -63) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.asr.nxv16i8( %pg, splat (i8 193), splat (i8 1)) @@ -27,7 +26,7 @@ define @constant_asr_i8_shift_by_1( %pg) #0 define @constant_asr_i8_shift_by_7( %pg) #0 { ; CHECK-LABEL: define @constant_asr_i8_shift_by_7( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.asr.nxv16i8( [[PG]], splat (i8 -128), splat (i8 7)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i8 -1), splat (i8 -128) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.asr.nxv16i8( %pg, splat (i8 128), splat (i8 7)) @@ -50,7 +49,7 @@ define @constant_asr_i8_shift_by_8( %pg) #0 define @constant_asr_i16_shift_by_15( %pg) #0 { ; CHECK-LABEL: define @constant_asr_i16_shift_by_15( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.asr.nxv8i16( [[PG]], splat (i16 -32768), splat (i16 15)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i16 -1), splat (i16 -32768) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.asr.nxv8i16( %pg, splat (i16 32768), splat (i16 15)) @@ -73,7 +72,7 @@ define @constant_asr_i16_shift_by_16( %pg) # define @constant_asr_i32_shift_by_31( %pg) #0 { ; CHECK-LABEL: define @constant_asr_i32_shift_by_31( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.asr.nxv4i32( [[PG]], splat (i32 -2147483648), splat (i32 31)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i32 -1), splat (i32 -2147483648) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.asr.nxv4i32( %pg, splat (i32 2147483648), splat (i32 31)) @@ -96,7 +95,7 @@ define @constant_asr_i32_shift_by_32( %pg) # define @constant_asr_i64_shift_by_63( %pg) #0 { ; CHECK-LABEL: define @constant_asr_i64_shift_by_63( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.asr.nxv2i64( [[PG]], splat (i64 -9223372036854775808), splat (i64 63)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i64 -1), splat (i64 -9223372036854775808) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.asr.nxv2i64( %pg, splat (i64 9223372036854775808), splat (i64 63)) @@ -118,8 +117,7 @@ define @constant_asr_i64_shift_by_64( %pg) # define @constant_lsl_i8_shift_by_0( %pg) #0 { ; CHECK-LABEL: define @constant_lsl_i8_shift_by_0( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsl.nxv16i8( [[PG]], splat (i8 7), zeroinitializer) -; CHECK-NEXT: ret [[R]] +; CHECK-NEXT: ret splat (i8 7) ; %r = call @llvm.aarch64.sve.lsl.nxv16i8( %pg, splat (i8 7), splat (i8 0)) ret %r @@ -128,7 +126,7 @@ define @constant_lsl_i8_shift_by_0( %pg) #0 define @constant_lsl_i8_shift_by_1( %pg) #0 { ; CHECK-LABEL: define @constant_lsl_i8_shift_by_1( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsl.nxv16i8( [[PG]], splat (i8 -63), splat (i8 1)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i8 -126), splat (i8 -63) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsl.nxv16i8( %pg, splat (i8 193), splat (i8 1)) @@ -139,7 +137,7 @@ define @constant_lsl_i8_shift_by_1( %pg) #0 define @constant_lsl_i8_shift_by_7( %pg) #0 { ; CHECK-LABEL: define @constant_lsl_i8_shift_by_7( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsl.nxv16i8( [[PG]], splat (i8 1), splat (i8 7)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i8 -128), splat (i8 1) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsl.nxv16i8( %pg, splat (i8 1), splat (i8 7)) @@ -161,7 +159,7 @@ define @constant_lsl_i8_shift_by_8( %pg) #0 define @constant_lsl_i16_shift_by_15( %pg) #0 { ; CHECK-LABEL: define @constant_lsl_i16_shift_by_15( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsl.nxv8i16( [[PG]], splat (i16 1), splat (i16 15)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i16 -32768), splat (i16 1) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsl.nxv8i16( %pg, splat (i16 1), splat (i16 15)) @@ -183,7 +181,7 @@ define @constant_lsl_i16_shift_by_16( %pg) # define @constant_lsl_i32_shift_by_31( %pg) #0 { ; CHECK-LABEL: define @constant_lsl_i32_shift_by_31( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsl.nxv4i32( [[PG]], splat (i32 1), splat (i32 31)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i32 -2147483648), splat (i32 1) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsl.nxv4i32( %pg, splat (i32 1), splat (i32 31)) @@ -205,7 +203,7 @@ define @constant_lsl_i32_shift_by_32( %pg) # define @constant_lsl_i64_shift_by_63( %pg) #0 { ; CHECK-LABEL: define @constant_lsl_i64_shift_by_63( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsl.nxv2i64( [[PG]], splat (i64 1), splat (i64 63)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i64 -9223372036854775808), splat (i64 1) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsl.nxv2i64( %pg, splat (i64 1), splat (i64 63)) @@ -226,8 +224,7 @@ define @constant_lsl_i64_shift_by_64( %pg) # define @constant_lsr_i8_shift_by_0( %pg) #0 { ; CHECK-LABEL: define @constant_lsr_i8_shift_by_0( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsr.nxv16i8( [[PG]], splat (i8 7), zeroinitializer) -; CHECK-NEXT: ret [[R]] +; CHECK-NEXT: ret splat (i8 7) ; %r = call @llvm.aarch64.sve.lsr.nxv16i8( %pg, splat (i8 7), splat (i8 0)) ret %r @@ -236,7 +233,7 @@ define @constant_lsr_i8_shift_by_0( %pg) #0 define @constant_lsr_i8_shift_by_1( %pg) #0 { ; CHECK-LABEL: define @constant_lsr_i8_shift_by_1( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsr.nxv16i8( [[PG]], splat (i8 -63), splat (i8 1)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i8 96), splat (i8 -63) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsr.nxv16i8( %pg, splat (i8 193), splat (i8 1)) @@ -247,7 +244,7 @@ define @constant_lsr_i8_shift_by_1( %pg) #0 define @constant_lsr_i8_shift_by_7( %pg) #0 { ; CHECK-LABEL: define @constant_lsr_i8_shift_by_7( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsr.nxv16i8( [[PG]], splat (i8 -128), splat (i8 7)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i8 1), splat (i8 -128) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsr.nxv16i8( %pg, splat (i8 128), splat (i8 7)) @@ -270,7 +267,7 @@ define @constant_lsr_i8_shift_by_8( %pg) #0 define @constant_lsr_i16_shift_by_15( %pg) #0 { ; CHECK-LABEL: define @constant_lsr_i16_shift_by_15( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsr.nxv8i16( [[PG]], splat (i16 -32768), splat (i16 15)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i16 1), splat (i16 -32768) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsr.nxv8i16( %pg, splat (i16 32768), splat (i16 15)) @@ -293,7 +290,7 @@ define @constant_lsr_i16_shift_by_16( %pg) # define @constant_lsr_i32_shift_by_31( %pg) #0 { ; CHECK-LABEL: define @constant_lsr_i32_shift_by_31( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsr.nxv4i32( [[PG]], splat (i32 -2147483648), splat (i32 31)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i32 1), splat (i32 -2147483648) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsr.nxv4i32( %pg, splat (i32 2147483648), splat (i32 31)) @@ -316,7 +313,7 @@ define @constant_lsr_i32_shift_by_32( %pg) # define @constant_lsr_i64_shift_by_63( %pg) #0 { ; CHECK-LABEL: define @constant_lsr_i64_shift_by_63( ; CHECK-SAME: [[PG:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: [[R:%.*]] = call @llvm.aarch64.sve.lsr.nxv2i64( [[PG]], splat (i64 -9223372036854775808), splat (i64 63)) +; CHECK-NEXT: [[R:%.*]] = select [[PG]], splat (i64 1), splat (i64 -9223372036854775808) ; CHECK-NEXT: ret [[R]] ; %r = call @llvm.aarch64.sve.lsr.nxv2i64( %pg, splat (i64 9223372036854775808), splat (i64 63))