Skip to content

Commit d6a688f

Browse files
paulwalker-armpaulwalker-arm
authored
[LLVM][CodeGen][SME] hasB16b16() is not sufficient to prove BFADD availability. (llvm#154143)
The FEAT_SVE_B16B16 arithmetic instructions are only available to streaming mode functions when SME2 is available.
1 parent 460e9a8 commit d6a688f

File tree

15 files changed

+36
-22
lines changed

15 files changed

+36
-22
lines changed

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1770,7 +1770,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
17701770
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
17711771
setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
17721772

1773-
if (Subtarget->hasSVEB16B16()) {
1773+
if (Subtarget->hasSVEB16B16() &&
1774+
Subtarget->isNonStreamingSVEorSME2Available()) {
17741775
setOperationAction(ISD::FADD, VT, Legal);
17751776
setOperationAction(ISD::FMA, VT, Custom);
17761777
setOperationAction(ISD::FMAXIMUM, VT, Custom);
@@ -1792,7 +1793,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
17921793
setOperationPromotedToType(Opcode, MVT::nxv8bf16, MVT::nxv8f32);
17931794
}
17941795

1795-
if (!Subtarget->hasSVEB16B16()) {
1796+
if (!Subtarget->hasSVEB16B16() ||
1797+
!Subtarget->isNonStreamingSVEorSME2Available()) {
17961798
for (auto Opcode : {ISD::FADD, ISD::FMA, ISD::FMAXIMUM, ISD::FMAXNUM,
17971799
ISD::FMINIMUM, ISD::FMINNUM, ISD::FMUL, ISD::FSUB}) {
17981800
setOperationPromotedToType(Opcode, MVT::nxv2bf16, MVT::nxv2f32);
@@ -18161,7 +18163,8 @@ bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(
1816118163
case MVT::f64:
1816218164
return true;
1816318165
case MVT::bf16:
18164-
return VT.isScalableVector() && Subtarget->hasSVEB16B16();
18166+
return VT.isScalableVector() && Subtarget->hasSVEB16B16() &&
18167+
Subtarget->isNonStreamingSVEorSME2Available();
1816518168
default:
1816618169
break;
1816718170
}

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ def HasFuseAES : Predicate<"Subtarget->hasFuseAES()">,
143143
"fuse-aes">;
144144
def HasSVE : Predicate<"Subtarget->isSVEAvailable()">,
145145
AssemblerPredicateWithAll<(all_of FeatureSVE), "sve">;
146-
def HasSVEB16B16 : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16()">,
146+
def HasSVEB16B16 : Predicate<"Subtarget->hasSVEB16B16()">,
147147
AssemblerPredicateWithAll<(all_of FeatureSVEB16B16), "sve-b16b16">;
148148
def HasSVE2 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2()">,
149149
AssemblerPredicateWithAll<(all_of FeatureSVE2), "sve2">;
@@ -248,6 +248,10 @@ def HasSVE_or_SME
248248
: Predicate<"Subtarget->isSVEorStreamingSVEAvailable()">,
249249
AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME),
250250
"sve or sme">;
251+
def HasNonStreamingSVE_or_SME2
252+
: Predicate<"Subtarget->isNonStreamingSVEorSME2Available()">,
253+
AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2),
254+
"sve or sme2">;
251255
def HasNonStreamingSVE_or_SME2p1
252256
: Predicate<"Subtarget->isSVEAvailable() ||"
253257
"(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p1())">,

‎llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4408,7 +4408,7 @@ def : InstAlias<"pfalse\t$Pd", (PFALSE PPRorPNR8:$Pd), 0>;
44084408
// Non-widening BFloat16 to BFloat16 instructions
44094409
//===----------------------------------------------------------------------===//
44104410

4411-
let Predicates = [HasSVEB16B16] in {
4411+
let Predicates = [HasSVEB16B16, HasNonStreamingSVE_or_SME2] in {
44124412
defm BFADD_ZZZ : sve_fp_3op_u_zd_bfloat<0b000, "bfadd", AArch64fadd>;
44134413
defm BFSUB_ZZZ : sve_fp_3op_u_zd_bfloat<0b001, "bfsub", AArch64fsub>;
44144414
defm BFMUL_ZZZ : sve_fp_3op_u_zd_bfloat<0b010, "bfmul", AArch64fmul>;
@@ -4441,17 +4441,17 @@ defm BFMLS_ZZZI : sve_fp_fma_by_indexed_elem_bfloat<"bfmls", 0b11, AArch64fmlsid
44414441
defm BFMUL_ZZZI : sve_fp_fmul_by_indexed_elem_bfloat<"bfmul", AArch64fmulidx>;
44424442

44434443
defm BFCLAMP_ZZZ : sve_fp_clamp_bfloat<"bfclamp", AArch64fclamp>;
4444-
} // End HasSVEB16B16
4444+
} // End HasSVEB16B16, HasNonStreamingSVE_or_SME2
44454445

4446-
let Predicates = [HasSVEB16B16, UseExperimentalZeroingPseudos] in {
4446+
let Predicates = [HasSVEB16B16, HasNonStreamingSVE_or_SME2, UseExperimentalZeroingPseudos] in {
44474447
defm BFADD_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fadd>;
44484448
defm BFSUB_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fsub>;
44494449
defm BFMUL_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fmul>;
44504450
defm BFMAXNM_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fmaxnm>;
44514451
defm BFMINNM_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fminnm>;
44524452
defm BFMIN_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fmin>;
44534453
defm BFMAX_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fmax>;
4454-
} // HasSVEB16B16, UseExperimentalZeroingPseudos
4454+
} // HasSVEB16B16, HasNonStreamingSVE_or_SME2, UseExperimentalZeroingPseudos
44554455

44564456
let Predicates = [HasSVEBFSCALE] in {
44574457
def BFSCALE_ZPZZ : sve_fp_2op_p_zds_bfscale<0b1001, "bfscale", DestructiveBinary>;

‎llvm/lib/Target/AArch64/AArch64Subtarget.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,13 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
212212
return hasSVE() || isStreamingSVEAvailable();
213213
}
214214

215+
/// Returns true if the target has access to either the full range of SVE
216+
/// instructions, or the streaming-compatible subset of SVE instructions
217+
/// available to SME2.
218+
bool isNonStreamingSVEorSME2Available() const {
219+
return isSVEAvailable() || (isSVEorStreamingSVEAvailable() && hasSME2());
220+
}
221+
215222
unsigned getMinVectorRegisterBitWidth() const {
216223
// Don't assume any minimum vector size when PSTATE.SM may not be 0, because
217224
// we don't yet support streaming-compatible codegen support that we trust

‎llvm/test/CodeGen/AArch64/sve-bf16-arith.ll‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2-
; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16
3-
; RUN: llc -mattr=+sve,+bf16,+sve-b16b16 < %s | FileCheck %s --check-prefixes=CHECK,B16B16
4-
; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16
5-
; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,B16B16
2+
; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16
3+
; RUN: llc -mattr=+sve,+bf16,+sve-b16b16 < %s | FileCheck %s --check-prefixes=CHECK,B16B16
4+
; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16
5+
; RUN: llc -mattr=+sme2,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,B16B16
66

77
target triple = "aarch64-unknown-linux-gnu"
88

‎llvm/test/MC/AArch64/SVE2p1/bfadd.s‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
22
// RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
3-
// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
3+
// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
44
// RUN: | FileCheck %s --check-prefix=CHECK-ERROR
55
// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
66
// RUN: | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST

‎llvm/test/MC/AArch64/SVE2p1/bfclamp.s‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
22
// RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
3-
// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
3+
// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
44
// RUN: | FileCheck %s --check-prefix=CHECK-ERROR
55
// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
66
// RUN: | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST

‎llvm/test/MC/AArch64/SVE2p1/bfmax.s‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
22
// RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
3-
// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
3+
// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
44
// RUN: | FileCheck %s --check-prefix=CHECK-ERROR
55
// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
66
// RUN: | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST

‎llvm/test/MC/AArch64/SVE2p1/bfmaxnm.s‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
22
// RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
3-
// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
3+
// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
44
// RUN: | FileCheck %s --check-prefix=CHECK-ERROR
55
// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
66
// RUN: | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST

‎llvm/test/MC/AArch64/SVE2p1/bfmin.s‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
22
// RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
3-
// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
3+
// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
44
// RUN: | FileCheck %s --check-prefix=CHECK-ERROR
55
// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
66
// RUN: | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST

0 commit comments

Comments
 (0)