fixup! Also add support for lowering to LDBFADD/LDBFMAX/LDBFMIN

Author: jthackray

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -979,14 +979,17 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f16, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f32, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::bf16, LibCall);
 
     setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f16, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f32, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::bf16, LibCall);
 
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f16, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f32, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::bf16, LibCall);
   }
 
   if (Subtarget->hasLSE128()) {

llvm/lib/Target/AArch64/AArch64InstrAtomics.td

@@ -548,9 +548,13 @@ defm atomic_load_fmin  : binary_atomic_op_fp<atomic_load_fmin>;
 defm atomic_load_fmax  : binary_atomic_op_fp<atomic_load_fmax>;
 
 let Predicates = [HasLSFE] in {
-  defm : LDFPOPregister_patterns<"LDFADD", "atomic_load_fadd">;
-  defm : LDFPOPregister_patterns<"LDFMAX", "atomic_load_fmax">;
-  defm : LDFPOPregister_patterns<"LDFMIN", "atomic_load_fmin">;
+  defm : LDFPOPregister_patterns<"LDFADD",   "atomic_load_fadd">;
+  defm : LDFPOPregister_patterns<"LDFMAXNM", "atomic_load_fmax">;
+  defm : LDFPOPregister_patterns<"LDFMINNM", "atomic_load_fmin">;
+
+  defm : LDBFPOPregister_patterns<"LDBFADD",   "atomic_load_fadd">;
+  defm : LDBFPOPregister_patterns<"LDBFMAXNM", "atomic_load_fmax">;
+  defm : LDBFPOPregister_patterns<"LDBFMINNM", "atomic_load_fmin">;
 }
 
 // v8.9a/v9.4a FEAT_LRCPC patterns

llvm/lib/Target/AArch64/AArch64InstrFormats.td

@@ -12509,6 +12509,10 @@ multiclass LDFPOPregister_patterns<string inst, string op> {
   defm : LDFPOPregister_patterns_ord<inst, "D", op, f64, (f64 FPR64:$Rm)>;
 }
 
+multiclass LDBFPOPregister_patterns<string inst, string op> {
+  defm : LDFPOPregister_patterns_ord<inst, "", op, bf16, (bf16 FPR16:$Rm)>;
+}
+
 let Predicates = [HasLSE] in
 multiclass CASregister_patterns_ord_dag<string inst, string suffix, string op,
                                         ValueType vt, dag OLD, dag NEW> {

llvm/lib/Target/AArch64/AArch64InstrInfo.td

@@ -10546,7 +10546,7 @@ let Predicates = [HasLSFE] in {
   defm LDFMAXNML  : AtomicFPLoad<0b01, 0b110, "ldfmaxnml">;
   defm LDFMINNMA  : AtomicFPLoad<0b10, 0b111, "ldfminnma">;
   defm LDFMINNMAL : AtomicFPLoad<0b11, 0b111, "ldfminnmal">;
-  defm LDFMINMN   : AtomicFPLoad<0b00, 0b111, "ldfminnm">;
+  defm LDFMINNM   : AtomicFPLoad<0b00, 0b111, "ldfminnm">;
   defm LDFMINNML  : AtomicFPLoad<0b01, 0b111, "ldfminnml">;
   // BFloat16
   def LDBFADDA    : BaseAtomicFPLoad<FPR16, 0b00, 0b10, 0b000, "ldbfadda">;