diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td index 2d7a9d6f00bd0..aaf8e1bfc8f0f 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td @@ -548,13 +548,13 @@ defm atomic_load_fmin : binary_atomic_op_fp; defm atomic_load_fmax : binary_atomic_op_fp; let Predicates = [HasLSFE] in { - defm : LDFPOPregister_patterns<"LDFADD", "atomic_load_fadd">; - defm : LDFPOPregister_patterns<"LDFMAXNM", "atomic_load_fmax">; - defm : LDFPOPregister_patterns<"LDFMINNM", "atomic_load_fmin">; + defm : LDFPOPregister_patterns<"FADD", "atomic_load_fadd">; + defm : LDFPOPregister_patterns<"FMAXNM", "atomic_load_fmax">; + defm : LDFPOPregister_patterns<"FMINNM", "atomic_load_fmin">; - defm : LDBFPOPregister_patterns<"LDBFADD", "atomic_load_fadd">; - defm : LDBFPOPregister_patterns<"LDBFMAXNM", "atomic_load_fmax">; - defm : LDBFPOPregister_patterns<"LDBFMINNM", "atomic_load_fmin">; + defm : LDBFPOPregister_patterns<"BFADD", "atomic_load_fadd">; + defm : LDBFPOPregister_patterns<"BFMAXNM", "atomic_load_fmax">; + defm : LDBFPOPregister_patterns<"BFMINNM", "atomic_load_fmin">; } // v8.9a/v9.4a FEAT_LRCPC patterns diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index 255cd0ec5840c..99bd85a448d83 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -12493,19 +12493,39 @@ multiclass LDOPregister_patterns_mod { (i32 (!cast(mod#Wrr) WZR, GPR32:$Rm))>; } +class AtomicStoreMonotonicFrag : PatFrag< + (ops node:$ptr, node:$value), + (atomic_op node:$ptr, node:$value), + [{ return SDValue(N, 0).use_empty() && + cast(N)->getMergedOrdering() == AtomicOrdering::Monotonic; }]>; + +class AtomicStoreReleaseFrag : PatFrag< + (ops node:$ptr, node:$value), + (atomic_op node:$ptr, node:$value), + [{ return SDValue(N, 0).use_empty() && + cast(N)->getMergedOrdering() == AtomicOrdering::Release; }]>; + let Predicates = [HasLSFE] in multiclass LDFPOPregister_patterns_ord_dag { + ValueType vt, dag data> { def : Pat<(!cast(op#"_"#vt#"_monotonic") FPR64:$Rn, data), - (!cast(inst # suffix) data, FPR64:$Rn)>; + (!cast("LD" # inst # suffix) data, FPR64:$Rn)>; def : Pat<(!cast(op#"_"#vt#"_acquire") FPR64:$Rn, data), - (!cast(inst # "A" # suffix) data, FPR64:$Rn)>; + (!cast("LD" # inst # "A" # suffix) data, FPR64:$Rn)>; def : Pat<(!cast(op#"_"#vt#"_release") FPR64:$Rn, data), - (!cast(inst # "L" # suffix) data, FPR64:$Rn)>; + (!cast("LD" # inst # "L" # suffix) data, FPR64:$Rn)>; def : Pat<(!cast(op#"_"#vt#"_acq_rel") FPR64:$Rn, data), - (!cast(inst # "AL" # suffix) data, FPR64:$Rn)>; + (!cast("LD" # inst # "AL" # suffix) data, FPR64:$Rn)>; def : Pat<(!cast(op#"_"#vt#"_seq_cst") FPR64:$Rn, data), - (!cast(inst # "AL" # suffix) data, FPR64:$Rn)>; + (!cast("LD" # inst # "AL" # suffix) data, FPR64:$Rn)>; + + let AddedComplexity = 5 in { + def : Pat<(AtomicStoreMonotonicFrag(op)> FPR64:$Rn, data), + (!cast("ST" # inst # suffix) data, FPR64:$Rn)>; + + def : Pat<(AtomicStoreReleaseFrag(op)> FPR64:$Rn, data), + (!cast("ST" # inst # "L" # suffix) data, FPR64:$Rn)>; + } } multiclass LDFPOPregister_patterns_ord int: return val if aligned else 1 -def all_atomicrmw(f, datatype, atomicrmw_ops): +def all_atomicrmw(f, datatype, atomicrmw_ops, featname): + instr = "atomicrmw" for op in atomicrmw_ops: for aligned in Aligned: for ty, val in datatype: alignval = align(val, aligned) for ordering in ATOMICRMW_ORDERS: name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}" - instr = "atomicrmw" f.write( textwrap.dedent( f""" @@ -211,6 +219,18 @@ def all_atomicrmw(f, datatype, atomicrmw_ops): """ ) ) + if generate_store_opt_test(featname, ordering, op, alignval): + name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}_to_store" + f.write( + textwrap.dedent( + f""" + define dso_local void @{name}(ptr %ptr, {ty} %value) {{ + %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {alignval} + ret void + }} + """ + ) + ) def all_load(f): @@ -340,7 +360,7 @@ def write_lit_tests(feature, datatypes, ops): with open(f"{triple}-atomicrmw-{feat.name}.ll", "w") as f: filter_args = r'--filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"' header(f, triple, [feat], filter_args) - all_atomicrmw(f, datatypes, ops) + all_atomicrmw(f, datatypes, ops, feat.name) # Floating point atomics only supported for atomicrmw currently if feature.test_scope() == "atomicrmw":