[RISCV][GISel] Stop miscompiling Zalasr and properly support it.

Our GISel specific atomic patterns didn't use relaxed_load which
caused Zalasr to miscompile and use normal load/store instructions
without a fence.

Fix this and add the necessary patterns to make Zalasr select correctly.

Author: topperc

llvm/lib/Target/RISCV/RISCVGISel.td

@@ -110,16 +110,15 @@ def : StPat<truncstorei8, SB, GPR, i16>;
 
 let Predicates = [HasAtomicLdSt] in {
   // Prefer unsigned due to no c.lb in Zcb.
-  def : LdPat<atomic_load_aext_8,    LBU, i16>;
-  def : LdPat<atomic_load_nonext_16, LH,  i16>;
+  def : LdPat<relaxed_load<atomic_load_aext_8>,    LBU, i16>;
+  def : LdPat<relaxed_load<atomic_load_nonext_16>, LH,  i16>;
 
-  def : StPat<atomic_store_8,  SB, GPR, i16>;
-  def : StPat<atomic_store_16, SH, GPR, i16>;
+  def : StPat<relaxed_store<atomic_store_8>,  SB, GPR, i16>;
+  def : StPat<relaxed_store<atomic_store_16>, SH, GPR, i16>;
 }
 
 let Predicates = [HasAtomicLdSt, IsRV64] in {
-  def : LdPat<atomic_load_nonext_32, LW, i32>;
-  def : StPat<atomic_store_32, SW, GPR, i32>;
+  def : StPat<relaxed_store<atomic_store_32>, SW, GPR, i32>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -202,3 +201,28 @@ let Predicates = [HasStdExtZbkb, NoStdExtZbb, IsRV64] in {
 def : Pat<(i64 (zext (i16 GPR:$rs))), (PACKW GPR:$rs, (XLenVT X0))>;
 def : Pat<(i32 (zext (i16 GPR:$rs))), (PACKW GPR:$rs, (XLenVT X0))>;
 }
+
+//===----------------------------------------------------------------------===//
+// Zalasr patterns not used by SelectionDAG
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtZalasr] in {
+  // the sequentially consistent loads use
+  //  .aq instead of .aqrl to match the psABI/A.7
+  def : PatLAQ<acquiring_load<atomic_load_aext_8>, LB_AQ, i16>;
+  def : PatLAQ<seq_cst_load<atomic_load_aext_8>, LB_AQ, i16>;
+
+  def : PatLAQ<acquiring_load<atomic_load_nonext_16>, LH_AQ, i16>;
+  def : PatLAQ<seq_cst_load<atomic_load_nonext_16>, LH_AQ, i16>;
+
+  def : PatSRL<releasing_store<atomic_store_8>, SB_RL, i16>;
+  def : PatSRL<seq_cst_store<atomic_store_8>, SB_RL, i16>;
+
+  def : PatSRL<releasing_store<atomic_store_16>, SH_RL, i16>;
+  def : PatSRL<seq_cst_store<atomic_store_16>, SH_RL, i16>;
+}
+
+let Predicates = [HasStdExtZalasr, IsRV64] in {
+  def : PatSRL<releasing_store<atomic_store_32>, SW_RL, i32>;
+  def : PatSRL<seq_cst_store<atomic_store_32>, SW_RL, i32>;
+}

llvm/lib/Target/RISCV/RISCVInstrInfoA.td

@@ -174,15 +174,14 @@ let Predicates = [HasAtomicLdSt] in {
   def : StPat<relaxed_store<atomic_store_8>,  SB, GPR, XLenVT>;
   def : StPat<relaxed_store<atomic_store_16>, SH, GPR, XLenVT>;
   def : StPat<relaxed_store<atomic_store_32>, SW, GPR, XLenVT>;
-}
 
-let Predicates = [HasAtomicLdSt, IsRV32] in {
-  def : LdPat<relaxed_load<atomic_load_nonext_32>, LW>;
+  // Used by GISel for RV32 and RV64.
+  def : LdPat<relaxed_load<atomic_load_nonext_32>, LW, i32>;
 }
 
 let Predicates = [HasAtomicLdSt, IsRV64] in {
-  def : LdPat<relaxed_load<atomic_load_asext_32>, LW>;
-  def : LdPat<relaxed_load<atomic_load_zext_32>, LWU>;
+  def : LdPat<relaxed_load<atomic_load_asext_32>, LW, i64>;
+  def : LdPat<relaxed_load<atomic_load_zext_32>, LWU, i64>;
   def : LdPat<relaxed_load<atomic_load_nonext_64>, LD, i64>;
   def : StPat<relaxed_store<atomic_store_64>, SD, GPR, i64>;
 }

llvm/lib/Target/RISCV/RISCVInstrInfoZalasr.td

@@ -63,13 +63,13 @@ defm SD : SRL_r_aq_rl<0b011, "sd">;
 //===----------------------------------------------------------------------===//
 
 class PatLAQ<SDPatternOperator OpNode, RVInst Inst, ValueType vt = XLenVT>
-    : Pat<(vt (OpNode (vt GPRMemZeroOffset:$rs1))), (Inst GPRMemZeroOffset:$rs1)>;
+    : Pat<(vt (OpNode (XLenVT GPRMemZeroOffset:$rs1))), (Inst GPRMemZeroOffset:$rs1)>;
 
 // n.b. this switches order of arguments
 //  to deal with the fact that SRL has addr, data
 //  while atomic_store has data, addr
 class PatSRL<SDPatternOperator OpNode, RVInst Inst, ValueType vt = XLenVT>
-    : Pat<(OpNode (vt GPR:$rs2), (vt GPRMemZeroOffset:$rs1)),
+    : Pat<(OpNode (vt GPR:$rs2), (XLenVT GPRMemZeroOffset:$rs1)),
           (Inst GPRMemZeroOffset:$rs1, GPR:$rs2)>;
 
 
@@ -92,21 +92,19 @@ let Predicates = [HasStdExtZalasr] in {
 
   def : PatSRL<releasing_store<atomic_store_32>, SW_RL>;
   def : PatSRL<seq_cst_store<atomic_store_32>, SW_RL>;
-} // Predicates = [HasStdExtZalasr]
-
-let Predicates = [HasStdExtZalasr, IsRV32] in {
-  def : PatLAQ<acquiring_load<atomic_load_nonext_32>, LW_AQ>;
-  def : PatLAQ<seq_cst_load<atomic_load_nonext_32>, LW_AQ>;
 
-} // Predicates = [HasStdExtZalasr, IsRV64]
+  // Used by GISel for RV32 and RV64.
+  def : PatLAQ<acquiring_load<atomic_load_nonext_32>, LW_AQ, i32>;
+  def : PatLAQ<seq_cst_load<atomic_load_nonext_32>, LW_AQ, i32>;
+} // Predicates = [HasStdExtZalasr]
 
 let Predicates = [HasStdExtZalasr, IsRV64] in {
-  def : PatLAQ<acquiring_load<atomic_load_asext_32>, LW_AQ>;
-  def : PatLAQ<seq_cst_load<atomic_load_asext_32>, LW_AQ>;
+  def : PatLAQ<acquiring_load<atomic_load_asext_32>, LW_AQ, i64>;
+  def : PatLAQ<seq_cst_load<atomic_load_asext_32>, LW_AQ, i64>;
 
-  def : PatLAQ<acquiring_load<atomic_load_nonext_64>, LD_AQ>;
-  def : PatLAQ<seq_cst_load<atomic_load_nonext_64>, LD_AQ>;
+  def : PatLAQ<acquiring_load<atomic_load_nonext_64>, LD_AQ, i64>;
+  def : PatLAQ<seq_cst_load<atomic_load_nonext_64>, LD_AQ, i64>;
 
-  def : PatSRL<releasing_store<atomic_store_64>, SD_RL>;
-  def : PatSRL<seq_cst_store<atomic_store_64>, SD_RL>;
+  def : PatSRL<releasing_store<atomic_store_64>, SD_RL, i64>;
+  def : PatSRL<seq_cst_store<atomic_store_64>, SD_RL, i64>;
 } // Predicates = [HasStdExtZalasr, IsRV64]