[RISCV] Allow hoisting VXRM writes out of loops speculatively

llvm/lib/Target/RISCV/RISCVFeatures.td

@@ -1453,6 +1453,9 @@ def TuneSiFive7 : SubtargetFeature<"sifive7", "RISCVProcFamily", "SiFive7",
 def TuneVentanaVeyron : SubtargetFeature<"ventana-veyron", "RISCVProcFamily", "VentanaVeyron",
                                          "Ventana Veyron-Series processors">;
 
+def TuneVXRMPipelineFlush : SubtargetFeature<"vxrm-pipeline-flush", "HasVXRMPipelineFlush",
+                                             "true", "VXRM writes causes pipeline flush">;
+
 // Assume that lock-free native-width atomics are available, even if the target
 // and operating system combination would not usually provide them. The user
 // is responsible for providing any necessary __sync implementations. Code

llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp

@@ -109,6 +109,35 @@ class VXRMInfo {
     return VXRMInfo::getUnknown();
   }
 
+  // Calculate the VXRMInfo visible to a block assuming this and Other
+  // are both predecessors. To allow speculatively running WriteVXRM
+  // we will ignore Unknowns if one of this and Other have valid
+  // WriteVXRM. Rationale: WriteVXRM causes a pipeline flush in some
+  // uarchs and moving it outside loops is very important for some
+  // workloads.
+  VXRMInfo intersectAnticipated(const VXRMInfo &Other) const {
+    // If the new value isn't valid, ignore it.
+    if (!Other.isValid())
+      return *this;
+
+    // If this value isn't valid, this must be the first predecessor, use it.
+    if (!isValid())
+      return Other;
+
+    // If either is unknown, the result is the other one.
+    if (isUnknown())
+      return Other;
+    if (Other.isUnknown())
+      return *this;
+
+    // If we have an exact match, return this.
+    if (*this == Other)
+      return *this;
+
+    // Otherwise the result is unknown.
+    return VXRMInfo::getUnknown();
+  }
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Support for debugging, callable in GDB: V->dump()
   LLVM_DUMP_METHOD void dump() const {
@@ -187,7 +216,7 @@ class RISCVInsertWriteVXRM : public MachineFunctionPass {
 private:
   bool computeVXRMChanges(const MachineBasicBlock &MBB);
   void computeAvailable(const MachineBasicBlock &MBB);
-  void computeAnticipated(const MachineBasicBlock &MBB);
+  void computeAnticipated(const MachineFunction &MF, const MachineBasicBlock &MBB);
   void emitWriteVXRM(MachineBasicBlock &MBB);
 };
 
@@ -279,8 +308,9 @@ void RISCVInsertWriteVXRM::computeAvailable(const MachineBasicBlock &MBB) {
   }
 }
 
-void RISCVInsertWriteVXRM::computeAnticipated(const MachineBasicBlock &MBB) {
+void RISCVInsertWriteVXRM::computeAnticipated(const MachineFunction &MF, const MachineBasicBlock &MBB) {
   BlockData &BBInfo = BlockInfo[MBB.getNumber()];
+  const RISCVSubtarget &ST = MF.getSubtarget<RISCVSubtarget>();
 
   BBInfo.InQueue = false;
 
@@ -289,7 +319,11 @@ void RISCVInsertWriteVXRM::computeAnticipated(const MachineBasicBlock &MBB) {
     Anticipated.setUnknown();
   } else {
     for (const MachineBasicBlock *S : MBB.successors())
-      Anticipated =
+      if (ST.hasVXRMPipelineFlush())
+        Anticipated =
+          Anticipated.intersectAnticipated(BlockInfo[S->getNumber()].AnticipatedIn);
+      else
+        Anticipated =
           Anticipated.intersect(BlockInfo[S->getNumber()].AnticipatedIn);
   }
 
@@ -453,7 +487,7 @@ bool RISCVInsertWriteVXRM::runOnMachineFunction(MachineFunction &MF) {
   while (!WorkList.empty()) {
     const MachineBasicBlock &MBB = *WorkList.front();
     WorkList.pop();
-    computeAnticipated(MBB);
+    computeAnticipated(MF, MBB);
   }
 
   // Phase 4 - Emit VXRM writes at the earliest place possible.

llvm/lib/Target/RISCV/RISCVProcessors.td

@@ -277,7 +277,8 @@ def SIFIVE_P470 : RISCVProcessorModel<"sifive-p470", SiFiveP400Model,
                                        FeatureUnalignedScalarMem,
                                        FeatureUnalignedVectorMem]),
                                       !listconcat(SiFiveP400TuneFeatures,
-                                                  [TuneNoSinkSplatOperands])>;
+                                                  [TuneNoSinkSplatOperands,
+                                                   TuneVXRMPipelineFlush])>;
 
 
 def SIFIVE_P670 : RISCVProcessorModel<"sifive-p670", SiFiveP600Model,
@@ -298,6 +299,7 @@ def SIFIVE_P670 : RISCVProcessorModel<"sifive-p670", SiFiveP600Model,
                                        TuneLUIADDIFusion,
                                        TuneAUIPCADDIFusion,
                                        TuneNoSinkSplatOperands,
+                                       TuneVXRMPipelineFlush,
                                        FeaturePostRAScheduler]>;
 
 def SYNTACORE_SCR1_BASE : RISCVProcessorModel<"syntacore-scr1-base",
@@ -510,7 +512,8 @@ def SPACEMIT_X60 : RISCVProcessorModel<"spacemit-x60",
                                        [TuneDLenFactor2,
                                         TuneOptimizedNF2SegmentLoadStore,
                                         TuneOptimizedNF3SegmentLoadStore,
-                                        TuneOptimizedNF4SegmentLoadStore]> {
+                                        TuneOptimizedNF4SegmentLoadStore,
+                                        TuneVXRMPipelineFlush]> {
   let MVendorID = 0x710;
   let MArchID = 0x8000000058000001;
   let MImpID = 0x1000000049772200;