[RISCV] Xqccmp Code Generation (#128815)

This adds support for Xqccmp to the following passes:
- Prolog Epilog Insertion - reusing much of the existing push/pop logic,
but extending it to cope with frame pointers and reorder the CFI
information correctly.
- Move Merger - extending it to support the `qc.` variants of the
double-move instructions.
- Push/Pop Optimizer - extending it to support the `qc.` variants of the
pop instructions.

The testing is based on existing Zcmp tests, but I have put them in
separate files as some of the Zcmp tests were getting quite long.

Author: lenary

llvm/docs/ReleaseNotes.md

@@ -109,8 +109,9 @@ Changes to the RISC-V Backend
 
 * Adds experimental assembler support for the Qualcomm uC 'Xqcilia` (Large Immediate Arithmetic)
   extension.
-* Adds experimental assembler support for the Qualcomm 'Xqccmp' extension, which
-  is a frame-pointer convention compatible version of Zcmp.
+* Adds experimental assembler and code generation support for the Qualcomm
+  'Xqccmp' extension, which is a frame-pointer convention compatible version of
+  Zcmp.
 * Added non-quadratic ``log-vrgather`` cost model for ``vrgather.vv`` instruction
 
 Changes to the WebAssembly Backend

llvm/lib/Target/RISCV/RISCVFrameLowering.cpp

@@ -750,6 +750,54 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
   }
 }
 
+static bool isPush(unsigned Opcode) {
+  switch (Opcode) {
+  case RISCV::CM_PUSH:
+  case RISCV::QC_CM_PUSH:
+  case RISCV::QC_CM_PUSHFP:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static bool isPop(unsigned Opcode) {
+  // There are other pops but these are the only ones introduced during this
+  // pass.
+  switch (Opcode) {
+  case RISCV::CM_POP:
+  case RISCV::QC_CM_POP:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static unsigned getPushOpcode(RISCVMachineFunctionInfo::PushPopKind Kind,
+                              bool HasFP) {
+  switch (Kind) {
+  case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
+    return RISCV::CM_PUSH;
+  case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
+    return HasFP ? RISCV::QC_CM_PUSHFP : RISCV::QC_CM_PUSH;
+  default:
+    llvm_unreachable("Unhandled PushPopKind");
+  }
+}
+
+static unsigned getPopOpcode(RISCVMachineFunctionInfo::PushPopKind Kind) {
+  // There are other pops but they are introduced later by the Push/Pop
+  // Optimizer.
+  switch (Kind) {
+  case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
+    return RISCV::CM_POP;
+  case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
+    return RISCV::QC_CM_POP;
+  default:
+    llvm_unreachable("Unhandled PushPopKind");
+  }
+}
+
 void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
                                       MachineBasicBlock &MBB) const {
   MachineFrameInfo &MFI = MF.getFrameInfo();
@@ -849,7 +897,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   }
 
   if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
-      FirstFrameSetup->getOpcode() == RISCV::CM_PUSH) {
+      isPush(FirstFrameSetup->getOpcode())) {
     // Use available stack adjustment in push instruction to allocate additional
     // stack space. Align the stack size down to a multiple of 16. This is
     // needed for RVE.
@@ -900,9 +948,15 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     // The frame pointer does need to be reserved from register allocation.
     assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
 
-    RI->adjustReg(MBB, MBBI, DL, FPReg, SPReg,
-                  StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
-                  MachineInstr::FrameSetup, getStackAlign());
+    // Xqccmp with hasFP will update FP using `qc.cm.pushfp`, so we don't need
+    // to update it again, but we do need to emit the `.cfi_def_cfa` below.
+    if (RVFI->getPushPopKind(MF) !=
+        RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp) {
+      RI->adjustReg(
+          MBB, MBBI, DL, FPReg, SPReg,
+          StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
+          MachineInstr::FrameSetup, getStackAlign());
+    }
 
     // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
     unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
@@ -1160,9 +1214,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   // Recover callee-saved registers.
   emitCFIForCSI<CFIRestoreRegisterEmitter>(MBB, MBBI, getUnmanagedCSI(MF, CSI));
 
-  bool ApplyPop = RVFI->isPushable(MF) && MBBI != MBB.end() &&
-                  MBBI->getOpcode() == RISCV::CM_POP;
-  if (ApplyPop) {
+  if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(MBBI->getOpcode())) {
     // Use available stack adjustment in pop instruction to deallocate stack
     // space. Align the stack size down to a multiple of 16. This is needed for
     // RVE.
@@ -1781,7 +1833,8 @@ bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
 
       if (FII != std::end(FixedCSRFIMap)) {
         int64_t Offset;
-        if (RVFI->isPushable(MF))
+        if (RVFI->getPushPopKind(MF) ==
+            RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp)
           Offset = -int64_t(RVFI->getRVPushRegs() - RegNum) * Size;
         else
           Offset = -int64_t(RegNum + 1) * Size;
@@ -1845,9 +1898,10 @@ bool RISCVFrameLowering::spillCalleeSavedRegisters(
     unsigned PushedRegNum = RVFI->getRVPushRegs();
     if (PushedRegNum > 0) {
       // Use encoded number to represent registers to spill.
+      unsigned Opcode = getPushOpcode(RVFI->getPushPopKind(*MF), hasFP(*MF));
       unsigned RegEnc = RISCVZC::encodeRlistNumRegs(PushedRegNum);
       MachineInstrBuilder PushBuilder =
-          BuildMI(MBB, MI, DL, TII.get(RISCV::CM_PUSH))
+          BuildMI(MBB, MI, DL, TII.get(Opcode))
               .setMIFlag(MachineInstr::FrameSetup);
       PushBuilder.addImm(RegEnc);
       PushBuilder.addImm(0);
@@ -2000,9 +2054,10 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters(
   if (RVFI->isPushable(*MF)) {
     unsigned PushedRegNum = RVFI->getRVPushRegs();
     if (PushedRegNum > 0) {
+      unsigned Opcode = getPopOpcode(RVFI->getPushPopKind(*MF));
       unsigned RegEnc = RISCVZC::encodeRlistNumRegs(PushedRegNum);
       MachineInstrBuilder PopBuilder =
-          BuildMI(MBB, MI, DL, TII.get(RISCV::CM_POP))
+          BuildMI(MBB, MI, DL, TII.get(Opcode))
               .setMIFlag(MachineInstr::FrameDestroy);
       // Use encoded number to represent registers to restore.
       PopBuilder.addImm(RegEnc);

llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.cpp

@@ -60,6 +60,27 @@ void yaml::RISCVMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
   MappingTraits<RISCVMachineFunctionInfo>::mapping(YamlIO, *this);
 }
 
+RISCVMachineFunctionInfo::PushPopKind
+RISCVMachineFunctionInfo::getPushPopKind(const MachineFunction &MF) const {
+  // We cannot use fixed locations for the callee saved spill slots if the
+  // function uses a varargs save area.
+  // TODO: Use a separate placement for vararg registers to enable Zcmp.
+  if (VarArgsSaveSize != 0)
+    return PushPopKind::None;
+
+  // Zcmp is not compatible with the frame pointer convention.
+  if (MF.getSubtarget<RISCVSubtarget>().hasStdExtZcmp() &&
+      !MF.getTarget().Options.DisableFramePointerElim(MF))
+    return PushPopKind::StdExtZcmp;
+
+  // Xqccmp is Zcmp but has a push order compatible with the frame-pointer
+  // convention.
+  if (MF.getSubtarget<RISCVSubtarget>().hasVendorXqccmp())
+    return PushPopKind::VendorXqccmp;
+
+  return PushPopKind::None;
+}
+
 void RISCVMachineFunctionInfo::initializeBaseYamlFields(
     const yaml::RISCVMachineFunctionInfo &YamlMFI) {
   VarArgsFrameIndex = YamlMFI.VarArgsFrameIndex;

llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h

@@ -136,13 +136,12 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo {
   unsigned getCalleeSavedStackSize() const { return CalleeSavedStackSize; }
   void setCalleeSavedStackSize(unsigned Size) { CalleeSavedStackSize = Size; }
 
+  enum class PushPopKind { None = 0, StdExtZcmp, VendorXqccmp };
+
+  PushPopKind getPushPopKind(const MachineFunction &MF) const;
+
   bool isPushable(const MachineFunction &MF) const {
-    // We cannot use fixed locations for the callee saved spill slots if the
-    // function uses a varargs save area.
-    // TODO: Use a separate placement for vararg registers to enable Zcmp.
-    return MF.getSubtarget<RISCVSubtarget>().hasStdExtZcmp() &&
-           !MF.getTarget().Options.DisableFramePointerElim(MF) &&
-           VarArgsSaveSize == 0;
+    return getPushPopKind(MF) != PushPopKind::None;
   }
 
   unsigned getRVPushRegs() const { return RVPushRegs; }

llvm/lib/Target/RISCV/RISCVMoveMerger.cpp

@@ -9,10 +9,12 @@
 // This file contains a pass that performs move related peephole optimizations
 // as Zcmp has specified. This pass should be run after register allocation.
 //
+// This pass also supports Xqccmp, which has identical instructions.
+//
 //===----------------------------------------------------------------------===//
 
 #include "RISCVInstrInfo.h"
-#include "RISCVMachineFunctionInfo.h"
+#include "RISCVSubtarget.h"
 
 using namespace llvm;
 
@@ -43,7 +45,7 @@ struct RISCVMoveMerge : public MachineFunctionPass {
   MachineBasicBlock::iterator
   findMatchingInst(MachineBasicBlock::iterator &MBBI, unsigned InstOpcode,
                    const DestSourcePair &RegPair);
-  bool mergeMoveSARegPair(MachineBasicBlock &MBB);
+  bool mergeMoveSARegPair(const RISCVSubtarget &STI, MachineBasicBlock &MBB);
   bool runOnMachineFunction(MachineFunction &Fn) override;
 
   StringRef getPassName() const override { return RISCV_MOVE_MERGE_NAME; }
@@ -56,6 +58,46 @@ char RISCVMoveMerge::ID = 0;
 INITIALIZE_PASS(RISCVMoveMerge, "riscv-move-merge", RISCV_MOVE_MERGE_NAME,
                 false, false)
 
+static bool isMoveFromAToS(unsigned Opcode) {
+  switch (Opcode) {
+  case RISCV::CM_MVA01S:
+  case RISCV::QC_CM_MVA01S:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static unsigned getMoveFromAToSOpcode(const RISCVSubtarget &STI) {
+  if (STI.hasStdExtZcmp())
+    return RISCV::CM_MVA01S;
+
+  if (STI.hasVendorXqccmp())
+    return RISCV::QC_CM_MVA01S;
+
+  llvm_unreachable("Unhandled subtarget with paired A to S move.");
+}
+
+static bool isMoveFromSToA(unsigned Opcode) {
+  switch (Opcode) {
+  case RISCV::CM_MVSA01:
+  case RISCV::QC_CM_MVSA01:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static unsigned getMoveFromSToAOpcode(const RISCVSubtarget &STI) {
+  if (STI.hasStdExtZcmp())
+    return RISCV::CM_MVSA01;
+
+  if (STI.hasVendorXqccmp())
+    return RISCV::QC_CM_MVSA01;
+
+  llvm_unreachable("Unhandled subtarget with paired S to A move");
+}
+
 // Check if registers meet CM.MVA01S constraints.
 bool RISCVMoveMerge::isCandidateToMergeMVA01S(const DestSourcePair &RegPair) {
   Register Destination = RegPair.Destination->getReg();
@@ -87,7 +129,7 @@ RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
   MachineBasicBlock::iterator NextI = next_nodbg(I, E);
   DestSourcePair FirstPair = TII->isCopyInstrImpl(*I).value();
   DestSourcePair PairedRegs = TII->isCopyInstrImpl(*Paired).value();
-  Register ARegInFirstPair = Opcode == RISCV::CM_MVA01S
+  Register ARegInFirstPair = isMoveFromAToS(Opcode)
                                  ? FirstPair.Destination->getReg()
                                  : FirstPair.Source->getReg();
 
@@ -104,7 +146,7 @@ RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
   //   mv a0, s2
   //   mv a1, s1    =>  cm.mva01s s2,s1
   bool StartWithX10 = ARegInFirstPair == RISCV::X10;
-  if (Opcode == RISCV::CM_MVA01S) {
+  if (isMoveFromAToS(Opcode)) {
     Sreg1 = StartWithX10 ? FirstPair.Source : PairedRegs.Source;
     Sreg2 = StartWithX10 ? PairedRegs.Source : FirstPair.Source;
   } else {
@@ -139,8 +181,7 @@ RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,
       Register SourceReg = SecondPair->Source->getReg();
       Register DestReg = SecondPair->Destination->getReg();
 
-      if (InstOpcode == RISCV::CM_MVA01S &&
-          isCandidateToMergeMVA01S(*SecondPair)) {
+      if (isMoveFromAToS(InstOpcode) && isCandidateToMergeMVA01S(*SecondPair)) {
         // If register pair is valid and destination registers are different.
         if ((RegPair.Destination->getReg() == DestReg))
           return E;
@@ -154,7 +195,7 @@ RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,
           return E;
 
         return I;
-      } else if (InstOpcode == RISCV::CM_MVSA01 &&
+      } else if (isMoveFromSToA(InstOpcode) &&
                  isCandidateToMergeMVSA01(*SecondPair)) {
         if ((RegPair.Source->getReg() == SourceReg) ||
             (RegPair.Destination->getReg() == DestReg))
@@ -176,7 +217,8 @@ RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,
 
 // Finds instructions, which could be represented as C.MV instructions and
 // merged into CM.MVA01S or CM.MVSA01.
-bool RISCVMoveMerge::mergeMoveSARegPair(MachineBasicBlock &MBB) {
+bool RISCVMoveMerge::mergeMoveSARegPair(const RISCVSubtarget &STI,
+                                        MachineBasicBlock &MBB) {
   bool Modified = false;
 
   for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
@@ -188,9 +230,9 @@ bool RISCVMoveMerge::mergeMoveSARegPair(MachineBasicBlock &MBB) {
       unsigned Opcode = 0;
 
       if (isCandidateToMergeMVA01S(*RegPair))
-        Opcode = RISCV::CM_MVA01S;
+        Opcode = getMoveFromAToSOpcode(STI);
       else if (isCandidateToMergeMVSA01(*RegPair))
-        Opcode = RISCV::CM_MVSA01;
+        Opcode = getMoveFromSToAOpcode(STI);
       else {
         ++MBBI;
         continue;
@@ -215,7 +257,7 @@ bool RISCVMoveMerge::runOnMachineFunction(MachineFunction &Fn) {
     return false;
 
   const RISCVSubtarget *Subtarget = &Fn.getSubtarget<RISCVSubtarget>();
-  if (!Subtarget->hasStdExtZcmp())
+  if (!(Subtarget->hasStdExtZcmp() || Subtarget->hasVendorXqccmp()))
     return false;
 
   TII = Subtarget->getInstrInfo();
@@ -227,7 +269,7 @@ bool RISCVMoveMerge::runOnMachineFunction(MachineFunction &Fn) {
   UsedRegUnits.init(*TRI);
   bool Modified = false;
   for (auto &MBB : Fn)
-    Modified |= mergeMoveSARegPair(MBB);
+    Modified |= mergeMoveSARegPair(*Subtarget, MBB);
   return Modified;
 }