@@ -109,12 +109,70 @@ bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
109109  //  expanded instructions for each pseudo is correct in the Size field of the
110110  //  tablegen definition for the pseudo.
111111  switch  (MBBI->getOpcode ()) {
112+   case  RISCV::PseudoAtomicSwap32:
113+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Xchg, false , 32 ,
114+                              NextMBBI);
115+   case  RISCV::PseudoAtomicSwap64:
116+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Xchg, false , 64 ,
117+                              NextMBBI);
118+   case  RISCV::PseudoAtomicLoadAdd32:
119+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Add, false , 32 ,
120+                              NextMBBI);
121+   case  RISCV::PseudoAtomicLoadAdd64:
122+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Add, false , 64 ,
123+                              NextMBBI);
124+   case  RISCV::PseudoAtomicLoadSub32:
125+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Sub, false , 32 ,
126+                              NextMBBI);
127+   case  RISCV::PseudoAtomicLoadSub64:
128+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Sub, false , 64 ,
129+                              NextMBBI);
130+   case  RISCV::PseudoAtomicLoadAnd32:
131+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::And, false , 32 ,
132+                              NextMBBI);
133+   case  RISCV::PseudoAtomicLoadAnd64:
134+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::And, false , 64 ,
135+                              NextMBBI);
136+   case  RISCV::PseudoAtomicLoadOr32:
137+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Or, false , 32 , NextMBBI);
138+   case  RISCV::PseudoAtomicLoadOr64:
139+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Or, false , 64 , NextMBBI);
140+   case  RISCV::PseudoAtomicLoadXor32:
141+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Xor, false , 32 ,
142+                              NextMBBI);
143+   case  RISCV::PseudoAtomicLoadXor64:
144+     return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Xor, false , 64 ,
145+                              NextMBBI);
112146  case  RISCV::PseudoAtomicLoadNand32:
113147    return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Nand, false , 32 ,
114148                             NextMBBI);
115149  case  RISCV::PseudoAtomicLoadNand64:
116150    return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Nand, false , 64 ,
117151                             NextMBBI);
152+   case  RISCV::PseudoAtomicLoadMin32:
153+     return  expandAtomicMinMaxOp (MBB, MBBI, AtomicRMWInst::Min, false , 32 ,
154+                                 NextMBBI);
155+   case  RISCV::PseudoAtomicLoadMin64:
156+     return  expandAtomicMinMaxOp (MBB, MBBI, AtomicRMWInst::Min, false , 64 ,
157+                                 NextMBBI);
158+   case  RISCV::PseudoAtomicLoadMax32:
159+     return  expandAtomicMinMaxOp (MBB, MBBI, AtomicRMWInst::Max, false , 32 ,
160+                                 NextMBBI);
161+   case  RISCV::PseudoAtomicLoadMax64:
162+     return  expandAtomicMinMaxOp (MBB, MBBI, AtomicRMWInst::Max, false , 64 ,
163+                                 NextMBBI);
164+   case  RISCV::PseudoAtomicLoadUMin32:
165+     return  expandAtomicMinMaxOp (MBB, MBBI, AtomicRMWInst::UMin, false , 32 ,
166+                                 NextMBBI);
167+   case  RISCV::PseudoAtomicLoadUMin64:
168+     return  expandAtomicMinMaxOp (MBB, MBBI, AtomicRMWInst::UMin, false , 64 ,
169+                                 NextMBBI);
170+   case  RISCV::PseudoAtomicLoadUMax32:
171+     return  expandAtomicMinMaxOp (MBB, MBBI, AtomicRMWInst::UMax, false , 32 ,
172+                                 NextMBBI);
173+   case  RISCV::PseudoAtomicLoadUMax64:
174+     return  expandAtomicMinMaxOp (MBB, MBBI, AtomicRMWInst::UMax, false , 64 ,
175+                                 NextMBBI);
118176  case  RISCV::PseudoMaskedAtomicSwap32:
119177    return  expandAtomicBinOp (MBB, MBBI, AtomicRMWInst::Xchg, true , 32 ,
120178                             NextMBBI);
@@ -277,6 +335,36 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
277335  switch  (BinOp) {
278336  default :
279337    llvm_unreachable (" Unexpected AtomicRMW BinOp" 
338+   case  AtomicRMWInst::Xchg:
339+     BuildMI (LoopMBB, DL, TII->get (RISCV::ADDI), ScratchReg)
340+         .addReg (IncrReg)
341+         .addImm (0 );
342+     break ;
343+   case  AtomicRMWInst::Add:
344+     BuildMI (LoopMBB, DL, TII->get (RISCV::ADD), ScratchReg)
345+         .addReg (DestReg)
346+         .addReg (IncrReg);
347+     break ;
348+   case  AtomicRMWInst::Sub:
349+     BuildMI (LoopMBB, DL, TII->get (RISCV::SUB), ScratchReg)
350+         .addReg (DestReg)
351+         .addReg (IncrReg);
352+     break ;
353+   case  AtomicRMWInst::And:
354+     BuildMI (LoopMBB, DL, TII->get (RISCV::AND), ScratchReg)
355+         .addReg (DestReg)
356+         .addReg (IncrReg);
357+     break ;
358+   case  AtomicRMWInst::Or:
359+     BuildMI (LoopMBB, DL, TII->get (RISCV::OR), ScratchReg)
360+         .addReg (DestReg)
361+         .addReg (IncrReg);
362+     break ;
363+   case  AtomicRMWInst::Xor:
364+     BuildMI (LoopMBB, DL, TII->get (RISCV::XOR), ScratchReg)
365+         .addReg (DestReg)
366+         .addReg (IncrReg);
367+     break ;
280368  case  AtomicRMWInst::Nand:
281369    BuildMI (LoopMBB, DL, TII->get (RISCV::AND), ScratchReg)
282370        .addReg (DestReg)
@@ -433,38 +521,85 @@ static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
433521      .addReg (ShamtReg);
434522}
435523
436- bool  RISCVExpandAtomicPseudo::expandAtomicMinMaxOp (
437-     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
438-     AtomicRMWInst::BinOp BinOp, bool  IsMasked, int  Width,
439-     MachineBasicBlock::iterator &NextMBBI) {
440-   assert (IsMasked == true  &&
441-          " Should only need to expand masked atomic max/min" 
442-   assert (Width == 32  && " Should never need to expand masked 64-bit operations" 
524+ static  void  doAtomicMinMaxOpExpansion (
525+     const  RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
526+     MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopHeadMBB,
527+     MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
528+     MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int  Width,
529+     const  RISCVSubtarget *STI) {
530+   Register DestReg = MI.getOperand (0 ).getReg ();
531+   Register ScratchReg = MI.getOperand (1 ).getReg ();
532+   Register AddrReg = MI.getOperand (2 ).getReg ();
533+   Register IncrReg = MI.getOperand (3 ).getReg ();
534+   AtomicOrdering Ordering =
535+       static_cast <AtomicOrdering>(MI.getOperand (4 ).getImm ());
443536
444-   MachineInstr &MI = *MBBI;
445-   DebugLoc DL = MI.getDebugLoc ();
446-   MachineFunction *MF = MBB.getParent ();
447-   auto  LoopHeadMBB = MF->CreateMachineBasicBlock (MBB.getBasicBlock ());
448-   auto  LoopIfBodyMBB = MF->CreateMachineBasicBlock (MBB.getBasicBlock ());
449-   auto  LoopTailMBB = MF->CreateMachineBasicBlock (MBB.getBasicBlock ());
450-   auto  DoneMBB = MF->CreateMachineBasicBlock (MBB.getBasicBlock ());
537+   //  .loophead:
538+   //    lr.[w|d] dest, (addr)
539+   //    mv scratch, dest
540+   //    ifnochangeneeded scratch, incr, .looptail
541+   BuildMI (LoopHeadMBB, DL, TII->get (getLRForRMW (Ordering, Width, STI)), DestReg)
542+       .addReg (AddrReg);
543+   BuildMI (LoopHeadMBB, DL, TII->get (RISCV::ADDI), ScratchReg)
544+       .addReg (DestReg)
545+       .addImm (0 );
546+   switch  (BinOp) {
547+   default :
548+     llvm_unreachable (" Unexpected AtomicRMW BinOp" 
549+   case  AtomicRMWInst::Max: {
550+     BuildMI (LoopHeadMBB, DL, TII->get (RISCV::BGE))
551+         .addReg (ScratchReg)
552+         .addReg (IncrReg)
553+         .addMBB (LoopTailMBB);
554+     break ;
555+   }
556+   case  AtomicRMWInst::Min: {
557+     BuildMI (LoopHeadMBB, DL, TII->get (RISCV::BGE))
558+         .addReg (IncrReg)
559+         .addReg (ScratchReg)
560+         .addMBB (LoopTailMBB);
561+     break ;
562+   }
563+   case  AtomicRMWInst::UMax:
564+     BuildMI (LoopHeadMBB, DL, TII->get (RISCV::BGEU))
565+         .addReg (ScratchReg)
566+         .addReg (IncrReg)
567+         .addMBB (LoopTailMBB);
568+     break ;
569+   case  AtomicRMWInst::UMin:
570+     BuildMI (LoopHeadMBB, DL, TII->get (RISCV::BGEU))
571+         .addReg (IncrReg)
572+         .addReg (ScratchReg)
573+         .addMBB (LoopTailMBB);
574+     break ;
575+   }
451576
452-   //  Insert new MBBs. 
453-   MF-> insert (++MBB. getIterator (), LoopHeadMBB); 
454-   MF-> insert (++LoopHeadMBB-> getIterator ( ), LoopIfBodyMBB); 
455-   MF-> insert (++LoopIfBodyMBB-> getIterator (), LoopTailMBB); 
456-   MF-> insert (++LoopTailMBB-> getIterator (), DoneMBB );
577+   //  .loopifbody: 
578+   //    mv scratch, incr 
579+   BuildMI (LoopIfBodyMBB, DL, TII-> get (RISCV::ADDI ), ScratchReg) 
580+       . addReg (IncrReg) 
581+       . addImm ( 0 );
457582
458-   //  Set up successors and transfer remaining instructions to DoneMBB.
459-   LoopHeadMBB->addSuccessor (LoopIfBodyMBB);
460-   LoopHeadMBB->addSuccessor (LoopTailMBB);
461-   LoopIfBodyMBB->addSuccessor (LoopTailMBB);
462-   LoopTailMBB->addSuccessor (LoopHeadMBB);
463-   LoopTailMBB->addSuccessor (DoneMBB);
464-   DoneMBB->splice (DoneMBB->end (), &MBB, MI, MBB.end ());
465-   DoneMBB->transferSuccessors (&MBB);
466-   MBB.addSuccessor (LoopHeadMBB);
583+   //  .looptail:
584+   //    sc.[w|d] scratch, scratch, (addr)
585+   //    bnez scratch, loop
586+   BuildMI (LoopTailMBB, DL, TII->get (getSCForRMW (Ordering, Width, STI)),
587+           ScratchReg)
588+       .addReg (ScratchReg)
589+       .addReg (AddrReg);
590+   BuildMI (LoopTailMBB, DL, TII->get (RISCV::BNE))
591+       .addReg (ScratchReg)
592+       .addReg (RISCV::X0)
593+       .addMBB (LoopHeadMBB);
594+ }
467595
596+ static  void  doMaskedAtomicMinMaxOpExpansion (
597+     const  RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
598+     MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopHeadMBB,
599+     MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
600+     MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int  Width,
601+     const  RISCVSubtarget *STI) {
602+   assert (Width == 32  && " Should never need to expand masked 64-bit operations" 
468603  Register DestReg = MI.getOperand (0 ).getReg ();
469604  Register Scratch1Reg = MI.getOperand (1 ).getReg ();
470605  Register Scratch2Reg = MI.getOperand (2 ).getReg ();
@@ -541,6 +676,44 @@ bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
541676      .addReg (Scratch1Reg)
542677      .addReg (RISCV::X0)
543678      .addMBB (LoopHeadMBB);
679+ }
680+ 
681+ bool  RISCVExpandAtomicPseudo::expandAtomicMinMaxOp (
682+     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
683+     AtomicRMWInst::BinOp BinOp, bool  IsMasked, int  Width,
684+     MachineBasicBlock::iterator &NextMBBI) {
685+ 
686+   MachineInstr &MI = *MBBI;
687+   DebugLoc DL = MI.getDebugLoc ();
688+   MachineFunction *MF = MBB.getParent ();
689+   auto  LoopHeadMBB = MF->CreateMachineBasicBlock (MBB.getBasicBlock ());
690+   auto  LoopIfBodyMBB = MF->CreateMachineBasicBlock (MBB.getBasicBlock ());
691+   auto  LoopTailMBB = MF->CreateMachineBasicBlock (MBB.getBasicBlock ());
692+   auto  DoneMBB = MF->CreateMachineBasicBlock (MBB.getBasicBlock ());
693+ 
694+   //  Insert new MBBs.
695+   MF->insert (++MBB.getIterator (), LoopHeadMBB);
696+   MF->insert (++LoopHeadMBB->getIterator (), LoopIfBodyMBB);
697+   MF->insert (++LoopIfBodyMBB->getIterator (), LoopTailMBB);
698+   MF->insert (++LoopTailMBB->getIterator (), DoneMBB);
699+ 
700+   //  Set up successors and transfer remaining instructions to DoneMBB.
701+   LoopHeadMBB->addSuccessor (LoopIfBodyMBB);
702+   LoopHeadMBB->addSuccessor (LoopTailMBB);
703+   LoopIfBodyMBB->addSuccessor (LoopTailMBB);
704+   LoopTailMBB->addSuccessor (LoopHeadMBB);
705+   LoopTailMBB->addSuccessor (DoneMBB);
706+   DoneMBB->splice (DoneMBB->end (), &MBB, MI, MBB.end ());
707+   DoneMBB->transferSuccessors (&MBB);
708+   MBB.addSuccessor (LoopHeadMBB);
709+ 
710+   if  (!IsMasked)
711+     doAtomicMinMaxOpExpansion (TII, MI, DL, &MBB, LoopHeadMBB, LoopIfBodyMBB,
712+                               LoopTailMBB, DoneMBB, BinOp, Width, STI);
713+   else 
714+     doMaskedAtomicMinMaxOpExpansion (TII, MI, DL, &MBB, LoopHeadMBB,
715+                                     LoopIfBodyMBB, LoopTailMBB, DoneMBB, BinOp,
716+                                     Width, STI);
544717
545718  NextMBBI = MBB.end ();
546719  MI.eraseFromParent ();
0 commit comments